diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a4fc5b7b4f..aa3b27a513 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,7 +22,6 @@ src/SPIN/* @julient31 src/USER-CGDNA/* @ohenrich src/USER-CGSDK/* @akohlmey src/USER-COLVARS/* @giacomofiorin -src/USER-DPD/* @timattox src/USER-INTEL/* @wmbrownintel src/USER-MANIFOLD/* @Pakketeretet2 src/USER-MEAMC/* @martok @@ -46,7 +45,7 @@ src/GPU/pair_vashishta_gpu.* @andeplane src/KOKKOS/pair_vashishta_kokkos.* @andeplane src/MANYBODY/pair_vashishta_table.* @andeplane src/MANYBODY/pair_atm.* @sergeylishchuk -src/USER-MISC/fix_bond_react.* @jrgissing +src/USER-REACTION/fix_bond_react.* @jrgissing src/USER-MISC/*_grem.* @dstelter92 src/USER-MISC/compute_stress_mop*.* @RomainVermorel @@ -116,9 +115,11 @@ src/min* @sjplimp @stanmoore1 # tools tools/msi2lmp/* @akohlmey tools/emacs/* @HaoZeke +tools/singularity/* @akohlmey @rbberger # cmake cmake/* @junghans @rbberger +cmake/Modules/Packages/USER-COLVARS.cmake @junghans @rbberger @giacomofiorin # python python/* @rbberger diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 91b7e2bf48..0d1d3b0364 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -9,24 +9,24 @@ assignees: '' **Summary** -_Please provide a clear and concise description of what the bug is._ + **LAMMPS Version and Platform** -_Please specify precisely which LAMMPS version this issue was detected with (the first line of the output) and what platform (operating system and its version, hardware) you are running on. If possible, test with the most recent LAMMPS patch version_ + **Expected Behavior** -_Describe the expected behavior. Quote from the LAMMPS manual where needed, or explain why the expected behavior is meaningful, especially when it differs from the manual_ + **Actual Behavior** -_Describe the actual behavior, how it differs from the expected behavior, and how this can be observed. Try to be specific and do **not** use vague terms like "doesn't work" or "wrong result". Do not assume that the person reading this has any experience with or knowledge of your specific area of research._ + **Steps to Reproduce** -_Describe the steps required to (quickly) reproduce the issue. You can attach (small) files to the section below or add URLs where to download an archive with all necessary files. Please try to create an input set that is as minimal and small as possible and reproduces the bug as quickly as possible. **NOTE:** the less effort and time it takes to reproduce your reported bug, the more likely it becomes, that somebody will look into it and fix the problem._ + **Further Information, Files, and Links** -_Put any additional information here, attach relevant text or image files and URLs to external sites, e.g. relevant publications_ + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a3a70c0d5a..0334e086fc 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -9,12 +9,12 @@ assignees: '' **Summary** -_Please provide a brief and concise description of the suggested feature or change_ + **Detailed Description** -_Please explain how you would like to see LAMMPS enhanced, what feature(s) you are looking for, what specific problems this will solve. If possible, provide references to relevant background information like publications or web pages, and whether you are planning to implement the enhancement yourself or would like to participate in the implementation. If applicable add a reference to an existing bug report or issue that this will address._ + **Further Information, Files, and Links** -_Put any additional information here, attach relevant text or image files and URLs to external sites, e.g. relevant publications_ + diff --git a/.github/ISSUE_TEMPLATE/generic.md b/.github/ISSUE_TEMPLATE/generic.md index 15d2763919..9e5d908bbc 100644 --- a/.github/ISSUE_TEMPLATE/generic.md +++ b/.github/ISSUE_TEMPLATE/generic.md @@ -9,13 +9,13 @@ assignees: '' **Summary** -_Please provide a clear and concise description of what this issue report is about._ + **LAMMPS Version and Platform** -_Please specify precisely which LAMMPS version this issue was detected with (the first line of the output) and what platform (operating system and its version, hardware) you are running on. If possible, test with the most recent LAMMPS patch version_ + **Details** -_Please explain the issue in detail here_ + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9e43aa0244..6ef945d8d4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,14 +1,14 @@ **Summary** -_Briefly describe the new feature(s), enhancement(s), or bugfix(es) included in this pull request._ + **Related Issues** -_If this addresses an open GitHub issue for this project, please mention the issue number here, and describe the relation. Use the phrases `fixes #221` or `closes #135`, when you want an issue to be automatically closed when the pull request is merged_ + **Author(s)** -_Please state name and affiliation of the author or authors that should be credited with the changes in this pull request. If this pull request adds new files to the distribution, please also provide a suitable "long-lived" e-mail address (ideally something that can outlive your institution's e-mail, in case you change jobs) for the *corresponding* author, i.e. the person the LAMMPS developers can contact directly with questions and requests related to maintenance and support of this contributed code._ + **Licensing** @@ -16,15 +16,15 @@ By submitting this pull request, I agree, that my contribution will be included **Backward Compatibility** -_Please state whether any changes in the pull request will break backward compatibility for inputs, and - if yes - explain what has been changed and why_ + **Implementation Notes** -_Provide any relevant details about how the changes are implemented, how correctness was verified, how other features - if any - in LAMMPS are affected_ + **Post Submission Checklist** -_Please check the fields below as they are completed **after** the pull request has been submitted. Delete lines that don't apply_ + - [ ] The feature or features in this pull request is complete - [ ] Licensing information is complete @@ -39,6 +39,6 @@ _Please check the fields below as they are completed **after** the pull request **Further Information, Files, and Links** -_Put any additional information here, attach relevant text or image files, and URLs to external sites (e.g. DOIs or webpages)_ + diff --git a/.github/PULL_REQUEST_TEMPLATE/bug_fix.md b/.github/PULL_REQUEST_TEMPLATE/bug_fix.md index cea04c6873..a5a0b8c489 100644 --- a/.github/PULL_REQUEST_TEMPLATE/bug_fix.md +++ b/.github/PULL_REQUEST_TEMPLATE/bug_fix.md @@ -9,15 +9,15 @@ assignees: '' **Summary** -_Briefly describe the bug or bugs, that are eliminated by this pull request._ + **Related Issue(s)** -_If this request addresses or is related to an existing (open) GitHub issue, e.g. a bug report, mention the issue number number here following a pound sign (aka hashmark), e.g.`#222`._ + **Author(s)** -_Please state name and affiliation of the author or authors that should be credited with the changes in this pull request_ + **Licensing** @@ -25,18 +25,18 @@ By submitting this pull request I implicitly accept, that my submission is subje **Backward Compatibility** -_Please state whether any changes in the pull request break backward compatibility for inputs, and - if yes - explain what has been changed and why_ + **Detailed Description** -_Provide any relevant details about how the fixed bug can be reproduced, how the changes are implemented, how correctness was verified, how other features - if any - in LAMMPS are affected_ + ## Post Submission Checklist -_Please check the fields below as they are completed *after* the pull request is submitted_ + - [ ] The code in this pull request is complete - [ ] The source code follows the LAMMPS formatting guidelines ## Further Information, Files, and Links -_Put any additional information here, attach relevant text or image files, and URLs to external sites (e.g. to download input decks for testing)_ + diff --git a/.github/PULL_REQUEST_TEMPLATE/maintenance_refactoring.md b/.github/PULL_REQUEST_TEMPLATE/maintenance_refactoring.md index b3599cb36d..b7bf40a1e8 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintenance_refactoring.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintenance_refactoring.md @@ -9,15 +9,15 @@ assignees: '' **Summary** -_Briefly describe the included changes._ + **Related Issue(s)** -_If this request addresses or is related to an existing (open) GitHub issue, e.g. a bug report, mention the issue number number here following a pound sign (aka hashmark), e.g.`#222`. + **Licensing** @@ -25,11 +25,11 @@ By submitting this pull request I implicitly accept, that my submission is subje **Detailed Description** -_Provide any relevant details about the included changes._ + ## Post Submission Checklist -_Please check the fields below as they are completed *after* the pull request is submitted_ + - [ ] The pull request is complete - [ ] The source code follows the LAMMPS formatting guidelines diff --git a/.github/PULL_REQUEST_TEMPLATE/new_feature.md b/.github/PULL_REQUEST_TEMPLATE/new_feature.md index e9d16a0fc7..563a9be4aa 100644 --- a/.github/PULL_REQUEST_TEMPLATE/new_feature.md +++ b/.github/PULL_REQUEST_TEMPLATE/new_feature.md @@ -9,34 +9,34 @@ assignees: '' **Summary** -_Briefly describe the new feature(s) included in this pull request._ + **Related Issues** -_If this addresses an existing (open) GitHub issue, e.g. a feature request, mention the issue number here following a pound sign (aka hashmark), e.g. `#331`._ + **Author(s)** -_Please state name and affiliation of the author or authors that should be credited with the features added in this pull request. Please provide a suitable "long-lived" e-mail address (e.g. from gmail, yahoo, outlook, etc.) for the *corresponding* author, i.e. the person the LAMMPS developers can contact directly with questions and requests related to maintenance and support of this code. now and in the future_ + **Licensing** -_Please add *yes* or *no* to the following two statements (please contact @lammps/core if you have questions about this)_ + My contribution may be licensed as GPL v2 (default LAMMPS license): My contribution may be licensed as LGPL (for use as a library with proprietary software): **Backward Compatibility** -_Please state if any of the changes in this pull request will affect backward compatibility for inputs, and - if yes - explain what has been changed and why_ + **Implementation Notes** -_Provide any relevant details about how the new features are implemented, how correctness was verified, what platforms (OS, compiler, MPI, hardware, number of processors, accelerator(s)) it was tested on_ + ## Post Submission Checklist -_Please check the fields below as they are completed *after* the pull request has been submitted_ + - [ ] The feature or features in this pull request is complete - [ ] Licensing information is complete @@ -51,6 +51,6 @@ _Please check the fields below as they are completed *after* the pull request ha ## Further Information, Files, and Links -_Put any additional information here, attach relevant text or image files, and URLs to external sites (e.g. DOIs or webpages)_ + diff --git a/.github/PULL_REQUEST_TEMPLATE/update_enhancement.md b/.github/PULL_REQUEST_TEMPLATE/update_enhancement.md index 52ef91e6c1..9edfb587c0 100644 --- a/.github/PULL_REQUEST_TEMPLATE/update_enhancement.md +++ b/.github/PULL_REQUEST_TEMPLATE/update_enhancement.md @@ -9,11 +9,11 @@ assignees: '' **Summary** -_Briefly describe what kind of updates or enhancements for a package or feature are included. If you are not the original author of the package or feature, please mention, whether your contribution was created independently or in collaboration/cooperation with the original author._ + **Author(s)** -_Please state name and affiliation of the author or authors that should be credited with the changes in this pull request_ + **Licensing** @@ -21,15 +21,15 @@ By submitting this pull request I implicitly accept, that my submission is subje **Backward Compatibility** -_Please state whether any changes in the pull request break backward compatibility for inputs, and - if yes - explain what has been changed and why_ + **Implementation Notes** -_Provide any relevant details about how the changes are implemented, how correctness was verified, how other features - if any - in LAMMPS are affected_ + **Post Submission Checklist** -_Please check the fields below as they are completed_ + - [ ] The feature or features in this pull request is complete - [ ] Suitable updates to the existing docs are included - [ ] One or more example input decks are included @@ -37,6 +37,6 @@ _Please check the fields below as they are completed_ **Further Information, Files, and Links** -_Put any additional information here, attach relevant text or image files, and URLs to external sites (e.g. DOIs or webpages)_ + diff --git a/.gitignore b/.gitignore index 52b8212b13..5c90b0f39c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.d *.x *.exe +*.sif *.dll *.pyc __pycache__ diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5e114acbfe..afea41f4bd 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -6,13 +6,16 @@ cmake_minimum_required(VERSION 3.10) project(lammps CXX) set(SOVERSION 0) -get_filename_component(LAMMPS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../src ABSOLUTE) -get_filename_component(LAMMPS_LIB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../lib ABSOLUTE) + +get_filename_component(LAMMPS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/.. ABSOLUTE) get_filename_component(LAMMPS_LIB_BINARY_DIR ${CMAKE_BINARY_DIR}/lib ABSOLUTE) -get_filename_component(LAMMPS_DOC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../doc ABSOLUTE) -get_filename_component(LAMMPS_TOOLS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../tools ABSOLUTE) -get_filename_component(LAMMPS_PYTHON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../python ABSOLUTE) -get_filename_component(LAMMPS_POTENTIALS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../potentials ABSOLUTE) + +set(LAMMPS_SOURCE_DIR ${LAMMPS_DIR}/src) +set(LAMMPS_LIB_SOURCE_DIR ${LAMMPS_DIR}/lib) +set(LAMMPS_DOC_DIR ${LAMMPS_DIR}/doc) +set(LAMMPS_TOOLS_DIR ${LAMMPS_DIR}/tools) +set(LAMMPS_PYTHON_DIR ${LAMMPS_DIR}/python) +set(LAMMPS_POTENTIALS_DIR ${LAMMPS_DIR}/potentials) find_package(Git) @@ -21,17 +24,12 @@ if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "$ENV{HOME}/.local" CACHE PATH "default install path" FORCE ) endif() -# To avoid conflicts with the conventional Makefile build system, we build everything here -file(GLOB LIB_SOURCES ${LAMMPS_SOURCE_DIR}/[^.]*.cpp) -file(GLOB LMP_SOURCES ${LAMMPS_SOURCE_DIR}/main.cpp) -list(REMOVE_ITEM LIB_SOURCES ${LMP_SOURCES}) - # Cmake modules/macros are in a subdirectory to keep this file cleaner set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Modules) include(LAMMPSUtils) -get_lammps_version(${LAMMPS_SOURCE_DIR}/version.h LAMMPS_VERSION) +get_lammps_version(${LAMMPS_SOURCE_DIR}/version.h PROJECT_VERSION) include(PreventInSourceBuilds) @@ -69,9 +67,10 @@ if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") set(CMAKE_TUNE_DEFAULT "-ffast-math -march=native") endif() -# we require C++11 +# we require C++11 without extensions set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) # GNU compiler specific features for testing if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") @@ -85,43 +84,33 @@ endif() ######################################################################## # User input options # ######################################################################## -option(BUILD_EXE "Build lmp binary" ON) -if(BUILD_EXE) - set(LAMMPS_MACHINE "" CACHE STRING "Suffix to append to lmp binary (WON'T enable any features automatically") - mark_as_advanced(LAMMPS_MACHINE) - if(LAMMPS_MACHINE) - set(LAMMPS_MACHINE "_${LAMMPS_MACHINE}") - endif() - set(LAMMPS_BINARY lmp${LAMMPS_MACHINE}) +set(LAMMPS_MACHINE "" CACHE STRING "Suffix to append to lmp binary (WON'T enable any features automatically") +mark_as_advanced(LAMMPS_MACHINE) +if(LAMMPS_MACHINE) + set(LAMMPS_MACHINE "_${LAMMPS_MACHINE}") endif() +set(LAMMPS_BINARY lmp${LAMMPS_MACHINE}) -option(BUILD_LIB "Build LAMMPS library" OFF) -if(BUILD_LIB) - option(BUILD_SHARED_LIBS "Build shared library" OFF) - if(BUILD_SHARED_LIBS) # for all pkg libs, mpi_stubs and linalg - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - endif() - set(LAMMPS_LIB_SUFFIX "" CACHE STRING "Suffix to append to liblammps and pkg-config file") - mark_as_advanced(LAMMPS_LIB_SUFFIX) - if(LAMMPS_LIB_SUFFIX) - set(LAMMPS_LIB_SUFFIX "_${LAMMPS_LIB_SUFFIX}") - endif() +option(BUILD_SHARED_LIBS "Build shared library" OFF) +if(BUILD_SHARED_LIBS) # for all pkg libs, mpi_stubs and linalg + set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() option(BUILD_TOOLS "Build and install LAMMPS tools (msi2lmp, binary2txt, chain)" OFF) -if(NOT BUILD_EXE AND NOT BUILD_LIB) - message(FATAL_ERROR "You need to at least enable one of two following options: BUILD_LIB or BUILD_EXE") -endif() +include(GNUInstallDirs) +file(GLOB ALL_SOURCES ${LAMMPS_SOURCE_DIR}/[^.]*.cpp) +file(GLOB MAIN_SOURCES ${LAMMPS_SOURCE_DIR}/main.cpp) +list(REMOVE_ITEM ALL_SOURCES ${MAIN_SOURCES}) +add_library(lammps ${ALL_SOURCES}) +add_executable(lmp ${MAIN_SOURCES}) +target_link_libraries(lmp PRIVATE lammps) +set_target_properties(lmp PROPERTIES OUTPUT_NAME ${LAMMPS_BINARY}) +install(TARGETS lmp EXPORT LAMMPS_Targets DESTINATION ${CMAKE_INSTALL_BINDIR}) option(CMAKE_VERBOSE_MAKEFILE "Generate verbose Makefiles" OFF) -include(GNUInstallDirs) -set(LAMMPS_LINK_LIBS) -set(LAMMPS_DEPS) -set(LAMMPS_API_DEFINES) - -set(DEFAULT_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE +set(STANDARD_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS DIPOLE GRANULAR KSPACE LATTE MANYBODY MC MESSAGE MISC MOLECULE PERI POEMS QEQ REPLICA RIGID SHOCK SPIN SNAP SRD KIM PYTHON MSCG MPIIO VORONOI USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-MESODPD USER-CGSDK USER-COLVARS @@ -130,33 +119,15 @@ set(DEFAULT_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE USER-NETCDF USER-PHONON USER-PLUMED USER-PTM USER-QTB USER-REACTION USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY USER-UEF USER-VTK USER-QUIP USER-QMMM USER-YAFF USER-ADIOS) -set(ACCEL_PACKAGES USER-OMP KOKKOS OPT USER-INTEL GPU) -foreach(PKG ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +set(SUFFIX_PACKAGES CORESHELL USER-OMP KOKKOS OPT USER-INTEL GPU) +foreach(PKG ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) option(PKG_${PKG} "Build ${PKG} Package" OFF) endforeach() -###################################################### -# download and unpack support binaries for compilation -# of windows binaries. -###################################################### -if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set(LAMMPS_THIRDPARTY_URL "http://download.lammps.org/thirdparty") - file(DOWNLOAD "${LAMMPS_THIRDPARTY_URL}/opencl-win-devel.tar.gz" "${CMAKE_CURRENT_BINARY_DIR}/opencl-win-devel.tar.gz" - EXPECTED_MD5 2c00364888d5671195598b44c2e0d44d) - execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf opencl-win-devel.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86") - set(OpenCL_LIBRARY "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win32/libOpenCL.dll") - elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") - set(OpenCL_LIBRARY "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win64/libOpenCL.dll") - endif() - set(OpenCL_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/include") -endif() - -###################################################### ###################################################### # packages with special compiler needs or external libs ###################################################### -include_directories(${LAMMPS_SOURCE_DIR}) +target_include_directories(lammps PUBLIC $) if(PKG_USER-ADIOS) # The search for ADIOS2 must come before MPI because @@ -164,35 +135,44 @@ if(PKG_USER-ADIOS) # script that defines the MPI::MPI_C target enable_language(C) find_package(ADIOS2 REQUIRED) - list(APPEND LAMMPS_LINK_LIBS adios2::adios2) + target_link_libraries(lammps PRIVATE adios2::adios2) endif() -# do MPI detection after language activation, -# in case MPI for these languages is required -set(MPI_CXX_SKIP_MPICXX TRUE) -find_package(MPI QUIET) -option(BUILD_MPI "Build MPI version" ${MPI_FOUND}) +if(NOT CMAKE_CROSSCOMPILING) + set(MPI_CXX_SKIP_MPICXX TRUE) + find_package(MPI QUIET) + option(BUILD_MPI "Build MPI version" ${MPI_FOUND}) +else() + option(BUILD_MPI "Build MPI version" OFF) +endif() if(BUILD_MPI) - # We use a non-standard procedure to compile with MPI on windows - if (CMAKE_SYSTEM_NAME STREQUAL Windows) + # We use a non-standard procedure to cross-compile with MPI on Windows + if((CMAKE_SYSTEM_NAME STREQUAL Windows) AND CMAKE_CROSSCOMPILING) include(MPI4WIN) + target_link_libraries(lammps PUBLIC MPI::MPI_CXX) else() find_package(MPI REQUIRED) - include_directories(${MPI_CXX_INCLUDE_PATH}) - add_definitions(-DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX=1) - list(APPEND LAMMPS_LINK_LIBS ${MPI_CXX_LIBRARIES}) + target_link_libraries(lammps PUBLIC MPI::MPI_CXX) option(LAMMPS_LONGLONG_TO_LONG "Workaround if your system or MPI version does not recognize 'long long' data types" OFF) if(LAMMPS_LONGLONG_TO_LONG) - add_definitions(-DLAMMPS_LONGLONG_TO_LONG) + target_compile_definitions(lammps PRIVATE -DLAMMPS_LONGLONG_TO_LONG) endif() endif() else() enable_language(C) file(GLOB MPI_SOURCES ${LAMMPS_SOURCE_DIR}/STUBS/mpi.c) add_library(mpi_stubs STATIC ${MPI_SOURCES}) - include_directories(${LAMMPS_SOURCE_DIR}/STUBS) - list(APPEND LAMMPS_LINK_LIBS mpi_stubs) + set_target_properties(mpi_stubs PROPERTIES OUTPUT_NAME lammps_mpi_stubs${LAMMPS_MACHINE}) + target_include_directories(mpi_stubs PUBLIC $ $) + install(FILES ${LAMMPS_SOURCE_DIR}/STUBS/mpi.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps/mpi) + if(BUILD_SHARED_LIBS) + target_link_libraries(lammps PRIVATE mpi_stubs) + target_include_directories(lammps INTERFACE $ $) + else() + target_link_libraries(lammps PUBLIC mpi_stubs) + endif() + add_library(MPI::MPI_CXX ALIAS mpi_stubs) endif() set(LAMMPS_SIZES "smallbig" CACHE STRING "LAMMPS integer sizes (smallsmall: all 32-bit, smallbig: 64-bit #atoms #timesteps, bigbig: also 64-bit imageint, 64-bit atom ids)") @@ -200,8 +180,7 @@ set(LAMMPS_SIZES_VALUES smallbig bigbig smallsmall) set_property(CACHE LAMMPS_SIZES PROPERTY STRINGS ${LAMMPS_SIZES_VALUES}) validate_option(LAMMPS_SIZES LAMMPS_SIZES_VALUES) string(TOUPPER ${LAMMPS_SIZES} LAMMPS_SIZES) -add_definitions(-DLAMMPS_${LAMMPS_SIZES}) -set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -DLAMMPS_${LAMMPS_SIZES}") +target_compile_definitions(lammps PUBLIC -DLAMMPS_${LAMMPS_SIZES}) # posix_memalign is not available on Windows if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") @@ -210,13 +189,12 @@ else() set(LAMMPS_MEMALIGN "64" CACHE STRING "enables the use of the posix_memalign() call instead of malloc() when large chunks or memory are allocated by LAMMPS. Set to 0 to disable") endif() if(NOT ${LAMMPS_MEMALIGN} STREQUAL "0") - add_definitions(-DLAMMPS_MEMALIGN=${LAMMPS_MEMALIGN}) + target_compile_definitions(lammps PRIVATE -DLAMMPS_MEMALIGN=${LAMMPS_MEMALIGN}) endif() option(LAMMPS_EXCEPTIONS "enable the use of C++ exceptions for error messages (useful for library interface)" OFF) if(LAMMPS_EXCEPTIONS) - add_definitions(-DLAMMPS_EXCEPTIONS) - set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -DLAMMPS_EXCEPTIONS") + target_compile_definitions(lammps PUBLIC -DLAMMPS_EXCEPTIONS) endif() # "hard" dependencies between packages resulting @@ -237,16 +215,7 @@ if(OpenMP_FOUND) endif() endif() -# TODO: this is a temporary workaround until a better solution is found. AK 2019-05-30 -# GNU GCC 9.x uses settings incompatible with our use of 'default(none)' in OpenMP pragmas -# where we assume older GCC semantics. For the time being, we disable OpenMP by default -# for GCC 9.x and beyond. People may manually turn it on, but need to run the script -# src/USER-OMP/hack_openmp_for_pgi_gcc9.sh on all sources to make it compatible with gcc 9. -if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.99.9)) - option(BUILD_OMP "Build with OpenMP support" OFF) -else() - option(BUILD_OMP "Build with OpenMP support" ${BUILD_OMP_DEFAULT}) -endif() +option(BUILD_OMP "Build with OpenMP support" ${BUILD_OMP_DEFAULT}) if(BUILD_OMP) find_package(OpenMP REQUIRED) @@ -254,8 +223,18 @@ if(BUILD_OMP) if(NOT HAVE_OMP_H_INCLUDE) message(FATAL_ERROR "Cannot find the 'omp.h' header file required for full OpenMP support") endif() - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + + if (((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.99.9)) OR + ((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.99.9)) OR + ((CMAKE_CXX_COMPILER_ID STREQUAL "Intel") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 18.99.9)) + ) + # GCC 9.x and later plus Clang 10.x and later implement strict OpenMP 4.0 semantics for consts. + # Intel 18.0 was tested to support both, so we switch to OpenMP 4+ from 19.x onward to be safe. + target_compile_definitions(lammps PRIVATE -DLAMMPS_OMP_COMPAT=4) + else() + target_compile_definitions(lammps PRIVATE -DLAMMPS_OMP_COMPAT=3) + endif() + target_link_libraries(lammps PRIVATE OpenMP::OpenMP_CXX) endif() if(PKG_MSCG OR PKG_USER-ATC OR PKG_USER-AWPMD OR PKG_USER-QUIP OR PKG_LATTE) @@ -270,6 +249,7 @@ if(PKG_MSCG OR PKG_USER-ATC OR PKG_USER-AWPMD OR PKG_USER-QUIP OR PKG_LATTE) enable_language(Fortran) file(GLOB LAPACK_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/linalg/[^.]*.[fF]) add_library(linalg STATIC ${LAPACK_SOURCES}) + set_target_properties(linalg PROPERTIES OUTPUT_NAME lammps_linalg${LAMMPS_MACHINE}) set(BLAS_LIBRARIES "$") set(LAPACK_LIBRARIES "$") else() @@ -282,9 +262,13 @@ find_package(JPEG QUIET) option(WITH_JPEG "Enable JPEG support" ${JPEG_FOUND}) if(WITH_JPEG) find_package(JPEG REQUIRED) - add_definitions(-DLAMMPS_JPEG) - include_directories(${JPEG_INCLUDE_DIR}) - list(APPEND LAMMPS_LINK_LIBS ${JPEG_LIBRARIES}) + target_compile_definitions(lammps PRIVATE -DLAMMPS_JPEG) + if(CMAKE_VERSION VERSION_LESS 3.12) + target_include_directories(lammps PRIVATE ${JPEG_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE ${JPEG_LIBRARIES}) + else() + target_link_libraries(lammps PRIVATE JPEG::JPEG) + endif() endif() find_package(PNG QUIET) @@ -297,9 +281,8 @@ endif() if(WITH_PNG) find_package(PNG REQUIRED) find_package(ZLIB REQUIRED) - include_directories(${PNG_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${PNG_LIBRARIES} ${ZLIB_LIBRARIES}) - add_definitions(-DLAMMPS_PNG) + target_link_libraries(lammps PRIVATE PNG::PNG ZLIB::ZLIB) + target_compile_definitions(lammps PRIVATE -DLAMMPS_PNG) endif() find_program(GZIP_EXECUTABLE gzip) @@ -309,7 +292,7 @@ if(WITH_GZIP) if(NOT GZIP_FOUND) message(FATAL_ERROR "gzip executable not found") endif() - add_definitions(-DLAMMPS_GZIP) + target_compile_definitions(lammps PRIVATE -DLAMMPS_GZIP) endif() find_program(FFMPEG_EXECUTABLE ffmpeg) @@ -319,7 +302,7 @@ if(WITH_FFMPEG) if(NOT FFMPEG_FOUND) message(FATAL_ERROR "ffmpeg executable not found") endif() - add_definitions(-DLAMMPS_FFMPEG) + target_compile_definitions(lammps PRIVATE -DLAMMPS_FFMPEG) endif() if(BUILD_SHARED_LIBS) @@ -332,29 +315,25 @@ else() set(CUDA_REQUEST_PIC) endif() -include(Packages/KSPACE) -include(Packages/PYTHON) -include(Packages/VORONOI) -include(Packages/USER-COLVARS) -include(Packages/USER-MOLFILE) -include(Packages/USER-NETCDF) -include(Packages/USER-PLUMED) -include(Packages/USER-QMMM) -include(Packages/USER-QUIP) -include(Packages/USER-SCAFACOS) -include(Packages/USER-SMD) -include(Packages/USER-VTK) -include(Packages/KIM) -include(Packages/LATTE) -include(Packages/MESSAGE) -include(Packages/MSCG) -include(Packages/COMPRESS) +foreach(PKG_WITH_INCL KSPACE PYTHON VORONOI USER-COLVARS USER-MOLFILE USER-NETCDF USER-PLUMED USER-QMMM + USER-QUIP USER-SCAFACOS USER-SMD USER-VTK KIM LATTE MESSAGE MSCG COMPRESS) + if(PKG_${PKG_WITH_INCL}) + include(Packages/${PKG_WITH_INCL}) + endif() +endforeach() set(CMAKE_TUNE_FLAGS "${CMAKE_TUNE_DEFAULT}" CACHE STRING "Compiler specific optimization or instrumentation") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_TUNE_FLAGS}") -if(CMAKE_Fortran_COMPILER) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${CMAKE_TUNE_FLAGS}") -endif() +separate_arguments(CMAKE_TUNE_FLAGS) +include(CheckCXXCompilerFlag) +foreach(_FLAG ${CMAKE_TUNE_FLAGS}) + string(REGEX REPLACE "[=\"]" "" _FLAGX ${_FLAG}) + check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAGX}) + if(COMPILER_SUPPORTS${_FLAGX}) + target_compile_options(lammps PRIVATE ${_FLAG}) + else() + message(WARNING "${_FLAG} found in CMAKE_TUNE_FLAGS, but not supported by the compiler, skipping") + endif() +endforeach() ######################################################################## # Basic system tests (standard libraries, headers, functions, types) # ######################################################################## @@ -367,15 +346,7 @@ endforeach(HEADER) set(MATH_LIBRARIES "m" CACHE STRING "math library") mark_as_advanced( MATH_LIBRARIES ) -include(CheckLibraryExists) -# RB: disabled this check because it breaks with KOKKOS CUDA enabled -#foreach(FUNC sin cos) -# check_library_exists(${MATH_LIBRARIES} ${FUNC} "" FOUND_${FUNC}_${MATH_LIBRARIES}) -# if(NOT FOUND_${FUNC}_${MATH_LIBRARIES}) -# message(FATAL_ERROR "Could not find needed math function - ${FUNC}") -# endif(NOT FOUND_${FUNC}_${MATH_LIBRARIES}) -#endforeach(FUNC) -list(APPEND LAMMPS_LINK_LIBS ${MATH_LIBRARIES}) +target_link_libraries(lammps PRIVATE ${MATH_LIBRARIES}) ###################################### # Generate Basic Style files @@ -386,7 +357,7 @@ RegisterStyles(${LAMMPS_SOURCE_DIR}) ############################################## # add sources of enabled packages ############################################ -foreach(PKG ${DEFAULT_PACKAGES}) +foreach(PKG ${STANDARD_PACKAGES}) set(${PKG}_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/${PKG}) file(GLOB ${PKG}_SOURCES ${${PKG}_SOURCES_DIR}/[^.]*.cpp) @@ -399,8 +370,8 @@ foreach(PKG ${DEFAULT_PACKAGES}) # detects styles in package and adds them to global list RegisterStyles(${${PKG}_SOURCES_DIR}) - list(APPEND LIB_SOURCES ${${PKG}_SOURCES}) - include_directories(${${PKG}_SOURCES_DIR}) + target_sources(lammps PRIVATE ${${PKG}_SOURCES}) + target_include_directories(lammps PRIVATE ${${PKG}_SOURCES_DIR}) endif() RegisterPackages(${${PKG}_SOURCES_DIR}) @@ -409,12 +380,12 @@ endforeach() # packages that need defines set foreach(PKG MPIIO) if(PKG_${PKG}) - add_definitions(-DLMP_${PKG}) + target_compile_definitions(lammps PRIVATE -DLMP_${PKG}) endif() endforeach() # dedicated check for entire contents of accelerator packages -foreach(PKG ${ACCEL_PACKAGES}) +foreach(PKG ${SUFFIX_PACKAGES}) set(${PKG}_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/${PKG}) file(GLOB ${PKG}_SOURCES ${${PKG}_SOURCES_DIR}/[^.]*.cpp) @@ -434,14 +405,11 @@ foreach(SIMPLE_LIB POEMS USER-ATC USER-AWPMD USER-H5MD) string(REGEX REPLACE "^USER-" "" PKG_LIB "${SIMPLE_LIB}") string(TOLOWER "${PKG_LIB}" PKG_LIB) file(GLOB_RECURSE ${PKG_LIB}_SOURCES - ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.F ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.c ${LAMMPS_LIB_SOURCE_DIR}/${PKG_LIB}/[^.]*.cpp) add_library(${PKG_LIB} STATIC ${${PKG_LIB}_SOURCES}) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(${PKG_LIB} mpi4win_build) - endif() - list(APPEND LAMMPS_LINK_LIBS ${PKG_LIB}) + set_target_properties(${PKG_LIB} PROPERTIES OUTPUT_NAME lammps_${PKG_LIB}${LAMMPS_MACHINE}) + target_link_libraries(lammps PRIVATE ${PKG_LIB}) if(PKG_LIB STREQUAL awpmd) target_include_directories(awpmd PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/awpmd/systems/interact ${LAMMPS_LIB_SOURCE_DIR}/awpmd/ivutils/include) elseif(PKG_LIB STREQUAL h5md) @@ -474,30 +442,36 @@ if(PKG_USER-MESONT) endif() if(PKG_USER-AWPMD) - target_link_libraries(awpmd ${LAPACK_LIBRARIES}) + target_link_libraries(awpmd PRIVATE ${LAPACK_LIBRARIES}) endif() if(PKG_USER-ATC) if(LAMMPS_SIZES STREQUAL BIGBIG) message(FATAL_ERROR "The USER-ATC Package is not compatible with -DLAMMPS_BIGBIG") endif() - target_link_libraries(atc ${LAPACK_LIBRARIES}) + target_link_libraries(atc PRIVATE ${LAPACK_LIBRARIES}) + if(BUILD_MPI) + target_link_libraries(atc PRIVATE MPI::MPI_CXX) + else() + target_link_libraries(atc PRIVATE mpi_stubs) + endif() + target_include_directories(atc PRIVATE ${LAMMPS_SOURCE_DIR}) + target_compile_definitions(atc PRIVATE -DLAMMPS_${LAMMPS_SIZES}) endif() -include(Packages/USER-H5MD) +if(PKG_USER-H5MD) + include(Packages/USER-H5MD) +endif() ###################################################################### # packages which selectively include variants based on enabled styles # e.g. accelerator packages ###################################################################### -include(Packages/CORESHELL) -include(Packages/QEQ) -include(Packages/USER-OMP) -include(Packages/USER-SDPD) -include(Packages/KOKKOS) -include(Packages/OPT) -include(Packages/USER-INTEL) -include(Packages/GPU) +foreach(PKG_WITH_INCL CORESHELL QEQ USER-OMP USER-SDPD KOKKOS OPT USER-INTEL GPU) + if(PKG_${PKG_WITH_INCL}) + include(Packages/${PKG_WITH_INCL}) + endif() +endforeach() ###################################################################### # the windows version of LAMMPS requires a couple extra libraries @@ -505,10 +479,7 @@ include(Packages/GPU) # and after everything else that is compiled locally ###################################################################### if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - if(LAMMPS_USE_MPI4WIN) - list(APPEND LAMMPS_LINK_LIBS ${MPI4WIN_LIBRARIES}) - endif() - list(APPEND LAMMPS_LINK_LIBS -lwsock32 -lpsapi) + target_link_libraries(lammps PRIVATE -lwsock32 -lpsapi) endif() ###################################################### @@ -521,14 +492,14 @@ set(LAMMPS_STYLE_HEADERS_DIR ${CMAKE_CURRENT_BINARY_DIR}/styles) GenerateStyleHeaders(${LAMMPS_STYLE_HEADERS_DIR}) GeneratePackagesHeaders(${LAMMPS_STYLE_HEADERS_DIR}) -include_directories(${LAMMPS_STYLE_HEADERS_DIR}) +target_include_directories(lammps PRIVATE ${LAMMPS_STYLE_HEADERS_DIR}) ###################################### # Generate lmpinstalledpkgs.h ###################################### set(temp "#ifndef LMP_INSTALLED_PKGS_H\n#define LMP_INSTALLED_PKGS_H\n") set(temp "${temp}const char * LAMMPS_NS::LAMMPS::installed_packages[] = {\n") -set(temp_PKG_LIST ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +set(temp_PKG_LIST ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) list(SORT temp_PKG_LIST) foreach(PKG ${temp_PKG_LIST}) if(PKG_${PKG}) @@ -544,13 +515,13 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${LAMMPS_STYLE_HE # Generate lmpgitversion.h ###################################### add_custom_target(gitversion COMMAND ${CMAKE_COMMAND} - -DCMAKE_CURRENT_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" + -DLAMMPS_DIR="${LAMMPS_DIR}" -DGIT_EXECUTABLE="${GIT_EXECUTABLE}" -DGIT_FOUND="${GIT_FOUND}" -DLAMMPS_STYLE_HEADERS_DIR="${LAMMPS_STYLE_HEADERS_DIR}" -P ${CMAKE_CURRENT_SOURCE_DIR}/Modules/generate_lmpgitversion.cmake) set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${LAMMPS_STYLE_HEADERS_DIR}/gitversion.h) -list(APPEND LAMMPS_DEPS gitversion) +add_dependencies(lammps gitversion) ########################################### # Actually add executable and lib to build @@ -558,86 +529,65 @@ list(APPEND LAMMPS_DEPS gitversion) get_property(LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) list (FIND LANGUAGES "Fortran" _index) if (${_index} GREATER -1) - list(APPEND LAMMPS_LINK_LIBS ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) + target_link_libraries(lammps PRIVATE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) endif() -list(REMOVE_DUPLICATES LAMMPS_LINK_LIBS) -if(BUILD_LIB) - add_library(lammps ${LIB_SOURCES}) - target_link_libraries(lammps ${LAMMPS_LINK_LIBS}) - if(LAMMPS_DEPS) - add_dependencies(lammps ${LAMMPS_DEPS}) +set(LAMMPS_CXX_HEADERS angle.h atom.h bond.h citeme.h comm.h compute.h dihedral.h domain.h error.h fix.h force.h group.h improper.h + input.h info.h kspace.h lammps.h lattice.h library.h lmppython.h lmptype.h memory.h modify.h neighbor.h neigh_list.h output.h + pair.h pointers.h region.h timer.h universe.h update.h variable.h) +if(LAMMPS_EXCEPTIONS) + list(APPEND LAMMPS_CXX_HEADERS exceptions.h) +endif() + +set_target_properties(lammps PROPERTIES OUTPUT_NAME lammps${LAMMPS_MACHINE}) +set_target_properties(lammps PROPERTIES SOVERSION ${SOVERSION}) +target_include_directories(lammps PUBLIC $) +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps) +foreach(_HEADER ${LAMMPS_CXX_HEADERS}) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${LAMMPS_SOURCE_DIR}/${_HEADER} ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER} DEPENDS ${LAMMPS_SOURCE_DIR}/${_HEADER}) + add_custom_target(${_HEADER} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/includes/lammps/${_HEADER}) + add_dependencies(lammps ${_HEADER}) + if(BUILD_SHARED_LIBS) + install(FILES ${LAMMPS_SOURCE_DIR}/${_HEADER} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) endif() - set(LAMMPS_CXX_HEADERS - ${LAMMPS_SOURCE_DIR}/angle.h - ${LAMMPS_SOURCE_DIR}/atom.h - ${LAMMPS_SOURCE_DIR}/bond.h - ${LAMMPS_SOURCE_DIR}/citeme.h - ${LAMMPS_SOURCE_DIR}/comm.h - ${LAMMPS_SOURCE_DIR}/compute.h - ${LAMMPS_SOURCE_DIR}/dihedral.h - ${LAMMPS_SOURCE_DIR}/domain.h - ${LAMMPS_SOURCE_DIR}/error.h - ${LAMMPS_SOURCE_DIR}/fix.h - ${LAMMPS_SOURCE_DIR}/force.h - ${LAMMPS_SOURCE_DIR}/group.h - ${LAMMPS_SOURCE_DIR}/improper.h - ${LAMMPS_SOURCE_DIR}/input.h - ${LAMMPS_SOURCE_DIR}/kspace.h - ${LAMMPS_SOURCE_DIR}/lammps.h - ${LAMMPS_SOURCE_DIR}/lattice.h - ${LAMMPS_SOURCE_DIR}/lmppython.h - ${LAMMPS_SOURCE_DIR}/memory.h - ${LAMMPS_SOURCE_DIR}/modify.h - ${LAMMPS_SOURCE_DIR}/neighbor.h - ${LAMMPS_SOURCE_DIR}/neigh_list.h - ${LAMMPS_SOURCE_DIR}/output.h - ${LAMMPS_SOURCE_DIR}/pair.h - ${LAMMPS_SOURCE_DIR}/pointers.h - ${LAMMPS_SOURCE_DIR}/region.h - ${LAMMPS_SOURCE_DIR}/timer.h - ${LAMMPS_SOURCE_DIR}/universe.h - ${LAMMPS_SOURCE_DIR}/update.h - ${LAMMPS_SOURCE_DIR}/variable.h) - - set_target_properties(lammps PROPERTIES OUTPUT_NAME lammps${LAMMPS_LIB_SUFFIX}) - set_target_properties(lammps PROPERTIES SOVERSION ${SOVERSION}) - install(TARGETS lammps LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) - install(FILES ${LAMMPS_SOURCE_DIR}/library.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) - install(FILES ${LAMMPS_CXX_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lammps) - configure_file(pkgconfig/liblammps.pc.in ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_LIB_SUFFIX}.pc @ONLY) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_LIB_SUFFIX}.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - configure_file(FindLAMMPS.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/FindLAMMPS${LAMMPS_LIB_SUFFIX}.cmake @ONLY) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/FindLAMMPS${LAMMPS_LIB_SUFFIX}.cmake DESTINATION ${CMAKE_INSTALL_DATADIR}/cmake/Modules) -else() - list(APPEND LMP_SOURCES ${LIB_SOURCES}) -endif() - -if(BUILD_EXE) - add_executable(lmp ${LMP_SOURCES}) - if(BUILD_LIB) - target_link_libraries(lmp lammps) - else() - target_link_libraries(lmp ${LAMMPS_LINK_LIBS}) - if(LAMMPS_DEPS) - add_dependencies(lmp ${LAMMPS_DEPS}) - endif() - endif() - - set_target_properties(lmp PROPERTIES OUTPUT_NAME ${LAMMPS_BINARY}) - install(TARGETS lmp DESTINATION ${CMAKE_INSTALL_BINDIR}) - install(FILES ${LAMMPS_DOC_DIR}/lammps.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 RENAME ${LAMMPS_BINARY}.1) +endforeach() +target_include_directories(lammps INTERFACE $) +add_library(LAMMPS::lammps ALIAS lammps) +get_target_property(LAMMPS_DEFINES lammps INTERFACE_COMPILE_DEFINITIONS) +set(LAMMPS_API_DEFINES) +foreach(_DEF ${LAMMPS_DEFINES}) + set(LAMMPS_API_DEFINES "${LAMMPS_API_DEFINES} -D${_DEF}") +endforeach() +if(BUILD_SHARED_LIBS) + install(TARGETS lammps EXPORT LAMMPS_Targets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + configure_file(pkgconfig/liblammps.pc.in ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_MACHINE}.pc @ONLY) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liblammps${LAMMPS_MACHINE}.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + install(EXPORT LAMMPS_Targets FILE LAMMPS_Targets.cmake NAMESPACE LAMMPS:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) + file(GLOB MODULE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/Modules/Find*.cmake) + install(FILES ${MODULE_FILES} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) + include(CMakePackageConfigHelpers) + configure_file(LAMMPSConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfig.cmake @ONLY) + write_basic_package_version_file("LAMMPSConfigVersion.cmake" VERSION ${PROJECT_VERSION} COMPATIBILITY ExactVersion) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/LAMMPSConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/LAMMPS) endif() +install(FILES ${LAMMPS_DOC_DIR}/lammps.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 RENAME ${LAMMPS_BINARY}.1) if(BUILD_TOOLS) add_executable(binary2txt ${LAMMPS_TOOLS_DIR}/binary2txt.cpp) + target_compile_definitions(binary2txt PRIVATE -DLAMMPS_${LAMMPS_SIZES}) install(TARGETS binary2txt DESTINATION ${CMAKE_INSTALL_BINDIR}) include(CheckGeneratorSupport) if(CMAKE_GENERATOR_SUPPORT_FORTRAN) - enable_language(Fortran) - add_executable(chain.x ${LAMMPS_TOOLS_DIR}/chain.f) - target_link_libraries(chain.x ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) - install(TARGETS chain.x DESTINATION ${CMAKE_INSTALL_BINDIR}) + include(CheckLanguage) + check_language(Fortran) + if(CMAKE_Fortran_COMPILER) + enable_language(Fortran) + add_executable(chain.x ${LAMMPS_TOOLS_DIR}/chain.f) + target_link_libraries(chain.x PRIVATE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES}) + install(TARGETS chain.x DESTINATION ${CMAKE_INSTALL_BINDIR}) + else() + message(WARNING "No suitable Fortran compiler found, skipping building 'chain.x'") + endif() else() message(WARNING "CMake build doesn't support fortran, skipping building 'chain.x'") endif() @@ -646,7 +596,7 @@ if(BUILD_TOOLS) get_filename_component(MSI2LMP_SOURCE_DIR ${LAMMPS_TOOLS_DIR}/msi2lmp/src ABSOLUTE) file(GLOB MSI2LMP_SOURCES ${MSI2LMP_SOURCE_DIR}/[^.]*.c) add_executable(msi2lmp ${MSI2LMP_SOURCES}) - target_link_libraries(msi2lmp m) + target_link_libraries(msi2lmp PRIVATE ${MATH_LIBRARIES}) install(TARGETS msi2lmp DESTINATION ${CMAKE_INSTALL_BINDIR}) install(FILES ${LAMMPS_DOC_DIR}/msi2lmp.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) endif() @@ -676,12 +626,19 @@ install( # conventional build. Only available, if a shared library is built. # This is primarily for people that only want to use the Python wrapper. ############################################################################### -if(BUILD_LIB AND BUILD_SHARED_LIBS) - find_package(PythonInterp) - if (PYTHONINTERP_FOUND) +if(BUILD_SHARED_LIBS) + if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonInterp) # Deprecated since version 3.12 + if(PYTHONINTERP_FOUND) + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() + else() + find_package(Python COMPONENTS Interpreter) + endif() + if (Python_EXECUTABLE) add_custom_target( install-python - ${PYTHON_EXECUTABLE} install.py -v ${LAMMPS_SOURCE_DIR}/version.h + ${Python_EXECUTABLE} install.py -v ${LAMMPS_SOURCE_DIR}/version.h -m ${LAMMPS_PYTHON_DIR}/lammps.py -l ${CMAKE_BINARY_DIR}/liblammps${CMAKE_SHARED_LIBRARY_SUFFIX} WORKING_DIRECTORY ${LAMMPS_PYTHON_DIR} @@ -702,10 +659,17 @@ endif() # LAMMPS for package managers and with different prefix settings. # This requires either a shared library or that the PYTHON package is included. ############################################################################### -if((BUILD_LIB AND BUILD_SHARED_LIBS) OR (PKG_PYTHON)) - find_package(PythonInterp) - if (PYTHONINTERP_FOUND) - execute_process(COMMAND ${PYTHON_EXECUTABLE} +if(BUILD_SHARED_LIBS OR PKG_PYTHON) + if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonInterp) # Deprecated since version 3.12 + if(PYTHONINTERP_FOUND) + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() + else() + find_package(Python COMPONENTS Interpreter) + endif() + if (Python_EXECUTABLE) + execute_process(COMMAND ${Python_EXECUTABLE} -c "import distutils.sysconfig as cg; print(cg.get_python_lib(1,0,prefix='${CMAKE_INSTALL_PREFIX}'))" OUTPUT_VARIABLE PYTHON_DEFAULT_INSTDIR OUTPUT_STRIP_TRAILING_WHITESPACE) set(PYTHON_INSTDIR ${PYTHON_DEFAULT_INSTDIR} CACHE PATH "Installation folder for LAMMPS Python module") @@ -719,67 +683,84 @@ include(CodeCoverage) ############################################################################### # Print package summary ############################################################################### -foreach(PKG ${DEFAULT_PACKAGES} ${ACCEL_PACKAGES}) +foreach(PKG ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) if(PKG_${PKG}) message(STATUS "Building package: ${PKG}") endif() endforeach() -get_directory_property(CPPFLAGS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS) +get_target_property(DEFINES lammps COMPILE_DEFINITIONS) include(FeatureSummary) feature_summary(DESCRIPTION "The following tools and libraries have been found and configured:" WHAT PACKAGES_FOUND) message(STATUS "<<< Build configuration >>> - Build type ${CMAKE_BUILD_TYPE} - Install path ${CMAKE_INSTALL_PREFIX} - Generator ${CMAKE_GENERATOR} using ${CMAKE_MAKE_PROGRAM} - Compilers and Flags: - C++ Compiler ${CMAKE_CXX_COMPILER} - Type ${CMAKE_CXX_COMPILER_ID} - Version ${CMAKE_CXX_COMPILER_VERSION} - C++ Flags ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}} - Defines ${CPPFLAGS}") + Build type: ${CMAKE_BUILD_TYPE} + Install path: ${CMAKE_INSTALL_PREFIX} + Generator: ${CMAKE_GENERATOR} using ${CMAKE_MAKE_PROGRAM} +-- <<< Compilers and Flags: >>> +-- C++ Compiler: ${CMAKE_CXX_COMPILER} + Type: ${CMAKE_CXX_COMPILER_ID} + Version: ${CMAKE_CXX_COMPILER_VERSION} + C++ Flags: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}} + Defines: ${DEFINES}") +get_target_property(OPTIONS lammps COMPILE_OPTIONS) +if(OPTIONS) + message(" Options: ${OPTIONS}") +endif() get_property(LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) list (FIND LANGUAGES "Fortran" _index) if (${_index} GREATER -1) - message(STATUS "Fortran Compiler ${CMAKE_Fortran_COMPILER} - Type ${CMAKE_Fortran_COMPILER_ID} - Version ${CMAKE_Fortran_COMPILER_VERSION} - Fortran Flags ${CMAKE_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS_${BTYPE}}") + message(STATUS "Fortran Compiler: ${CMAKE_Fortran_COMPILER} + Type: ${CMAKE_Fortran_COMPILER_ID} + Version: ${CMAKE_Fortran_COMPILER_VERSION} + Fortran Flags:${CMAKE_Fortran_FLAGS} ${CMAKE_Fortran_FLAGS_${BTYPE}}") endif() list (FIND LANGUAGES "C" _index) if (${_index} GREATER -1) - message(STATUS "C compiler ${CMAKE_C_COMPILER} - Type ${CMAKE_C_COMPILER_ID} - Version ${CMAKE_C_COMPILER_VERSION} - C Flags ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BTYPE}}") + message(STATUS "C compiler: ${CMAKE_C_COMPILER} + Type: ${CMAKE_C_COMPILER_ID} + Version: ${CMAKE_C_COMPILER_VERSION} + C Flags: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BTYPE}}") endif() +message(STATUS "<<< Linker flags: >>>") +message(STATUS "Executable name: ${LAMMPS_BINARY}") if(CMAKE_EXE_LINKER_FLAGS) - message(STATUS "Linker flags: - Executable ${CMAKE_EXE_LINKER_FLAGS}") + message(STATUS "Executable linker flags: ${CMAKE_EXE_LINKER_FLAGS}") endif() if(BUILD_SHARED_LIBS) - message(STATUS "Shared library flags: ${CMAKE_SHARED_LINKER_FLAGS}") + message(STATUS "Shared library flags: ${CMAKE_SHARED_LINKER_FLAGS}") else() - message(STATUS "Static library flags: ${CMAKE_STATIC_LINKER_FLAGS}") + message(STATUS "Static library flags: ${CMAKE_STATIC_LINKER_FLAGS}") endif() -message(STATUS "Link libraries: ${LAMMPS_LINK_LIBS}") if(BUILD_MPI) - message(STATUS "Using MPI with headers in ${MPI_CXX_INCLUDE_PATH} and these libraries: ${MPI_CXX_LIBRARIES};${MPI_Fortran_LIBRARIES}") + message(STATUS "<<< MPI flags >>> +-- MPI_defines: ${MPI_CXX_COMPILE_DEFINITIONS} +-- MPI includes: ${MPI_CXX_INCLUDE_PATH} +-- MPI libraries: ${MPI_CXX_LIBRARIES};${MPI_Fortran_LIBRARIES}") endif() if(PKG_GPU) - message(STATUS "GPU API: ${GPU_API}") + message(STATUS "<<< GPU package settings >>> +-- GPU API: ${GPU_API}") if(GPU_API STREQUAL "CUDA") message(STATUS "GPU architecture: ${GPU_ARCH}") elseif(GPU_API STREQUAL "OPENCL") - message(STATUS "OpenCL parameter tuning: ${OCL_TUNE}") + message(STATUS "OpenCL tuning: ${OCL_TUNE}") + elseif(GPU_API STREQUAL "HIP") + message(STATUS "HIP platform: ${HIP_PLATFORM}") + message(STATUS "HIP architecture: ${HIP_ARCH}") + if(HIP_USE_DEVICE_SORT) + message(STATUS "HIP GPU sorting: on") + else() + message(STATUS "HIP GPU sorting: off") + endif() endif() - message(STATUS "GPU precision: ${GPU_PREC}") + message(STATUS "GPU precision: ${GPU_PREC}") endif() if(PKG_KOKKOS) message(STATUS "Kokkos Arch: ${KOKKOS_ARCH}") endif() if(PKG_KSPACE) - message(STATUS "Using ${FFT} as primary FFT library") + message(STATUS "<<< FFT settings >>> +-- Primary FFT lib: ${FFT}") if(FFT_SINGLE) message(STATUS "Using single precision FFTs") else() @@ -791,7 +772,7 @@ if(PKG_KSPACE) message(STATUS "Using non-threaded FFTs") endif() if(PKG_KOKKOS) - if(KOKKOS_ENABLE_CUDA) + if(Kokkos_ENABLE_CUDA) if (${FFT} STREQUAL "KISS") message(STATUS "Kokkos FFT: KISS") else() diff --git a/cmake/FindLAMMPS.cmake.in b/cmake/FindLAMMPS.cmake.in deleted file mode 100644 index 586df83c2d..0000000000 --- a/cmake/FindLAMMPS.cmake.in +++ /dev/null @@ -1,48 +0,0 @@ -# - Find liblammps -# Find the native liblammps headers and libraries. -# -# The following variables will set: -# LAMMPS_INCLUDE_DIRS - where to find lammps/library.h, etc. -# LAMMPS_LIBRARIES - List of libraries when using lammps. -# LAMMPS_API_DEFINES - lammps library api defines -# LAMMPS_VERSION - lammps library version -# LAMMPS_FOUND - True if liblammps found. -# -# In addition a LAMMPS::LAMMPS imported target is getting created. -# -# LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator -# http://lammps.sandia.gov, Sandia National Laboratories -# Steve Plimpton, sjplimp@sandia.gov -# -# Copyright (2003) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains -# certain rights in this software. This software is distributed under -# the GNU General Public License. -# -# See the README file in the top-level LAMMPS directory. -# - -find_package(PkgConfig) - -pkg_check_modules(PC_LAMMPS liblammps@LAMMPS_LIB_SUFFIX@) -find_path(LAMMPS_INCLUDE_DIR lammps/library.h HINTS ${PC_LAMMPS_INCLUDE_DIRS} @CMAKE_INSTALL_FULL_INCLUDEDIR@) - -set(LAMMPS_VERSION @LAMMPS_VERSION@) -set(LAMMPS_API_DEFINES @LAMMPS_API_DEFINES@) - -find_library(LAMMPS_LIBRARY NAMES lammps@LAMMPS_LIB_SUFFIX@ HINTS ${PC_LAMMPS_LIBRARY_DIRS} @CMAKE_INSTALL_FULL_LIBDIR@) - -set(LAMMPS_INCLUDE_DIRS "${LAMMPS_INCLUDE_DIR}") -set(LAMMPS_LIBRARIES "${LAMMPS_LIBRARY}") - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set LAMMPS_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(LAMMPS REQUIRED_VARS LAMMPS_LIBRARY LAMMPS_INCLUDE_DIR VERSION_VAR LAMMPS_VERSION) - -mark_as_advanced(LAMMPS_INCLUDE_DIR LAMMPS_LIBRARY) - -if(LAMMPS_FOUND AND NOT TARGET LAMMPS::LAMMPS) - add_library(LAMMPS::LAMMPS UNKNOWN IMPORTED) - set_target_properties(LAMMPS::LAMMPS PROPERTIES IMPORTED_LOCATION "${LAMMPS_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${LAMMPS_INCLUDE_DIR}" INTERFACE_COMPILE_DEFINITIONS "${LAMMPS_API_DEFINES}") -endif() diff --git a/cmake/LAMMPSConfig.cmake.in b/cmake/LAMMPSConfig.cmake.in new file mode 100644 index 0000000000..48c262c664 --- /dev/null +++ b/cmake/LAMMPSConfig.cmake.in @@ -0,0 +1,11 @@ +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) +include(CMakeFindDependencyMacro) +if(@BUILD_MPI@) + find_dependency(MPI REQUIRED CXX) +endif() +if(@PKG_KSPACE@) + if(@FFT@ STREQUAL "FFTW3") + find_dependency(@FFTW@ REQUIRED MODULE) + endif() +endif() +include("${CMAKE_CURRENT_LIST_DIR}/LAMMPS_Targets.cmake") diff --git a/cmake/Modules/Documentation.cmake b/cmake/Modules/Documentation.cmake index feff66a9b2..929d6ebd4b 100644 --- a/cmake/Modules/Documentation.cmake +++ b/cmake/Modules/Documentation.cmake @@ -1,10 +1,8 @@ ############################################################################### # Build documentation ############################################################################### -option(BUILD_DOC "Build LAMMPS documentation" OFF) +option(BUILD_DOC "Build LAMMPS HTML documentation" OFF) if(BUILD_DOC) - include(ProcessorCount) - ProcessorCount(NPROCS) find_package(PythonInterp 3 REQUIRED) set(VIRTUALENV ${PYTHON_EXECUTABLE} -m virtualenv) @@ -26,15 +24,49 @@ if(BUILD_DOC) COMMAND ${DOCENV_BINARY_DIR}/pip install --upgrade ${LAMMPS_DOC_DIR}/utils/converters ) + # download mathjax distribution and unpack to folder "mathjax" + file(DOWNLOAD "https://github.com/mathjax/MathJax/archive/3.0.5.tar.gz" + "${CMAKE_CURRENT_BINARY_DIR}/mathjax.tar.gz" + EXPECTED_MD5 5d9d3799cce77a1a95eee6be04eb68e7) + + if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/mathjax) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf mathjax.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + file(GLOB MATHJAX_VERSION_DIR ${CMAKE_CURRENT_BINARY_DIR}/MathJax-*) + execute_process(COMMAND ${CMAKE_COMMAND} -E rename ${MATHJAX_VERSION_DIR} ${CMAKE_CURRENT_BINARY_DIR}/mathjax) + endif() + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html/_static/mathjax) + file(COPY ${CMAKE_CURRENT_BINARY_DIR}/mathjax/es5 DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/html/_static/mathjax/) + + # note, this may run in parallel with other tasks, so we must not use multiple processes here add_custom_command( OUTPUT html DEPENDS ${DOC_SOURCES} docenv requirements.txt - COMMAND ${DOCENV_BINARY_DIR}/sphinx-build -j ${NPROCS} -b html -c ${LAMMPS_DOC_DIR}/utils/sphinx-config -d ${CMAKE_BINARY_DIR}/doctrees ${LAMMPS_DOC_DIR}/src html + COMMAND ${DOCENV_BINARY_DIR}/sphinx-build -b html -c ${LAMMPS_DOC_DIR}/utils/sphinx-config -d ${CMAKE_BINARY_DIR}/doctrees ${LAMMPS_DOC_DIR}/src html ) + # copy selected image files to html output tree + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/html/JPG) + set(HTML_EXTRA_IMAGES balance_nonuniform.jpg balance_rcb.jpg + balance_uniform.jpg bow_tutorial_01.png bow_tutorial_02.png + bow_tutorial_03.png bow_tutorial_04.png bow_tutorial_05.png + dump1.jpg dump2.jpg examples_mdpd.gif gran_funnel.png gran_mixer.png + hop1.jpg hop2.jpg saed_ewald_intersect.jpg saed_mesh.jpg + screenshot_atomeye.jpg screenshot_gl.jpg screenshot_pymol.jpg + screenshot_vmd.jpg sinusoid.jpg xrd_mesh.jpg) + set(HTML_IMAGE_TARGETS "") + foreach(_IMG ${HTML_EXTRA_IMAGES}) + string(PREPEND _IMG JPG/) + list(APPEND HTML_IMAGE_TARGETS "html/${_IMG}") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/html/${_IMG} + DEPENDS ${LAMMPS_DOC_DIR}/src/${_IMG} html/JPG + COMMAND ${CMAKE_COMMAND} -E copy ${LAMMPS_DOC_DIR}/src/${_IMG} ${CMAKE_BINARY_DIR}/html/${_IMG} + ) + endforeach() + add_custom_target( doc ALL - DEPENDS html + DEPENDS html html/_static/mathjax/es5 ${HTML_IMAGE_TARGETS} SOURCES ${LAMMPS_DOC_DIR}/utils/requirements.txt ${DOC_SOURCES} ) diff --git a/cmake/Modules/FindCUB.cmake b/cmake/Modules/FindCUB.cmake new file mode 100644 index 0000000000..848e68e815 --- /dev/null +++ b/cmake/Modules/FindCUB.cmake @@ -0,0 +1,16 @@ +# - Find CUB +# Find the CUB header library +# +# CUB_INCLUDE_DIRS - where to find cub/cub.cuh +# CUB_FOUND - True if CUB found. +# + +find_path(CUB_INCLUDE_DIR cub.cuh PATH_SUFFIXES cub) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set CUB_FOUND to TRUE +# if all listed variables are TRUE + +find_package_handle_standard_args(CUB DEFAULT_MSG CUB_INCLUDE_DIR) + +mark_as_advanced(CUB_INCLUDE_DIR) diff --git a/cmake/Modules/FindFFTW3.cmake b/cmake/Modules/FindFFTW3.cmake index 63752f85df..708ec49509 100644 --- a/cmake/Modules/FindFFTW3.cmake +++ b/cmake/Modules/FindFFTW3.cmake @@ -14,14 +14,34 @@ find_path(FFTW3_INCLUDE_DIR fftw3.h HINTS ${PC_FFTW3_INCLUDE_DIRS}) find_library(FFTW3_LIBRARY NAMES fftw3 HINTS ${PC_FFTW3_LIBRARY_DIRS}) find_library(FFTW3_OMP_LIBRARY NAMES fftw3_omp HINTS ${PC_FFTW3_LIBRARY_DIRS}) -set(FFTW3_INCLUDE_DIRS ${FFTW3_INCLUDE_DIR}) -set(FFTW3_LIBRARIES ${FFTW3_LIBRARY}) -set(FFTW3_OMP_LIBRARIES ${FFTW3_OMP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set FFTW3_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(FFTW3 DEFAULT_MSG FFTW3_LIBRARY FFTW3_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(FFTW3_FOUND) + set(FFTW3_LIBRARIES ${FFTW3_LIBRARY} ) + set(FFTW3_INCLUDE_DIRS ${FFTW3_INCLUDE_DIR} ) + + if(NOT TARGET FFTW3::FFTW3) + add_library(FFTW3::FFTW3 UNKNOWN IMPORTED) + set_target_properties(FFTW3::FFTW3 PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3_INCLUDE_DIRS}") + endif() + if(FFTW3_OMP_LIBRARY) + set(FFTW3_OMP_LIBRARIES ${FFTW3_OMP_LIBRARY}) + if(NOT TARGET FFTW3::FFTW3_OMP) + add_library(FFTW3::FFTW3_OMP UNKNOWN IMPORTED) + set_target_properties(FFTW3::FFTW3_OMP PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3_OMP_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3_INCLUDE_DIRS}") + endif() + endif() +endif() + mark_as_advanced(FFTW3_INCLUDE_DIR FFTW3_LIBRARY FFTW3_OMP_LIBRARY) diff --git a/cmake/Modules/FindFFTW3F.cmake b/cmake/Modules/FindFFTW3F.cmake index c67aa5faf1..3dbcdaa04e 100644 --- a/cmake/Modules/FindFFTW3F.cmake +++ b/cmake/Modules/FindFFTW3F.cmake @@ -13,14 +13,34 @@ find_path(FFTW3F_INCLUDE_DIR fftw3.h HINTS ${PC_FFTW3F_INCLUDE_DIRS}) find_library(FFTW3F_LIBRARY NAMES fftw3f HINTS ${PC_FFTW3F_LIBRARY_DIRS}) find_library(FFTW3F_OMP_LIBRARY NAMES fftw3f_omp HINTS ${PC_FFTW3F_LIBRARY_DIRS}) -set(FFTW3F_INCLUDE_DIRS ${FFTW3F_INCLUDE_DIR}) -set(FFTW3F_LIBRARIES ${FFTW3F_LIBRARY}) -set(FFTW3F_OMP_LIBRARIES ${FFTW3F_OMP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set FFTW3F_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(FFTW3F DEFAULT_MSG FFTW3F_LIBRARY FFTW3F_INCLUDE_DIR) -mark_as_advanced(FFTW3F_INCLUDE_DIR FFTW3F_LIBRARY) +# Copy the results to the output variables and target. +if(FFTW3F_FOUND) + set(FFTW3F_LIBRARIES ${FFTW3F_LIBRARY} ) + set(FFTW3F_INCLUDE_DIRS ${FFTW3F_INCLUDE_DIR} ) + + if(NOT TARGET FFTW3F::FFTW3F) + add_library(FFTW3F::FFTW3F UNKNOWN IMPORTED) + set_target_properties(FFTW3F::FFTW3F PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3F_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3F_INCLUDE_DIRS}") + endif() + if(FFTW3F_OMP_LIBRARY) + set(FFTW3F_OMP_LIBRARIES ${FFTW3F_OMP_LIBRARY}) + if(NOT TARGET FFTW3F::FFTW3F_OMP) + add_library(FFTW3F::FFTW3F_OMP UNKNOWN IMPORTED) + set_target_properties(FFTW3F::FFTW3F_OMP PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${FFTW3F_OMP_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${FFTW3F_INCLUDE_DIRS}") + endif() + endif() +endif() + +mark_as_advanced(FFTW3F_INCLUDE_DIR FFTW3F_LIBRARY FFTW3F_OMP_LIBRARY) diff --git a/cmake/Modules/FindKIM-API.cmake b/cmake/Modules/FindKIM-API.cmake deleted file mode 100644 index 17c195e62f..0000000000 --- a/cmake/Modules/FindKIM-API.cmake +++ /dev/null @@ -1,85 +0,0 @@ -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the Common Development -# and Distribution License Version 1.0 (the "License"). -# -# You can obtain a copy of the license at -# http://www.opensource.org/licenses/CDDL-1.0. See the License for the -# specific language governing permissions and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each file and -# include the License file in a prominent location with the name LICENSE.CDDL. -# If applicable, add the following below this CDDL HEADER, with the fields -# enclosed by brackets "[]" replaced with your own identifying information: -# -# Portions Copyright (c) [yyyy] [name of copyright owner]. All rights reserved. -# -# CDDL HEADER END -# - -# -# Copyright (c) 2013--2019, Regents of the University of Minnesota. -# All rights reserved. -# -# Contributors: -# Richard Berger -# Christoph Junghans -# Ryan S. Elliott -# - -# - Find KIM-API -# -# sets standard pkg_check_modules variables plus: -# -# KIM-API-CMAKE_C_COMPILER -# KIM-API-CMAKE_CXX_COMPILER -# KIM-API-CMAKE_Fortran_COMPILER -# - -function(_KIMAPI_GET_VERSION _OUT_ver _version_hdr) - if(NOT EXISTS ${_version_hdr}) - message(FATAL_ERROR "Header file ${_version_hdr} not found (check value of KIM-API_INCLUDE_DIR)") - endif() - foreach(_var KIM_VERSION_MAJOR KIM_VERSION_MINOR KIM_VERSION_PATCH) - file(STRINGS ${_version_hdr} _contents REGEX "#define ${_var}[ \t]+") - if(_contents) - string(REGEX REPLACE ".*#define ${_var}[ \t]+([0-9]+).*" "\\1" _${_var} "${_contents}") - if(${_${_var}} STREQUAL "") - message(FATAL_ERROR "Version parsing failed for ${_var} in ${_version_hdr}, got empty return!") - elseif(NOT ${_${_var}} MATCHES "^[0-9]+$") - message(FATAL_ERROR "Version parsing failed for ${_var} in ${_version_hdr}, excepted a number but got ${_${_var}}!") - endif() - else() - message(FATAL_ERROR "No ${_var} line found in include file ${_version_hdr}") - endif() - endforeach() - set(${_OUT_ver} ${_KIM_VERSION_MAJOR}.${_KIM_VERSION_MINOR}.${_KIM_VERSION_PATCH} PARENT_SCOPE) -endfunction() - -if(KIM-API_FIND_QUIETLY) - set(REQ_OR_QUI "QUIET") -else() - set(REQ_OR_QUI "REQUIRED") -endif() - -find_package(PkgConfig ${REQ_OR_QUI}) -include(FindPackageHandleStandardArgs) - -pkg_check_modules(KIM-API ${REQ_OR_QUI} libkim-api>=2.0) - -if(KIM-API_FOUND) - pkg_get_variable(KIM-API-CMAKE_C_COMPILER libkim-api CMAKE_C_COMPILER) - pkg_get_variable(KIM-API-CMAKE_CXX_COMPILER libkim-api CMAKE_CXX_COMPILER) - pkg_get_variable(KIM-API_CMAKE_Fortran_COMPILER libkim-api CMAKE_Fortran_COMPILER) -endif() - -if(KIM-API_INCLUDEDIR) - _KIMAPI_GET_VERSION(KIM-API_VERSION ${KIM-API_INCLUDEDIR}/KIM_Version.h) -else() - set(KIM-API_VERSION 0) -endif() - -# handle the QUIETLY and REQUIRED arguments and set KIM-API_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(KIM-API REQUIRED_VARS KIM-API_LIBRARIES VERSION_VAR KIM-API_VERSION) diff --git a/cmake/Modules/FindLATTE.cmake b/cmake/Modules/FindLATTE.cmake deleted file mode 100644 index 74d5173bf0..0000000000 --- a/cmake/Modules/FindLATTE.cmake +++ /dev/null @@ -1,18 +0,0 @@ -# - Find latte -# Find the native LATTE libraries. -# -# LATTE_LIBRARIES - List of libraries when using latte. -# LATTE_FOUND - True if latte found. -# - -find_library(LATTE_LIBRARY NAMES latte) - -set(LATTE_LIBRARIES ${LATTE_LIBRARY}) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set LATTE_FOUND to TRUE -# if all listed variables are TRUE - -find_package_handle_standard_args(LATTE DEFAULT_MSG LATTE_LIBRARY) - -mark_as_advanced(LATTE_LIBRARY) diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index 4246062103..e3dc60ad97 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -10,13 +10,22 @@ find_path(MKL_INCLUDE_DIR mkl_dfti.h HINTS $ENV{MKLROOT}/include) find_library(MKL_LIBRARY NAMES mkl_rt HINTS $ENV{MKLROOT}/lib $ENV{MKLROOT}/lib/intel64) -set(MKL_LIBRARIES ${MKL_LIBRARY}) -set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set MKL_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(MKL DEFAULT_MSG MKL_LIBRARY MKL_INCLUDE_DIR) +if(MKL_FOUND) + set(MKL_LIBRARIES ${MKL_LIBRARY}) + set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR}) + + if(NOT TARGET MKL::MKL) + add_library(MKL::MKL UNKNOWN IMPORTED) + set_target_properties(MKL::MKL PROPERTIES + IMPORTED_LOCATION "${MKL_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(MKL_INCLUDE_DIR MKL_LIBRARY ) diff --git a/cmake/Modules/FindMSCG.cmake b/cmake/Modules/FindMSCG.cmake index 311ff78038..b301303dbe 100644 --- a/cmake/Modules/FindMSCG.cmake +++ b/cmake/Modules/FindMSCG.cmake @@ -10,13 +10,23 @@ find_path(MSCG_INCLUDE_DIR mscg.h PATH_SUFFIXES mscg) find_library(MSCG_LIBRARY NAMES mscg) -set(MSCG_LIBRARIES ${MSCG_LIBRARY}) -set(MSCG_INCLUDE_DIRS ${MSCG_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set MSCG_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(MSCG DEFAULT_MSG MSCG_LIBRARY MSCG_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(MSCG_FOUND) + set(MSCG_LIBRARIES ${MSCG_LIBRARY}) + set(MSCG_INCLUDE_DIRS ${MSCG_INCLUDE_DIR}) + + if(NOT TARGET MSCG::MSCG) + add_library(MSCG::MSCG UNKNOWN IMPORTED) + set_target_properties(MSCG::MSCG PROPERTIES + IMPORTED_LOCATION "${MSCG_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${MSCG_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(MSCG_INCLUDE_DIR MSCG_LIBRARY ) diff --git a/cmake/Modules/FindNetCDF.cmake b/cmake/Modules/FindNetCDF.cmake index 2a992b6b3b..5c8caa9208 100644 --- a/cmake/Modules/FindNetCDF.cmake +++ b/cmake/Modules/FindNetCDF.cmake @@ -120,3 +120,14 @@ set (NETCDF_INCLUDE_DIRS ${NetCDF_includes}) include (FindPackageHandleStandardArgs) find_package_handle_standard_args (NetCDF DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_HAS_INTERFACES) + +# Copy the results to the output variables and target. +if(NetCDF_FOUND) + if(NOT TARGET NetCDF::NetCDF) + add_library(NetCDF::NetCDF UNKNOWN IMPORTED) + set_target_properties(NetCDF::NetCDF PROPERTIES + IMPORTED_LOCATION "${NETCDF_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${NETCDF_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${NETCDF_LIBRARIES}") + endif() +endif() diff --git a/cmake/Modules/FindPNetCDF.cmake b/cmake/Modules/FindPNetCDF.cmake index bc3a5f9538..e501e9af49 100644 --- a/cmake/Modules/FindPNetCDF.cmake +++ b/cmake/Modules/FindPNetCDF.cmake @@ -53,3 +53,12 @@ include (FindPackageHandleStandardArgs) find_package_handle_standard_args (PNetCDF DEFAULT_MSG PNETCDF_LIBRARIES PNETCDF_INCLUDES) mark_as_advanced (PNETCDF_LIBRARIES PNETCDF_INCLUDES) + +if(PNetCDF_FOUND) + if(NOT TARGET PNetCDF::PNetCDF) + add_library(PNetCDF::PNetCDF UNKNOWN IMPORTED) + set_target_properties(PNetCDF::PNetCDF PROPERTIES + IMPORTED_LOCATION "${PNETCDF_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${PNETCDF_INCLUDES}") + endif() +endif() diff --git a/cmake/Modules/FindQUIP.cmake b/cmake/Modules/FindQUIP.cmake index b6d87d11fa..277cfae49e 100644 --- a/cmake/Modules/FindQUIP.cmake +++ b/cmake/Modules/FindQUIP.cmake @@ -7,12 +7,21 @@ find_library(QUIP_LIBRARY NAMES quip) -set(QUIP_LIBRARIES ${QUIP_LIBRARY}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set QUIP_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(QUIP DEFAULT_MSG QUIP_LIBRARY) +# Copy the results to the output variables and target. +if(QUIP_FOUND) + set(QUIP_LIBRARIES ${QUIP_LIBRARY}) + + if(NOT TARGET QUIP::QUIP) + add_library(QUIP::QUIP UNKNOWN IMPORTED) + set_target_properties(QUIP::QUIP PROPERTIES + IMPORTED_LOCATION "${QUIP_LIBRARY}") + endif() +endif() + mark_as_advanced(QUIP_LIBRARY) diff --git a/cmake/Modules/FindTBB.cmake b/cmake/Modules/FindTBB.cmake deleted file mode 100644 index 8cc050817e..0000000000 --- a/cmake/Modules/FindTBB.cmake +++ /dev/null @@ -1,46 +0,0 @@ -# - Find parts of TBB -# Find the native TBB headers and libraries. -# -# TBB_INCLUDE_DIRS - where to find tbb.h, etc. -# TBB_LIBRARIES - List of libraries when using tbb. -# TBB_FOUND - True if tbb found. -# - -######################################################## -# TBB - -# TODO use more generic FindTBB - -find_path(TBB_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) -find_library(TBB_LIBRARY NAMES tbb PATHS $ENV{TBBROOT}/lib/intel64/gcc4.7 - $ENV{TBBROOT}/lib/intel64/gcc4.4 - $ENV{TBBROOT}/lib/intel64/gcc4.1) -set(TBB_LIBRARIES ${TBB_LIBRARY}) -set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set TBB_FOUND to TRUE -# if all listed variables are TRUE - -find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARY TBB_INCLUDE_DIR) - -mark_as_advanced(TBB_INCLUDE_DIR TBB_LIBRARY ) - -######################################################## -# TBB Malloc - -find_path(TBB_MALLOC_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) -find_library(TBB_MALLOC_LIBRARY NAMES tbbmalloc PATHS $ENV{TBBROOT}/lib/intel64/gcc4.7 - $ENV{TBBROOT}/lib/intel64/gcc4.4 - $ENV{TBBROOT}/lib/intel64/gcc4.1) - -set(TBB_MALLOC_LIBRARIES ${TBB_MALLOC_LIBRARY}) -set(TBB_MALLOC_INCLUDE_DIRS ${TBB_MALLOC_INCLUDE_DIR}) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set TBB_MALLOC_FOUND to TRUE -# if all listed variables are TRUE - -find_package_handle_standard_args(TBB_MALLOC DEFAULT_MSG TBB_MALLOC_LIBRARY TBB_MALLOC_INCLUDE_DIR) - -mark_as_advanced(TBB_MALLOC_INCLUDE_DIR TBB_MALLOC_LIBRARY ) diff --git a/cmake/Modules/FindTBB_MALLOC.cmake b/cmake/Modules/FindTBB_MALLOC.cmake new file mode 100644 index 0000000000..776c2c4749 --- /dev/null +++ b/cmake/Modules/FindTBB_MALLOC.cmake @@ -0,0 +1,37 @@ +# - Find parts of TBB_MALLOC +# Find the native TBB_MALLOC headers and libraries. +# +# TBB_MALLOC_INCLUDE_DIRS - where to find tbb.h, etc. +# TBB_MALLOC_LIBRARIES - List of libraries when using tbb. +# TBB_MALLOC_FOUND - True if tbb found. +# + + +######################################################## +# TBB Malloc + +find_path(TBB_MALLOC_INCLUDE_DIR NAMES tbb/tbb.h PATHS $ENV{TBBROOT}/include) +find_library(TBB_MALLOC_LIBRARY NAMES tbbmalloc PATHS $ENV{TBBROOT}/lib/intel64/gcc4.8 + $ENV{TBBROOT}/lib/intel64/gcc4.7 + $ENV{TBBROOT}/lib/intel64/gcc4.4 + $ENV{TBBROOT}/lib/intel64/gcc4.1) + +include(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set TBB_MALLOC_FOUND to TRUE +# if all listed variables are TRUE + +find_package_handle_standard_args(TBB_MALLOC DEFAULT_MSG TBB_MALLOC_LIBRARY TBB_MALLOC_INCLUDE_DIR) + +if(TBB_MALLOC_FOUND) + set(TBB_MALLOC_LIBRARIES ${TBB_MALLOC_LIBRARY}) + set(TBB_MALLOC_INCLUDE_DIRS ${TBB_MALLOC_INCLUDE_DIR}) + + if(NOT TARGET TBB::TBB_MALLOC) + add_library(TBB::TBB_MALLOC UNKNOWN IMPORTED) + set_target_properties(TBB::TBB_MALLOC PROPERTIES + IMPORTED_LOCATION "${TBB_MALLOC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${TBB_MALLOC_INCLUDE_DIR}") + endif() +endif() + +mark_as_advanced(TBB_MALLOC_INCLUDE_DIR TBB_MALLOC_LIBRARY ) diff --git a/cmake/Modules/FindVORO.cmake b/cmake/Modules/FindVORO.cmake index b0cccbcd1d..3f0fe98ff1 100644 --- a/cmake/Modules/FindVORO.cmake +++ b/cmake/Modules/FindVORO.cmake @@ -10,13 +10,23 @@ find_path(VORO_INCLUDE_DIR voro++.hh PATH_SUFFIXES voro++) find_library(VORO_LIBRARY NAMES voro++) -set(VORO_LIBRARIES ${VORO_LIBRARY}) -set(VORO_INCLUDE_DIRS ${VORO_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set VORO_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args(VORO DEFAULT_MSG VORO_LIBRARY VORO_INCLUDE_DIR) +# Copy the results to the output variables and target. +if(VORO_FOUND) + set(VORO_LIBRARIES ${VORO_LIBRARY}) + set(VORO_INCLUDE_DIRS ${VORO_INCLUDE_DIR}) + + if(NOT TARGET VORO::VORO) + add_library(VORO::VORO UNKNOWN IMPORTED) + set_target_properties(VORO::VORO PROPERTIES + IMPORTED_LOCATION "${VORO_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${VORO_INCLUDE_DIR}") + endif() +endif() + mark_as_advanced(VORO_INCLUDE_DIR VORO_LIBRARY ) diff --git a/cmake/Modules/FindZMQ.cmake b/cmake/Modules/FindZMQ.cmake index 608ccda777..7d612c2eb3 100644 --- a/cmake/Modules/FindZMQ.cmake +++ b/cmake/Modules/FindZMQ.cmake @@ -1,8 +1,19 @@ find_path(ZMQ_INCLUDE_DIR zmq.h) find_library(ZMQ_LIBRARY NAMES zmq) -set(ZMQ_LIBRARIES ${ZMQ_LIBRARY}) -set(ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR}) - include(FindPackageHandleStandardArgs) find_package_handle_standard_args(ZMQ DEFAULT_MSG ZMQ_LIBRARY ZMQ_INCLUDE_DIR) + +# Copy the results to the output variables and target. +if(ZMQ_FOUND) + set(ZMQ_LIBRARIES ${ZMQ_LIBRARY}) + set(ZMQ_INCLUDE_DIRS ${ZMQ_INCLUDE_DIR}) + + if(NOT TARGET ZMQ::ZMQ) + add_library(ZMQ::ZMQ UNKNOWN IMPORTED) + set_target_properties(ZMQ::ZMQ PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${ZMQ_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${ZMQ_INCLUDE_DIR}") + endif() +endif() diff --git a/cmake/Modules/GenerateBinaryHeader.cmake b/cmake/Modules/GenerateBinaryHeader.cmake new file mode 100644 index 0000000000..512f69e09d --- /dev/null +++ b/cmake/Modules/GenerateBinaryHeader.cmake @@ -0,0 +1,3 @@ +# utility script to call GenerateBinaryHeader function +include(${SOURCE_DIR}/Modules/LAMMPSUtils.cmake) +GenerateBinaryHeader(${VARNAME} ${HEADER_FILE} ${SOURCE_FILES}) diff --git a/cmake/Modules/LAMMPSUtils.cmake b/cmake/Modules/LAMMPSUtils.cmake index 3ea2b3cb7e..02de630c82 100644 --- a/cmake/Modules/LAMMPSUtils.cmake +++ b/cmake/Modules/LAMMPSUtils.cmake @@ -69,3 +69,19 @@ macro(pkg_depends PKG1 PKG2) message(FATAL_ERROR "${PKG1} package needs LAMMPS to be build with ${PKG2}") endif() endmacro() + +# CMake-only replacement for bin2c and xxd +function(GenerateBinaryHeader varname outfile files) + message("Creating ${outfile}...") + file(WRITE ${outfile} "// CMake generated file\n") + math(EXPR ARG_END "${ARGC}-1") + + foreach(IDX RANGE 2 ${ARG_END}) + list(GET ARGV ${IDX} filename) + file(READ ${filename} content HEX) + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," content "${content}") + string(REGEX REPLACE ",$" "" content "${content}") + file(APPEND ${outfile} "const unsigned char ${varname}[] = { ${content} };\n") + file(APPEND ${outfile} "const unsigned int ${varname}_size = sizeof(${varname});\n") + endforeach() +endfunction(GenerateBinaryHeader) diff --git a/cmake/Modules/MPI4WIN.cmake b/cmake/Modules/MPI4WIN.cmake index 1f3195041a..035c6a22f3 100644 --- a/cmake/Modules/MPI4WIN.cmake +++ b/cmake/Modules/MPI4WIN.cmake @@ -16,8 +16,15 @@ else() endif() ExternalProject_get_property(mpi4win_build SOURCE_DIR) -add_definitions(-DMPICH_SKIP_MPICXX) -include_directories("${SOURCE_DIR}/include") -set(MPI4WIN_LIBRARIES "${SOURCE_DIR}/lib/libmpi.a") -list(APPEND LAMMPS_DEPS mpi4win_build) -set(LAMMPS_USE_MPI4WIN ON) +file(MAKE_DIRECTORY "${SOURCE_DIR}/include") +add_library(MPI::MPI_CXX UNKNOWN IMPORTED) +set_target_properties(MPI::MPI_CXX PROPERTIES + IMPORTED_LOCATION "${SOURCE_DIR}/lib/libmpi.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/include" + INTERFACE_COMPILE_DEFINITIONS "MPICH_SKIP_MPICXX") +add_dependencies(MPI::MPI_CXX mpi4win_build) + +# set variables for status reporting at the end of CMake run +set(MPI_CXX_INCLUDE_PATH "${SOURCE_DIR}/include") +set(MPI_CXX_COMPILE_DEFINITIONS "MPICH_SKIP_MPICXX") +set(MPI_CXX_LIBRARIES "${SOURCE_DIR}/lib/libmpi.a") diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index 864b868865..801a38a215 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -1,5 +1,2 @@ -if(PKG_COMPRESS) - find_package(ZLIB REQUIRED) - include_directories(${ZLIB_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${ZLIB_LIBRARIES}) -endif() +find_package(ZLIB REQUIRED) +target_link_libraries(lammps PRIVATE ZLIB::ZLIB) diff --git a/cmake/Modules/Packages/CORESHELL.cmake b/cmake/Modules/Packages/CORESHELL.cmake index 591477c899..4e12d2ee36 100644 --- a/cmake/Modules/Packages/CORESHELL.cmake +++ b/cmake/Modules/Packages/CORESHELL.cmake @@ -1,13 +1,11 @@ -if(PKG_CORESHELL) - set(CORESHELL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/CORESHELL) - set(CORESHELL_SOURCES) - set_property(GLOBAL PROPERTY "CORESHELL_SOURCES" "${CORESHELL_SOURCES}") + set(CORESHELL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/CORESHELL) + set(CORESHELL_SOURCES) + set_property(GLOBAL PROPERTY "CORESHELL_SOURCES" "${CORESHELL_SOURCES}") - # detects styles which have a CORESHELL version - RegisterStylesExt(${CORESHELL_SOURCES_DIR} cs CORESHELL_SOURCES) + # detects styles which have a CORESHELL version + RegisterStylesExt(${CORESHELL_SOURCES_DIR} cs CORESHELL_SOURCES) - get_property(CORESHELL_SOURCES GLOBAL PROPERTY CORESHELL_SOURCES) + get_property(CORESHELL_SOURCES GLOBAL PROPERTY CORESHELL_SOURCES) - list(APPEND LIB_SOURCES ${CORESHELL_SOURCES}) - include_directories(${CORESHELL_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${CORESHELL_SOURCES}) + target_include_directories(lammps PRIVATE ${CORESHELL_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index abbcb1f495..5033c1f0f1 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -1,197 +1,371 @@ -if(PKG_GPU) - set(GPU_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/GPU) - set(GPU_SOURCES ${GPU_SOURCES_DIR}/gpu_extra.h - ${GPU_SOURCES_DIR}/fix_gpu.h - ${GPU_SOURCES_DIR}/fix_gpu.cpp) +set(GPU_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/GPU) +set(GPU_SOURCES ${GPU_SOURCES_DIR}/gpu_extra.h + ${GPU_SOURCES_DIR}/fix_gpu.h + ${GPU_SOURCES_DIR}/fix_gpu.cpp) - set(GPU_API "opencl" CACHE STRING "API used by GPU package") - set(GPU_API_VALUES opencl cuda) - set_property(CACHE GPU_API PROPERTY STRINGS ${GPU_API_VALUES}) - validate_option(GPU_API GPU_API_VALUES) - string(TOUPPER ${GPU_API} GPU_API) +set(GPU_API "opencl" CACHE STRING "API used by GPU package") +set(GPU_API_VALUES opencl cuda hip) +set_property(CACHE GPU_API PROPERTY STRINGS ${GPU_API_VALUES}) +validate_option(GPU_API GPU_API_VALUES) +string(TOUPPER ${GPU_API} GPU_API) - set(GPU_PREC "mixed" CACHE STRING "LAMMPS GPU precision") - set(GPU_PREC_VALUES double mixed single) - set_property(CACHE GPU_PREC PROPERTY STRINGS ${GPU_PREC_VALUES}) - validate_option(GPU_PREC GPU_PREC_VALUES) - string(TOUPPER ${GPU_PREC} GPU_PREC) +set(GPU_PREC "mixed" CACHE STRING "LAMMPS GPU precision") +set(GPU_PREC_VALUES double mixed single) +set_property(CACHE GPU_PREC PROPERTY STRINGS ${GPU_PREC_VALUES}) +validate_option(GPU_PREC GPU_PREC_VALUES) +string(TOUPPER ${GPU_PREC} GPU_PREC) - if(GPU_PREC STREQUAL "DOUBLE") - set(GPU_PREC_SETTING "DOUBLE_DOUBLE") - elseif(GPU_PREC STREQUAL "MIXED") - set(GPU_PREC_SETTING "SINGLE_DOUBLE") - elseif(GPU_PREC STREQUAL "SINGLE") - set(GPU_PREC_SETTING "SINGLE_SINGLE") - endif() - - file(GLOB GPU_LIB_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cpp) - file(MAKE_DIRECTORY ${LAMMPS_LIB_BINARY_DIR}/gpu) - - if(GPU_API STREQUAL "CUDA") - find_package(CUDA REQUIRED) - find_program(BIN2C bin2c) - if(NOT BIN2C) - message(FATAL_ERROR "Could not find bin2c, use -DBIN2C=/path/to/bin2c to help cmake finding it.") - endif() - option(CUDPP_OPT "Enable CUDPP_OPT" ON) - option(CUDA_MPS_SUPPORT "Enable tweaks to support CUDA Multi-process service (MPS)" OFF) - if(CUDA_MPS_SUPPORT) - set(GPU_CUDA_MPS_FLAGS "-DCUDA_PROXY") - endif() - - set(GPU_ARCH "sm_30" CACHE STRING "LAMMPS GPU CUDA SM primary architecture (e.g. sm_60)") - - file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) - list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) - - cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu ${LAMMPS_LIB_BINARY_DIR}/gpu) - - if(CUDPP_OPT) - cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) - file(GLOB GPU_LIB_CUDPP_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cpp) - file(GLOB GPU_LIB_CUDPP_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cu) - endif() - - # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice - # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture - set(GPU_CUDA_GENCODE "-arch=${GPU_ARCH} ") - # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 - if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") - endif() - # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later - if(CUDA_VERSION VERSION_GREATER "4.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") - endif() - # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later - if(CUDA_VERSION VERSION_GREATER "5.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") - endif() - # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later - if(CUDA_VERSION VERSION_GREATER "7.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") - endif() - # Volta (GPU Arch 7.0) is supported by CUDA 9 and later - if(CUDA_VERSION VERSION_GREATER "8.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") - endif() - # Turing (GPU Arch 7.5) is supported by CUDA 10 and later - if(CUDA_VERSION VERSION_GREATER "9.9") - string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") - endif() - - cuda_compile_fatbin(GPU_GEN_OBJS ${GPU_LIB_CU} OPTIONS - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) - - cuda_compile(GPU_OBJS ${GPU_LIB_CUDPP_CU} OPTIONS ${CUDA_REQUEST_PIC} - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) - - foreach(CU_OBJ ${GPU_GEN_OBJS}) - get_filename_component(CU_NAME ${CU_OBJ} NAME_WE) - string(REGEX REPLACE "^.*_lal_" "" CU_NAME "${CU_NAME}") - add_custom_command(OUTPUT ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h - COMMAND ${BIN2C} -c -n ${CU_NAME} ${CU_OBJ} > ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h - DEPENDS ${CU_OBJ} - COMMENT "Generating ${CU_NAME}_cubin.h") - list(APPEND GPU_LIB_SOURCES ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h) - endforeach() - set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h") - - - add_library(gpu STATIC ${GPU_LIB_SOURCES} ${GPU_LIB_CUDPP_SOURCES} ${GPU_OBJS}) - target_link_libraries(gpu ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) - target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu ${CUDA_INCLUDE_DIRS}) - target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT ${GPU_CUDA_MPS_FLAGS}) - if(CUDPP_OPT) - target_include_directories(gpu PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) - target_compile_definitions(gpu PRIVATE -DUSE_CUDPP) - endif() - - list(APPEND LAMMPS_LINK_LIBS gpu) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(gpu mpi4win_build) - endif() - - add_executable(nvc_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) - target_compile_definitions(nvc_get_devices PRIVATE -DUCL_CUDADR) - target_link_libraries(nvc_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) - target_include_directories(nvc_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) - - - elseif(GPU_API STREQUAL "OPENCL") - find_package(OpenCL REQUIRED) - set(OCL_TUNE "generic" CACHE STRING "OpenCL Device Tuning") - set(OCL_TUNE_VALUES intel fermi kepler cypress generic) - set_property(CACHE OCL_TUNE PROPERTY STRINGS ${OCL_TUNE_VALUES}) - validate_option(OCL_TUNE OCL_TUNE_VALUES) - string(TOUPPER ${OCL_TUNE} OCL_TUNE) - - include(OpenCLUtils) - set(OCL_COMMON_HEADERS ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_preprocessor.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_aux_fun1.h) - - file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu) - list(REMOVE_ITEM GPU_LIB_CU - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu - ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu - ) - - foreach(GPU_KERNEL ${GPU_LIB_CU}) - get_filename_component(basename ${GPU_KERNEL} NAME_WE) - string(SUBSTRING ${basename} 4 -1 KERNEL_NAME) - GenerateOpenCLHeader(${KERNEL_NAME} ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h ${OCL_COMMON_HEADERS} ${GPU_KERNEL}) - list(APPEND GPU_LIB_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h) - endforeach() - - GenerateOpenCLHeader(gayberne ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu) - GenerateOpenCLHeader(gayberne_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu) - GenerateOpenCLHeader(re_squared ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu) - GenerateOpenCLHeader(re_squared_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu) - GenerateOpenCLHeader(tersoff ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu) - GenerateOpenCLHeader(tersoff_zbl ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu) - GenerateOpenCLHeader(tersoff_mod ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu) - - list(APPEND GPU_LIB_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h - ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h - ) - - add_library(gpu STATIC ${GPU_LIB_SOURCES}) - target_link_libraries(gpu ${OpenCL_LIBRARIES}) - target_include_directories(gpu PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/gpu ${OpenCL_INCLUDE_DIRS}) - target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -D${OCL_TUNE}_OCL -DMPI_GERYON -DUCL_NO_EXIT) - target_compile_definitions(gpu PRIVATE -DUSE_OPENCL) - - list(APPEND LAMMPS_LINK_LIBS gpu) - if(LAMMPS_USE_MPI4WIN) - add_dependencies(gpu mpi4win_build) - endif() - - add_executable(ocl_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) - target_compile_definitions(ocl_get_devices PRIVATE -DUCL_OPENCL) - target_link_libraries(ocl_get_devices PRIVATE ${OpenCL_LIBRARIES}) - target_include_directories(ocl_get_devices PRIVATE ${OpenCL_INCLUDE_DIRS}) - endif() - - # GPU package - FindStyleHeaders(${GPU_SOURCES_DIR} FIX_CLASS fix_ FIX) - - set_property(GLOBAL PROPERTY "GPU_SOURCES" "${GPU_SOURCES}") - - # detects styles which have GPU version - RegisterStylesExt(${GPU_SOURCES_DIR} gpu GPU_SOURCES) - - get_property(GPU_SOURCES GLOBAL PROPERTY GPU_SOURCES) - - list(APPEND LIB_SOURCES ${GPU_SOURCES}) - include_directories(${GPU_SOURCES_DIR}) +if(GPU_PREC STREQUAL "DOUBLE") + set(GPU_PREC_SETTING "DOUBLE_DOUBLE") +elseif(GPU_PREC STREQUAL "MIXED") + set(GPU_PREC_SETTING "SINGLE_DOUBLE") +elseif(GPU_PREC STREQUAL "SINGLE") + set(GPU_PREC_SETTING "SINGLE_SINGLE") endif() + +file(GLOB GPU_LIB_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cpp) +file(MAKE_DIRECTORY ${LAMMPS_LIB_BINARY_DIR}/gpu) + +if(GPU_API STREQUAL "CUDA") + find_package(CUDA REQUIRED) + find_program(BIN2C bin2c) + if(NOT BIN2C) + message(FATAL_ERROR "Could not find bin2c, use -DBIN2C=/path/to/bin2c to help cmake finding it.") + endif() + option(CUDPP_OPT "Enable CUDPP_OPT" ON) + option(CUDA_MPS_SUPPORT "Enable tweaks to support CUDA Multi-process service (MPS)" OFF) + if(CUDA_MPS_SUPPORT) + set(GPU_CUDA_MPS_FLAGS "-DCUDA_PROXY") + endif() + + set(GPU_ARCH "sm_50" CACHE STRING "LAMMPS GPU CUDA SM primary architecture (e.g. sm_60)") + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) + + cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu ${LAMMPS_LIB_BINARY_DIR}/gpu) + + if(CUDPP_OPT) + cuda_include_directories(${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) + file(GLOB GPU_LIB_CUDPP_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cpp) + file(GLOB GPU_LIB_CUDPP_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini/[^.]*.cu) + endif() + + # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice + # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture + set(GPU_CUDA_GENCODE "-arch=${GPU_ARCH} ") + # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 + if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") + endif() + # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later + if(CUDA_VERSION VERSION_GREATER "4.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") + endif() + # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later + if(CUDA_VERSION VERSION_GREATER "5.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") + endif() + # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later + if(CUDA_VERSION VERSION_GREATER "7.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") + endif() + # Volta (GPU Arch 7.0) is supported by CUDA 9 and later + if(CUDA_VERSION VERSION_GREATER "8.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") + endif() + # Turing (GPU Arch 7.5) is supported by CUDA 10 and later + if(CUDA_VERSION VERSION_GREATER "9.9") + string(APPEND GPU_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") + endif() + + cuda_compile_fatbin(GPU_GEN_OBJS ${GPU_LIB_CU} OPTIONS + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + + cuda_compile(GPU_OBJS ${GPU_LIB_CUDPP_CU} OPTIONS ${CUDA_REQUEST_PIC} + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + + foreach(CU_OBJ ${GPU_GEN_OBJS}) + get_filename_component(CU_NAME ${CU_OBJ} NAME_WE) + string(REGEX REPLACE "^.*_lal_" "" CU_NAME "${CU_NAME}") + add_custom_command(OUTPUT ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h + COMMAND ${BIN2C} -c -n ${CU_NAME} ${CU_OBJ} > ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h + DEPENDS ${CU_OBJ} + COMMENT "Generating ${CU_NAME}_cubin.h") + list(APPEND GPU_LIB_SOURCES ${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h) + endforeach() + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h") + + add_library(gpu STATIC ${GPU_LIB_SOURCES} ${GPU_LIB_CUDPP_SOURCES} ${GPU_OBJS}) + target_link_libraries(gpu PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu ${CUDA_INCLUDE_DIRS}) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT ${GPU_CUDA_MPS_FLAGS}) + if(CUDPP_OPT) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/gpu/cudpp_mini) + target_compile_definitions(gpu PRIVATE -DUSE_CUDPP) + endif() + + target_link_libraries(lammps PRIVATE gpu) + + add_executable(nvc_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(nvc_get_devices PRIVATE -DUCL_CUDADR) + target_link_libraries(nvc_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + target_include_directories(nvc_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) + +elseif(GPU_API STREQUAL "OPENCL") + if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + # download and unpack support binaries for compilation of windows binaries. + set(LAMMPS_THIRDPARTY_URL "http://download.lammps.org/thirdparty") + file(DOWNLOAD "${LAMMPS_THIRDPARTY_URL}/opencl-win-devel.tar.gz" "${CMAKE_CURRENT_BINARY_DIR}/opencl-win-devel.tar.gz" + EXPECTED_MD5 2c00364888d5671195598b44c2e0d44d) + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf opencl-win-devel.tar.gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + add_library(OpenCL::OpenCL UNKNOWN IMPORTED) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86") + set_target_properties(OpenCL::OpenCL PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win32/libOpenCL.dll") + elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") + set_target_properties(OpenCL::OpenCL PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/lib_win64/libOpenCL.dll") + endif() + set_target_properties(OpenCL::OpenCL PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/OpenCL/include") + else() + find_package(OpenCL REQUIRED) + endif() + set(OCL_TUNE "generic" CACHE STRING "OpenCL Device Tuning") + set(OCL_TUNE_VALUES intel fermi kepler cypress generic) + set_property(CACHE OCL_TUNE PROPERTY STRINGS ${OCL_TUNE_VALUES}) + validate_option(OCL_TUNE OCL_TUNE_VALUES) + string(TOUPPER ${OCL_TUNE} OCL_TUNE) + + include(OpenCLUtils) + set(OCL_COMMON_HEADERS ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_preprocessor.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_aux_fun1.h) + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu + ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu + ) + + foreach(GPU_KERNEL ${GPU_LIB_CU}) + get_filename_component(basename ${GPU_KERNEL} NAME_WE) + string(SUBSTRING ${basename} 4 -1 KERNEL_NAME) + GenerateOpenCLHeader(${KERNEL_NAME} ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h ${OCL_COMMON_HEADERS} ${GPU_KERNEL}) + list(APPEND GPU_LIB_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/gpu/${KERNEL_NAME}_cl.h) + endforeach() + + GenerateOpenCLHeader(gayberne ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne.cu) + GenerateOpenCLHeader(gayberne_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_gayberne_lj.cu) + GenerateOpenCLHeader(re_squared ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared.cu) + GenerateOpenCLHeader(re_squared_lj ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_ellipsoid_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_re_squared_lj.cu) + GenerateOpenCLHeader(tersoff ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff.cu) + GenerateOpenCLHeader(tersoff_zbl ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_zbl.cu) + GenerateOpenCLHeader(tersoff_mod ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h ${OCL_COMMON_HEADERS} ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod_extra.h ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_tersoff_mod.cu) + + list(APPEND GPU_LIB_SOURCES + ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/gayberne_lj_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/re_squared_lj_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_zbl_cl.h + ${CMAKE_CURRENT_BINARY_DIR}/gpu/tersoff_mod_cl.h + ) + + add_library(gpu STATIC ${GPU_LIB_SOURCES}) + target_link_libraries(gpu PRIVATE OpenCL::OpenCL) + target_include_directories(gpu PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/gpu) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -D${OCL_TUNE}_OCL -DMPI_GERYON -DUCL_NO_EXIT) + target_compile_definitions(gpu PRIVATE -DUSE_OPENCL) + + target_link_libraries(lammps PRIVATE gpu) + + add_executable(ocl_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(ocl_get_devices PRIVATE -DUCL_OPENCL) + target_link_libraries(ocl_get_devices PRIVATE OpenCL::OpenCL) +elseif(GPU_API STREQUAL "HIP") + if(NOT DEFINED HIP_PATH) + if(NOT DEFINED ENV{HIP_PATH}) + set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") + else() + set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") + endif() + endif() + set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) + find_package(HIP REQUIRED) + option(HIP_USE_DEVICE_SORT "Use GPU sorting" ON) + + if(NOT DEFINED HIP_PLATFORM) + if(NOT DEFINED ENV{HIP_PLATFORM}) + set(HIP_PLATFORM "hcc" CACHE PATH "HIP Platform to be used during compilation") + else() + set(HIP_PLATFORM $ENV{HIP_PLATFORM} CACHE PATH "HIP Platform used during compilation") + endif() + endif() + + set(ENV{HIP_PLATFORM} ${HIP_PLATFORM}) + + if(HIP_PLATFORM STREQUAL "hcc") + set(HIP_ARCH "gfx906" CACHE STRING "HIP target architecture") + elseif(HIP_PLATFORM STREQUAL "nvcc") + find_package(CUDA REQUIRED) + set(HIP_ARCH "sm_50" CACHE STRING "HIP primary CUDA architecture (e.g. sm_60)") + + # build arch/gencode commands for nvcc based on CUDA toolkit version and use choice + # --arch translates directly instead of JIT, so this should be for the preferred or most common architecture + set(HIP_CUDA_GENCODE "-arch=${HIP_ARCH} ") + # Fermi (GPU Arch 2.x) is supported by CUDA 3.2 to CUDA 8.0 + if((CUDA_VERSION VERSION_GREATER "3.1") AND (CUDA_VERSION VERSION_LESS "9.0")) + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_20,code=[sm_20,compute_20] ") + endif() + # Kepler (GPU Arch 3.x) is supported by CUDA 5 and later + if(CUDA_VERSION VERSION_GREATER "4.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_35,code=[sm_35,compute_35] ") + endif() + # Maxwell (GPU Arch 5.x) is supported by CUDA 6 and later + if(CUDA_VERSION VERSION_GREATER "5.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] ") + endif() + # Pascal (GPU Arch 6.x) is supported by CUDA 8 and later + if(CUDA_VERSION VERSION_GREATER "7.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] ") + endif() + # Volta (GPU Arch 7.0) is supported by CUDA 9 and later + if(CUDA_VERSION VERSION_GREATER "8.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_70,code=[sm_70,compute_70] ") + endif() + # Turing (GPU Arch 7.5) is supported by CUDA 10 and later + if(CUDA_VERSION VERSION_GREATER "9.9") + string(APPEND HIP_CUDA_GENCODE "-gencode arch=compute_75,code=[sm_75,compute_75] ") + endif() + endif() + + file(GLOB GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/[^.]*.cu ${CMAKE_CURRENT_SOURCE_DIR}/gpu/[^.]*.cu) + list(REMOVE_ITEM GPU_LIB_CU ${LAMMPS_LIB_SOURCE_DIR}/gpu/lal_pppm.cu) + + set(GPU_LIB_CU_HIP "") + foreach(CU_FILE ${GPU_LIB_CU}) + get_filename_component(CU_NAME ${CU_FILE} NAME_WE) + string(REGEX REPLACE "^.*lal_" "" CU_NAME "${CU_NAME}") + + set(CU_CPP_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}.cu.cpp") + set(CUBIN_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}.cubin") + set(CUBIN_H_FILE "${LAMMPS_LIB_BINARY_DIR}/gpu/${CU_NAME}_cubin.h") + + if(HIP_PLATFORM STREQUAL "hcc") + configure_file(${CU_FILE} ${CU_CPP_FILE} COPYONLY) + + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} + DEPENDS ${CU_CPP_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + elseif(HIP_PLATFORM STREQUAL "nvcc") + add_custom_command(OUTPUT ${CUBIN_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE} + DEPENDS ${CU_FILE} + COMMENT "Generating ${CU_NAME}.cubin") + endif() + + add_custom_command(OUTPUT ${CUBIN_H_FILE} + COMMAND ${CMAKE_COMMAND} -D SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -D VARNAME=${CU_NAME} -D HEADER_FILE=${CUBIN_H_FILE} -D SOURCE_FILES=${CUBIN_FILE} -P ${CMAKE_CURRENT_SOURCE_DIR}/Modules/GenerateBinaryHeader.cmake + DEPENDS ${CUBIN_FILE} + COMMENT "Generating ${CU_NAME}_cubin.h") + + list(APPEND GPU_LIB_SOURCES ${CUBIN_H_FILE}) + endforeach() + + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${LAMMPS_LIB_BINARY_DIR}/gpu/*_cubin.h ${LAMMPS_LIB_BINARY_DIR}/gpu/*.cu.cpp") + + hip_add_library(gpu STATIC ${GPU_LIB_SOURCES}) + target_include_directories(gpu PRIVATE ${LAMMPS_LIB_BINARY_DIR}/gpu) + target_compile_definitions(gpu PRIVATE -D_${GPU_PREC_SETTING} -DMPI_GERYON -DUCL_NO_EXIT) + target_compile_definitions(gpu PRIVATE -DUSE_HIP) + + if(HIP_USE_DEVICE_SORT) + # add hipCUB + target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) + target_compile_definitions(gpu PRIVATE -DUSE_HIP_DEVICE_SORT) + + if(HIP_PLATFORM STREQUAL "nvcc") + find_package(CUB) + + if(CUB_FOUND) + set(DOWNLOAD_CUB_DEFAULT OFF) + else() + set(DOWNLOAD_CUB_DEFAULT ON) + endif() + + option(DOWNLOAD_CUB "Download and compile the CUB library instead of using an already installed one" ${DOWNLOAD_CUB_DEFAULT}) + + if(DOWNLOAD_CUB) + message(STATUS "CUB download requested") + include(ExternalProject) + + ExternalProject_Add(CUB + GIT_REPOSITORY https://github.com/NVlabs/cub + TIMEOUT 5 + PREFIX "${CMAKE_CURRENT_BINARY_DIR}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" + ) + ExternalProject_get_property(CUB SOURCE_DIR) + set(CUB_INCLUDE_DIR ${SOURCE_DIR}) + else() + find_package(CUB) + if(NOT CUB_FOUND) + message(FATAL_ERROR "CUB library not found. Help CMake to find it by setting CUB_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") + endif() + endif() + + target_include_directories(gpu PRIVATE ${CUB_INCLUDE_DIR}) + endif() + endif() + + hip_add_executable(hip_get_devices ${LAMMPS_LIB_SOURCE_DIR}/gpu/geryon/ucl_get_devices.cpp) + target_compile_definitions(hip_get_devices PRIVATE -DUCL_HIP) + + if(HIP_PLATFORM STREQUAL "nvcc") + target_compile_definitions(gpu PRIVATE -D__HIP_PLATFORM_NVCC__) + target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) + target_include_directories(gpu PRIVATE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(gpu PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + + target_compile_definitions(hip_get_devices PRIVATE -D__HIP_PLATFORM_NVCC__) + target_include_directories(hip_get_devices PRIVATE ${HIP_ROOT_DIR}/include) + target_include_directories(hip_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) + target_link_libraries(hip_get_devices PRIVATE ${CUDA_LIBRARIES} ${CUDA_CUDA_LIBRARY}) + elseif(HIP_PLATFORM STREQUAL "hcc") + target_compile_definitions(gpu PRIVATE -D__HIP_PLATFORM_HCC__) + target_include_directories(gpu PRIVATE ${HIP_ROOT_DIR}/../include) + + target_compile_definitions(hip_get_devices PRIVATE -D__HIP_PLATFORM_HCC__) + target_include_directories(hip_get_devices PRIVATE ${HIP_ROOT_DIR}/../include) + endif() + + target_link_libraries(lammps PRIVATE gpu) +endif() + +# GPU package +FindStyleHeaders(${GPU_SOURCES_DIR} FIX_CLASS fix_ FIX) + +set_property(GLOBAL PROPERTY "GPU_SOURCES" "${GPU_SOURCES}") + +# detects styles which have GPU version +RegisterStylesExt(${GPU_SOURCES_DIR} gpu GPU_SOURCES) + +get_property(GPU_SOURCES GLOBAL PROPERTY GPU_SOURCES) + +if(NOT BUILD_MPI) + # mpistubs is aliased to MPI::MPI_CXX, but older versions of cmake won't work forward the include path + target_link_libraries(gpu PRIVATE mpi_stubs) +else() + target_link_libraries(gpu PRIVATE MPI::MPI_CXX) +endif() +target_compile_definitions(gpu PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(gpu PROPERTIES OUTPUT_NAME lammps_gpu${LAMMPS_MACHINE}) +target_sources(lammps PRIVATE ${GPU_SOURCES}) +target_include_directories(lammps PRIVATE ${GPU_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/KIM.cmake b/cmake/Modules/Packages/KIM.cmake index 617068ce13..d74b82b036 100644 --- a/cmake/Modules/Packages/KIM.cmake +++ b/cmake/Modules/Packages/KIM.cmake @@ -1,67 +1,63 @@ -if(PKG_KIM) - set(KIM-API_MIN_VERSION 2.1) - find_package(CURL) - if(CURL_FOUND) - include_directories(${CURL_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${CURL_LIBRARIES}) - add_definitions(-DLMP_KIM_CURL) - set(LMP_DEBUG_CURL OFF CACHE STRING "Set libcurl verbose mode on/off. If on, it displays a lot of verbose information about its operations.") - mark_as_advanced(LMP_DEBUG_CURL) - if(LMP_DEBUG_CURL) - add_definitions(-DLMP_DEBUG_CURL) - endif() - set(LMP_NO_SSL_CHECK OFF CACHE STRING "Tell libcurl to not verify the peer. If on, the connection succeeds regardless of the names in the certificate. Insecure - Use with caution!") - mark_as_advanced(LMP_NO_SSL_CHECK) - if(LMP_NO_SSL_CHECK) - add_definitions(-DLMP_NO_SSL_CHECK) - endif() - endif() - find_package(KIM-API QUIET) - if(KIM-API_FOUND) - if (KIM-API_VERSION VERSION_LESS ${KIM-API_MIN_VERSION}) - if ("${DOWNLOAD_KIM}" STREQUAL "") - message(WARNING "Unsuitable KIM-API version \"${KIM-API_VERSION}\" found, but required is at least \"${KIM-API_MIN_VERSION}\". Default behavior set to download and build our own.") - endif() - set(DOWNLOAD_KIM_DEFAULT ON) - else() - set(DOWNLOAD_KIM_DEFAULT OFF) - endif() +set(KIM-API_MIN_VERSION 2.1.3) +find_package(CURL) +if(CURL_FOUND) + if(CMAKE_VERSION VERSION_LESS 3.12) + target_include_directories(lammps PRIVATE ${CURL_INCLUDE_DIRS}) + target_link_libraries(lammps PRIVATE ${CURL_LIBRARIES}) else() - if ("${DOWNLOAD_KIM}" STREQUAL "") - message(WARNING "KIM-API package not found. Default behavior set to download and build our own") - endif() - set(DOWNLOAD_KIM_DEFAULT ON) + target_link_libraries(lammps PRIVATE CURL::libcurl) endif() - option(DOWNLOAD_KIM "Download KIM-API from OpenKIM instead of using an already installed one" ${DOWNLOAD_KIM_DEFAULT}) - if(DOWNLOAD_KIM) - message(STATUS "KIM-API download requested - we will build our own") - # Workaround for cross compilation with MinGW where ${CMAKE_INSTALL_LIBDIR} - # is a full path, so we need to remove the prefix - string(REPLACE ${CMAKE_INSTALL_PREFIX} "" _KIM_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - include(ExternalProject) - enable_language(C) - enable_language(Fortran) - ExternalProject_Add(kim_build - URL https://s3.openkim.org/kim-api/kim-api-2.1.3.txz - URL_MD5 6ee829a1bbba5f8b9874c88c4c4ebff8 - BINARY_DIR build - CMAKE_ARGS ${CMAKE_REQUEST_PIC} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} - -DCMAKE_INSTALL_PREFIX= - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_BYPRODUCTS /${_KIM_LIBDIR}/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX} - ) - ExternalProject_get_property(kim_build INSTALL_DIR) - set(KIM-API_INCLUDE_DIRS ${INSTALL_DIR}/include/kim-api) - set(KIM-API_LDFLAGS ${INSTALL_DIR}/${_KIM_LIBDIR}/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX}) - list(APPEND LAMMPS_DEPS kim_build) - else() - find_package(KIM-API ${KIM-API_MIN_VERSION} REQUIRED) + target_compile_definitions(lammps PRIVATE -DLMP_KIM_CURL) + set(LMP_DEBUG_CURL OFF CACHE STRING "Set libcurl verbose mode on/off. If on, it displays a lot of verbose information about its operations.") + mark_as_advanced(LMP_DEBUG_CURL) + if(LMP_DEBUG_CURL) + target_compile_definitions(lammps PRIVATE -DLMP_DEBUG_CURL) + endif() + set(LMP_NO_SSL_CHECK OFF CACHE STRING "Tell libcurl to not verify the peer. If on, the connection succeeds regardless of the names in the certificate. Insecure - Use with caution!") + mark_as_advanced(LMP_NO_SSL_CHECK) + if(LMP_NO_SSL_CHECK) + target_compile_definitions(lammps PRIVATE -DLMP_NO_SSL_CHECK) endif() - list(APPEND LAMMPS_LINK_LIBS "${KIM-API_LDFLAGS}") - include_directories(${KIM-API_INCLUDE_DIRS}) +endif() +find_package(PkgConfig QUIET) +set(DOWNLOAD_KIM_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(KIM-API QUIET libkim-api>=${KIM-API_MIN_VERSION}) + if(KIM-API_FOUND) + set(DOWNLOAD_KIM_DEFAULT OFF) + endif() +endif() +option(DOWNLOAD_KIM "Download KIM-API from OpenKIM instead of using an already installed one" ${DOWNLOAD_KIM_DEFAULT}) +if(DOWNLOAD_KIM) + message(STATUS "KIM-API download requested - we will build our own") + include(ExternalProject) + enable_language(C) + enable_language(Fortran) + ExternalProject_Add(kim_build + URL https://s3.openkim.org/kim-api/kim-api-2.1.3.txz + URL_MD5 6ee829a1bbba5f8b9874c88c4c4ebff8 + BINARY_DIR build + CMAKE_ARGS ${CMAKE_REQUEST_PIC} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_BYPRODUCTS /lib/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX} + ) + ExternalProject_get_property(kim_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include/kim-api) + add_library(LAMMPS::KIM UNKNOWN IMPORTED) + set_target_properties(LAMMPS::KIM PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libkim-api${CMAKE_SHARED_LIBRARY_SUFFIX}" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include/kim-api") + target_link_libraries(lammps PRIVATE LAMMPS::KIM) + add_dependencies(LAMMPS::KIM kim_build) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(KIM-API REQUIRED IMPORTED_TARGET libkim-api>=${KIM-API_MIN_VERSION}) + target_link_libraries(lammps PRIVATE PkgConfig::KIM-API) endif() diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index 29beaca957..620f1c65c6 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -1,74 +1,119 @@ -if(PKG_KOKKOS) - # TODO: this option needs to be documented when this works with a - # regular release version of KOKKOS, and a version compatibility check - # of external KOKKOS lib versus what the KOKKOS package needs is required. - option(EXTERNAL_KOKKOS "Build against external kokkos library") - if(EXTERNAL_KOKKOS) - find_package(Kokkos REQUIRED) - list(APPEND LAMMPS_LINK_LIBS Kokkos::kokkos) - else() - set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) - set(LAMMPS_LIB_KOKKOS_BIN_DIR ${LAMMPS_LIB_BINARY_DIR}/kokkos) - add_subdirectory(${LAMMPS_LIB_KOKKOS_SRC_DIR} ${LAMMPS_LIB_KOKKOS_BIN_DIR}) +######################################################################## +# consistency checks and Kokkos options/settings required by LAMMPS +if(Kokkos_ENABLE_CUDA) + message(STATUS "KOKKOS: Enabling CUDA LAMBDA function support") + set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "" FORCE) +endif() +# Adding OpenMP compiler flags without the checks done for +# BUILD_OMP can result in compile failures. Enforce consistency. +if(Kokkos_ENABLE_OPENMP AND NOT BUILD_OMP) + message(FATAL_ERROR "Must enable BUILD_OMP with Kokkos_ENABLE_OPENMP") +endif() +######################################################################## - set(Kokkos_INCLUDE_DIRS ${LAMMPS_LIB_KOKKOS_SRC_DIR}/core/src - ${LAMMPS_LIB_KOKKOS_SRC_DIR}/containers/src - ${LAMMPS_LIB_KOKKOS_SRC_DIR}/algorithms/src - ${LAMMPS_LIB_KOKKOS_BIN_DIR}) - include_directories(${Kokkos_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS kokkos) +option(EXTERNAL_KOKKOS "Build against external kokkos library" OFF) +option(DOWNLOAD_KOKKOS "Download the KOKKOS library instead of using the bundled one" OFF) +if(DOWNLOAD_KOKKOS) + # extract Kokkos-related variables and values so we can forward them to the Kokkos library build + get_cmake_property(_VARS VARIABLES) + list(FILTER _VARS INCLUDE REGEX ^Kokkos_) + foreach(_VAR IN LISTS _VARS) + list(APPEND KOKKOS_LIB_BUILD_ARGS "-D${_VAR}=${${_VAR}}") + endforeach() + message(STATUS "KOKKOS download requested - we will build our own") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_INSTALL_PREFIX=") + if(CMAKE_REQUEST_PIC) + list(APPEND KOKKOS_LIB_BUILD_ARGS ${CMAKE_REQUEST_PIC}) endif() - add_definitions(-DLMP_KOKKOS) + # append other CMake variables that need to be forwarded to CMAKE_ARGS + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_INSTALL_LIBDIR=lib") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_CXX_EXTENSIONS=${CMAKE_CXX_EXTENSIONS}") + list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") + include(ExternalProject) + ExternalProject_Add(kokkos_build + URL https://github.com/kokkos/kokkos/archive/3.1.01.tar.gz + URL_MD5 3ccb2100f7fc316891e7dad3bc33fa37 + CMAKE_ARGS ${KOKKOS_LIB_BUILD_ARGS} + BUILD_BYPRODUCTS /lib/libkokkoscore.a + ) + ExternalProject_get_property(kokkos_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + add_library(LAMMPS::KOKKOS UNKNOWN IMPORTED) + set_target_properties(LAMMPS::KOKKOS PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libkokkoscore.a" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include" + INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}) + target_link_libraries(lammps PRIVATE LAMMPS::KOKKOS) + add_dependencies(LAMMPS::KOKKOS kokkos_build) +elseif(EXTERNAL_KOKKOS) + find_package(Kokkos 3.1.01 REQUIRED CONFIG) + target_link_libraries(lammps PRIVATE Kokkos::kokkos) +else() + set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) + set(LAMMPS_LIB_KOKKOS_BIN_DIR ${LAMMPS_LIB_BINARY_DIR}/kokkos) + add_subdirectory(${LAMMPS_LIB_KOKKOS_SRC_DIR} ${LAMMPS_LIB_KOKKOS_BIN_DIR}) - set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) - set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/min_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/min_linesearch_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neighbor_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neigh_list_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/neigh_bond_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/fix_nh_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/domain_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/modify_kokkos.cpp) + set(Kokkos_INCLUDE_DIRS ${LAMMPS_LIB_KOKKOS_SRC_DIR}/core/src + ${LAMMPS_LIB_KOKKOS_SRC_DIR}/containers/src + ${LAMMPS_LIB_KOKKOS_SRC_DIR}/algorithms/src + ${LAMMPS_LIB_KOKKOS_BIN_DIR}) + target_include_directories(lammps PRIVATE ${Kokkos_INCLUDE_DIRS}) + target_link_libraries(lammps PRIVATE kokkos) +endif() +target_compile_definitions(lammps PRIVATE -DLMP_KOKKOS) - if(PKG_KSPACE) - list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/fft3d_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/gridcomm_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/remap_kokkos.cpp) - if(KOKKOS_ENABLE_CUDA) - if(NOT ${FFT} STREQUAL "KISS") - add_definitions(-DFFT_CUFFT) - list(APPEND LAMMPS_LINK_LIBS cufft) - endif() +set(KOKKOS_PKG_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/KOKKOS) +set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/min_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/min_linesearch_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neighbor_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neigh_list_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/neigh_bond_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/fix_nh_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/domain_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/modify_kokkos.cpp) + +if(PKG_KSPACE) + list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/fft3d_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/gridcomm_kokkos.cpp + ${KOKKOS_PKG_SOURCES_DIR}/remap_kokkos.cpp) + if(Kokkos_ENABLE_CUDA) + if(NOT ${FFT} STREQUAL "KISS") + target_compile_definitions(lammps PRIVATE -DFFT_CUFFT) + target_link_libraries(lammps PRIVATE cufft) endif() endif() - - set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") - - # detects styles which have KOKKOS version - RegisterStylesExt(${KOKKOS_PKG_SOURCES_DIR} kokkos KOKKOS_PKG_SOURCES) - - # register kokkos-only styles - RegisterNBinStyle(${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.h) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.h) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.h) - - if(PKG_USER-DPD) - get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) - list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.cpp) - RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.h) - set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") - endif() - - get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) - - list(APPEND LIB_SOURCES ${KOKKOS_PKG_SOURCES}) - include_directories(${KOKKOS_PKG_SOURCES_DIR}) endif() + +set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") + +# detects styles which have KOKKOS version +RegisterStylesExt(${KOKKOS_PKG_SOURCES_DIR} kokkos KOKKOS_PKG_SOURCES) + +# register kokkos-only styles +RegisterNBinStyle(${KOKKOS_PKG_SOURCES_DIR}/nbin_kokkos.h) +RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_kokkos.h) +RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_halffull_kokkos.h) + +if(PKG_USER-DPD) + get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) + list(APPEND KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.cpp) + RegisterNPairStyle(${KOKKOS_PKG_SOURCES_DIR}/npair_ssa_kokkos.h) + set_property(GLOBAL PROPERTY "KOKKOS_PKG_SOURCES" "${KOKKOS_PKG_SOURCES}") +endif() + +get_property(KOKKOS_PKG_SOURCES GLOBAL PROPERTY KOKKOS_PKG_SOURCES) + +target_sources(lammps PRIVATE ${KOKKOS_PKG_SOURCES}) +target_include_directories(lammps PRIVATE ${KOKKOS_PKG_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/KSPACE.cmake b/cmake/Modules/Packages/KSPACE.cmake index 07612447f9..dce0c15ef6 100644 --- a/cmake/Modules/Packages/KSPACE.cmake +++ b/cmake/Modules/Packages/KSPACE.cmake @@ -1,60 +1,56 @@ -if(PKG_KSPACE) - option(FFT_SINGLE "Use single precision FFTs instead of double precision FFTs" OFF) - set(FFTW "FFTW3") - if(FFT_SINGLE) - set(FFTW "FFTW3F") - add_definitions(-DFFT_SINGLE) - endif() - find_package(${FFTW} QUIET) - if(${FFTW}_FOUND) - set(FFT "FFTW3" CACHE STRING "FFT library for KSPACE package") - else() - set(FFT "KISS" CACHE STRING "FFT library for KSPACE package") - endif() - set(FFT_VALUES KISS FFTW3 MKL) - set_property(CACHE FFT PROPERTY STRINGS ${FFT_VALUES}) - validate_option(FFT FFT_VALUES) - string(TOUPPER ${FFT} FFT) - - if(FFT STREQUAL "FFTW3") - find_package(${FFTW} REQUIRED) - add_definitions(-DFFT_FFTW3) - include_directories(${${FFTW}_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${${FFTW}_LIBRARIES}) - if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) - option(FFT_FFTW_THREADS "Use threaded FFTW library" ON) - else() - option(FFT_FFTW_THREADS "Use threaded FFT library" OFF) - endif() - - if(FFT_FFTW_THREADS) - if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) - add_definitions(-DFFT_FFTW_THREADS) - list(APPEND LAMMPS_LINK_LIBS ${${FFTW}_OMP_LIBRARIES}) - else() - message(FATAL_ERROR "Need OpenMP enabled FFTW3 library for FFT_THREADS") - endif() - endif() - elseif(FFT STREQUAL "MKL") - find_package(MKL REQUIRED) - add_definitions(-DFFT_MKL) - option(FFT_MKL_THREADS "Use threaded MKL FFT" ON) - if(FFT_MKL_THREADS) - add_definitions(-DFFT_MKL_THREADS) - endif() - include_directories(${MKL_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${MKL_LIBRARIES}) - else() - # last option is KISSFFT - add_definitions(-DFFT_KISS) - endif() - - set(FFT_PACK "array" CACHE STRING "Optimization for FFT") - set(FFT_PACK_VALUES array pointer memcpy) - set_property(CACHE FFT_PACK PROPERTY STRINGS ${FFT_PACK_VALUES}) - validate_option(FFT_PACK FFT_PACK_VALUES) - if(NOT FFT_PACK STREQUAL "array") - string(TOUPPER ${FFT_PACK} FFT_PACK) - add_definitions(-DFFT_PACK_${FFT_PACK}) - endif() +option(FFT_SINGLE "Use single precision FFTs instead of double precision FFTs" OFF) +set(FFTW "FFTW3") +if(FFT_SINGLE) + set(FFTW "FFTW3F") + target_compile_definitions(lammps PUBLIC -DFFT_SINGLE) +endif() +find_package(${FFTW} QUIET) +if(${FFTW}_FOUND) + set(FFT "FFTW3" CACHE STRING "FFT library for KSPACE package") +else() + set(FFT "KISS" CACHE STRING "FFT library for KSPACE package") +endif() +set(FFT_VALUES KISS FFTW3 MKL) +set_property(CACHE FFT PROPERTY STRINGS ${FFT_VALUES}) +validate_option(FFT FFT_VALUES) +string(TOUPPER ${FFT} FFT) + +if(FFT STREQUAL "FFTW3") + find_package(${FFTW} REQUIRED) + target_compile_definitions(lammps PUBLIC -DFFT_FFTW3) + target_link_libraries(lammps PUBLIC ${FFTW}::${FFTW}) + if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) + option(FFT_FFTW_THREADS "Use threaded FFTW library" ON) + else() + option(FFT_FFTW_THREADS "Use threaded FFT library" OFF) + endif() + + if(FFT_FFTW_THREADS) + if(FFTW3_OMP_LIBRARY OR FFTW3F_OMP_LIBRARY) + target_compile_definitions(lammps PRIVATE -DFFT_FFTW_THREADS) + target_link_libraries(lammps PRIVATE ${FFTW}::${FFTW}_OMP) + else() + message(FATAL_ERROR "Need OpenMP enabled FFTW3 library for FFT_THREADS") + endif() + endif() +elseif(FFT STREQUAL "MKL") + find_package(MKL REQUIRED) + target_compile_definitions(lammps PRIVATE -DFFT_MKL) + option(FFT_MKL_THREADS "Use threaded MKL FFT" ON) + if(FFT_MKL_THREADS) + target_compile_definitions(lammps PRIVATE -DFFT_MKL_THREADS) + endif() + target_link_libraries(lammps PRIVATE MKL::MKL) +else() + # last option is KISSFFT + target_compile_definitions(lammps PRIVATE -DFFT_KISS) +endif() + +set(FFT_PACK "array" CACHE STRING "Optimization for FFT") +set(FFT_PACK_VALUES array pointer memcpy) +set_property(CACHE FFT_PACK PROPERTY STRINGS ${FFT_PACK_VALUES}) +validate_option(FFT_PACK FFT_PACK_VALUES) +if(NOT FFT_PACK STREQUAL "array") + string(TOUPPER ${FFT_PACK} FFT_PACK) + target_compile_definitions(lammps PRIVATE -DFFT_PACK_${FFT_PACK}) endif() diff --git a/cmake/Modules/Packages/LATTE.cmake b/cmake/Modules/Packages/LATTE.cmake index 9f665d7f0e..e66f83fa43 100644 --- a/cmake/Modules/Packages/LATTE.cmake +++ b/cmake/Modules/Packages/LATTE.cmake @@ -1,40 +1,40 @@ -if(PKG_LATTE) - enable_language(Fortran) - find_package(LATTE) - if(LATTE_FOUND) - set(DOWNLOAD_LATTE_DEFAULT OFF) - else() - set(DOWNLOAD_LATTE_DEFAULT ON) - endif() - option(DOWNLOAD_LATTE "Download the LATTE library instead of using an already installed one" ${DOWNLOAD_LATTE_DEFAULT}) - if(DOWNLOAD_LATTE) - message(STATUS "LATTE download requested - we will build our own") - # Workaround for cross compilation with MinGW where ${CMAKE_INSTALL_LIBDIR} - # is a full path, so we need to remove the prefix - string(REPLACE ${CMAKE_INSTALL_PREFIX} "" _LATTE_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - include(ExternalProject) - ExternalProject_Add(latte_build - URL https://github.com/lanl/LATTE/archive/v1.2.1.tar.gz - URL_MD5 85ac414fdada2d04619c8f936344df14 - SOURCE_SUBDIR cmake - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} - -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS} - -DCMAKE_Fortran_FLAGS_${BTYPE}=${CMAKE_Fortran_FLAGS_${BTYPE}} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_BYPRODUCTS /${_LATTE_LIBDIR}/liblatte.a - ) - list(APPEND LAMMPS_DEPS latte_build) - ExternalProject_get_property(latte_build INSTALL_DIR) - set(LATTE_LIBRARIES ${INSTALL_DIR}/${_LATTE_LIBDIR}/liblatte.a) - else() - find_package(LATTE) - if(NOT LATTE_FOUND) - message(FATAL_ERROR "LATTE library not found, help CMake to find it by setting LATTE_LIBRARY, or set DOWNLOAD_LATTE=ON to download it") - endif() - endif() - if(NOT LAPACK_FOUND) - add_dependencies(latte_build linalg) - endif() - list(APPEND LAMMPS_LINK_LIBS ${LATTE_LIBRARIES} ${LAPACK_LIBRARIES}) +enable_language(Fortran) + +# using lammps in a super-build setting +if(TARGET LATTE::latte) + target_link_libraries(lammps PRIVATE LATTE::latte) + return() +endif() + +find_package(LATTE 1.2.2 CONFIG) +if(LATTE_FOUND) + set(DOWNLOAD_LATTE_DEFAULT OFF) +else() + set(DOWNLOAD_LATTE_DEFAULT ON) +endif() +option(DOWNLOAD_LATTE "Download the LATTE library instead of using an already installed one" ${DOWNLOAD_LATTE_DEFAULT}) +if(DOWNLOAD_LATTE) + message(STATUS "LATTE download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(latte_build + URL https://github.com/lanl/LATTE/archive/v1.2.2.tar.gz + URL_MD5 820e73a457ced178c08c71389a385de7 + SOURCE_SUBDIR cmake + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= ${CMAKE_REQUEST_PIC} -DCMAKE_INSTALL_LIBDIR=lib + -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS} + -DCMAKE_Fortran_FLAGS_${BTYPE}=${CMAKE_Fortran_FLAGS_${BTYPE}} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_BYPRODUCTS /lib/liblatte.a + ) + ExternalProject_get_property(latte_build INSTALL_DIR) + add_library(LAMMPS::LATTE UNKNOWN IMPORTED) + set_target_properties(LAMMPS::LATTE PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/liblatte.a" + INTERFACE_LINK_LIBRARIES "${LAPACK_LIBRARIES}") + target_link_libraries(lammps PRIVATE LAMMPS::LATTE) + add_dependencies(LAMMPS::LATTE latte_build) +else() + find_package(LATTE 1.2.2 REQUIRED CONFIG) + target_link_libraries(lammps PRIVATE LATTE::latte) endif() diff --git a/cmake/Modules/Packages/MESSAGE.cmake b/cmake/Modules/Packages/MESSAGE.cmake index aff9c2964a..fb62763828 100644 --- a/cmake/Modules/Packages/MESSAGE.cmake +++ b/cmake/Modules/Packages/MESSAGE.cmake @@ -1,32 +1,32 @@ -if(PKG_MESSAGE) - if(LAMMPS_SIZES STREQUAL BIGBIG) - message(FATAL_ERROR "The MESSAGE Package is not compatible with -DLAMMPS_BIGBIG") - endif() - option(MESSAGE_ZMQ "Use ZeroMQ in MESSAGE package" OFF) - file(GLOB_RECURSE cslib_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.F - ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.c - ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.cpp) - - add_library(cslib STATIC ${cslib_SOURCES}) - if(BUILD_MPI) - target_compile_definitions(cslib PRIVATE -DMPI_YES) - set_target_properties(cslib PROPERTIES OUTPUT_NAME "csmpi") - else() - target_compile_definitions(cslib PRIVATE -DMPI_NO) - target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_MPI) - set_target_properties(cslib PROPERTIES OUTPUT_NAME "csnompi") - endif() - - if(MESSAGE_ZMQ) - target_compile_definitions(cslib PRIVATE -DZMQ_YES) - find_package(ZMQ REQUIRED) - target_include_directories(cslib PRIVATE ${ZMQ_INCLUDE_DIRS}) - target_link_libraries(cslib PUBLIC ${ZMQ_LIBRARIES}) - else() - target_compile_definitions(cslib PRIVATE -DZMQ_NO) - target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_ZMQ) - endif() - - list(APPEND LAMMPS_LINK_LIBS cslib) - include_directories(${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src) +if(LAMMPS_SIZES STREQUAL BIGBIG) + message(FATAL_ERROR "The MESSAGE Package is not compatible with -DLAMMPS_BIGBIG") endif() +option(MESSAGE_ZMQ "Use ZeroMQ in MESSAGE package" OFF) +file(GLOB_RECURSE cslib_SOURCES ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.F + ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.c + ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/[^.]*.cpp) + +add_library(cslib STATIC ${cslib_SOURCES}) +target_compile_definitions(cslib PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(cslib PROPERTIES OUTPUT_NAME lammps_cslib${LAMMPS_MACHINE}) +if(BUILD_MPI) + target_compile_definitions(cslib PRIVATE -DMPI_YES) + set_target_properties(cslib PROPERTIES OUTPUT_NAME "csmpi") + target_link_libraries(cslib PRIVATE MPI::MPI_CXX) +else() + target_compile_definitions(cslib PRIVATE -DMPI_NO) + target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_MPI) + set_target_properties(cslib PROPERTIES OUTPUT_NAME "csnompi") +endif() + +if(MESSAGE_ZMQ) + target_compile_definitions(cslib PRIVATE -DZMQ_YES) + find_package(ZMQ REQUIRED) + target_link_libraries(cslib PUBLIC ZMQ::ZMQ) +else() + target_compile_definitions(cslib PRIVATE -DZMQ_NO) + target_include_directories(cslib PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src/STUBS_ZMQ) +endif() + +target_link_libraries(lammps PRIVATE cslib) +target_include_directories(lammps PRIVATE ${LAMMPS_LIB_SOURCE_DIR}/message/cslib/src) diff --git a/cmake/Modules/Packages/MSCG.cmake b/cmake/Modules/Packages/MSCG.cmake index 35f0c57449..55f71588fe 100644 --- a/cmake/Modules/Packages/MSCG.cmake +++ b/cmake/Modules/Packages/MSCG.cmake @@ -1,48 +1,45 @@ -if(PKG_MSCG) - find_package(GSL REQUIRED) - find_package(MSCG QUIET) - if(MSGC_FOUND) - set(DOWNLOAD_MSCG_DEFAULT OFF) - else() - set(DOWNLOAD_MSCG_DEFAULT ON) - endif() - option(DOWNLOAD_MSCG "Download MSCG library instead of using an already installed one)" ${DOWNLOAD_MSCG_DEFAULT}) - if(DOWNLOAD_MSCG) - include(ExternalProject) - if(NOT LAPACK_FOUND) - set(EXTRA_MSCG_OPTS "-DLAPACK_LIBRARIES=${CMAKE_CURRENT_BINARY_DIR}/liblinalg.a") - endif() - ExternalProject_Add(mscg_build - URL https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz - URL_MD5 8c45e269ee13f60b303edd7823866a91 - SOURCE_SUBDIR src/CMake - CMAKE_ARGS ${CMAKE_REQUEST_PIC} ${EXTRA_MSCG_OPTS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} - -DCMAKE_INSTALL_PREFIX= - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - BUILD_COMMAND ${CMAKE_COMMAND} --build . --target mscg - INSTALL_COMMAND "" - BUILD_BYPRODUCTS /libmscg.a - ) - ExternalProject_get_property(mscg_build BINARY_DIR) - set(MSCG_LIBRARIES ${BINARY_DIR}/libmscg.a) - ExternalProject_get_property(mscg_build SOURCE_DIR) - set(MSCG_INCLUDE_DIRS ${SOURCE_DIR}/src) - list(APPEND LAMMPS_DEPS mscg_build) - if(NOT LAPACK_FOUND) - file(MAKE_DIRECTORY ${MSCG_INCLUDE_DIRS}) - add_dependencies(mscg_build linalg) - endif() - else() - find_package(MSCG) - if(NOT MSCG_FOUND) - message(FATAL_ERROR "MSCG not found, help CMake to find it by setting MSCG_LIBRARY and MSCG_INCLUDE_DIRS, or set DOWNLOAD_MSCG=ON to download it") - endif() - endif() - list(APPEND LAMMPS_LINK_LIBS ${MSCG_LIBRARIES} ${GSL_LIBRARIES} ${LAPACK_LIBRARIES}) - include_directories(${MSCG_INCLUDE_DIRS}) +find_package(GSL REQUIRED) +find_package(MSCG QUIET) +if(MSGC_FOUND) + set(DOWNLOAD_MSCG_DEFAULT OFF) +else() + set(DOWNLOAD_MSCG_DEFAULT ON) endif() +option(DOWNLOAD_MSCG "Download MSCG library instead of using an already installed one)" ${DOWNLOAD_MSCG_DEFAULT}) +if(DOWNLOAD_MSCG) + include(ExternalProject) + ExternalProject_Add(mscg_build + URL https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz + URL_MD5 8c45e269ee13f60b303edd7823866a91 + SOURCE_SUBDIR src/CMake + CMAKE_ARGS ${CMAKE_REQUEST_PIC} ${EXTRA_MSCG_OPTS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DBLAS_LIBRARIES=${BLAS_LIBRARIES} -DLAPACK_LIBRARIES=${LAPACK_LIBRARIES} + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + BUILD_COMMAND ${CMAKE_COMMAND} --build . --target mscg + INSTALL_COMMAND "" + BUILD_BYPRODUCTS /libmscg.a + ) + ExternalProject_get_property(mscg_build BINARY_DIR) + ExternalProject_get_property(mscg_build SOURCE_DIR) + file(MAKE_DIRECTORY ${SOURCE_DIR}/src) + add_library(LAMMPS::MSCG UNKNOWN IMPORTED) + set_target_properties(LAMMPS::MSCG PROPERTIES + IMPORTED_LOCATION "${BINARY_DIR}/libmscg.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/src" + INTERFACE_LINK_LIBRARIES "${LAPACK_LIBRARIES}") + target_link_libraries(lammps PRIVATE LAMMPS::MSCG) + add_dependencies(LAMMPS::MSCG mscg_build) +else() + find_package(MSCG) + if(NOT MSCG_FOUND) + message(FATAL_ERROR "MSCG not found, help CMake to find it by setting MSCG_LIBRARY and MSCG_INCLUDE_DIRS, or set DOWNLOAD_MSCG=ON to download it") + endif() + target_link_libraries(lammps PRIVATE MSCG::MSCG) +endif() +target_link_libraries(lammps PRIVATE GSL::gsl ${LAPACK_LIBRARIES}) diff --git a/cmake/Modules/Packages/OPT.cmake b/cmake/Modules/Packages/OPT.cmake index f2802c757b..bcf572b087 100644 --- a/cmake/Modules/Packages/OPT.cmake +++ b/cmake/Modules/Packages/OPT.cmake @@ -1,13 +1,11 @@ -if(PKG_OPT) - set(OPT_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/OPT) - set(OPT_SOURCES) - set_property(GLOBAL PROPERTY "OPT_SOURCES" "${OPT_SOURCES}") + set(OPT_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/OPT) + set(OPT_SOURCES) + set_property(GLOBAL PROPERTY "OPT_SOURCES" "${OPT_SOURCES}") - # detects styles which have OPT version - RegisterStylesExt(${OPT_SOURCES_DIR} opt OPT_SOURCES) + # detects styles which have OPT version + RegisterStylesExt(${OPT_SOURCES_DIR} opt OPT_SOURCES) - get_property(OPT_SOURCES GLOBAL PROPERTY OPT_SOURCES) + get_property(OPT_SOURCES GLOBAL PROPERTY OPT_SOURCES) - list(APPEND LIB_SOURCES ${OPT_SOURCES}) - include_directories(${OPT_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${OPT_SOURCES}) + target_include_directories(lammps PRIVATE ${OPT_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/PYTHON.cmake b/cmake/Modules/Packages/PYTHON.cmake index 4f8959ae38..a577f824fe 100644 --- a/cmake/Modules/Packages/PYTHON.cmake +++ b/cmake/Modules/Packages/PYTHON.cmake @@ -1,6 +1,9 @@ -if(PKG_PYTHON) - find_package(PythonLibs REQUIRED) - add_definitions(-DLMP_PYTHON) - include_directories(${PYTHON_INCLUDE_DIR}) - list(APPEND LAMMPS_LINK_LIBS ${PYTHON_LIBRARY}) +if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonLibs REQUIRED) # Deprecated since version 3.12 + target_include_directories(lammps PRIVATE ${PYTHON_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE ${PYTHON_LIBRARY}) +else() + find_package(Python REQUIRED COMPONENTS Development) + target_link_libraries(lammps PRIVATE Python::Python) endif() +target_compile_definitions(lammps PRIVATE -DLMP_PYTHON) diff --git a/cmake/Modules/Packages/QEQ.cmake b/cmake/Modules/Packages/QEQ.cmake index 94cca30540..1bdaa02f43 100644 --- a/cmake/Modules/Packages/QEQ.cmake +++ b/cmake/Modules/Packages/QEQ.cmake @@ -1,20 +1,18 @@ # Fix qeq/fire requires MANYBODY (i.e. COMB and COMB3) to be installed -if(PKG_QEQ) - set(QEQ_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/QEQ) - file(GLOB QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix*.h) - file(GLOB QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix*.cpp) +set(QEQ_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/QEQ) +file(GLOB QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix*.h) +file(GLOB QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix*.cpp) - if(NOT PKG_MANYBODY) - list(REMOVE_ITEM QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix_qeq_fire.h) - list(REMOVE_ITEM QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix_qeq_fire.cpp) - endif() - set_property(GLOBAL PROPERTY "QEQ_SOURCES" "${QEQ_SOURCES}") - - foreach(MY_HEADER ${QEQ_HEADERS}) - AddStyleHeader(${MY_HEADER} FIX) - endforeach() - - get_property(QEQ_SOURCES GLOBAL PROPERTY QEQ_SOURCES) - list(APPEND LIB_SOURCES ${QEQ_SOURCES}) - include_directories(${QEQ_SOURCES_DIR}) +if(NOT PKG_MANYBODY) + list(REMOVE_ITEM QEQ_HEADERS ${QEQ_SOURCES_DIR}/fix_qeq_fire.h) + list(REMOVE_ITEM QEQ_SOURCES ${QEQ_SOURCES_DIR}/fix_qeq_fire.cpp) endif() +set_property(GLOBAL PROPERTY "QEQ_SOURCES" "${QEQ_SOURCES}") + +foreach(MY_HEADER ${QEQ_HEADERS}) + AddStyleHeader(${MY_HEADER} FIX) +endforeach() + +get_property(QEQ_SOURCES GLOBAL PROPERTY QEQ_SOURCES) +target_sources(lammps PRIVATE ${QEQ_SOURCES}) +target_include_directories(lammps PRIVATE ${QEQ_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-COLVARS.cmake b/cmake/Modules/Packages/USER-COLVARS.cmake index a112fbb6aa..92595a9c8a 100644 --- a/cmake/Modules/Packages/USER-COLVARS.cmake +++ b/cmake/Modules/Packages/USER-COLVARS.cmake @@ -1,27 +1,30 @@ -if(PKG_USER-COLVARS) +set(COLVARS_SOURCE_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars) - set(COLVARS_SOURCE_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars) +file(GLOB COLVARS_SOURCES ${COLVARS_SOURCE_DIR}/[^.]*.cpp) - file(GLOB COLVARS_SOURCES ${COLVARS_SOURCE_DIR}/[^.]*.cpp) - - # Build Lepton by default - option(COLVARS_LEPTON "Build and link the Lepton library" ON) - - if(COLVARS_LEPTON) - set(LEPTON_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars/lepton) - file(GLOB LEPTON_SOURCES ${LEPTON_DIR}/src/[^.]*.cpp) - add_library(lepton STATIC ${LEPTON_SOURCES}) - target_include_directories(lepton PRIVATE ${LEPTON_DIR}/include) - endif() - - add_library(colvars STATIC ${COLVARS_SOURCES}) - target_include_directories(colvars PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/colvars) - list(APPEND LAMMPS_LINK_LIBS colvars) - - if(COLVARS_LEPTON) - list(APPEND LAMMPS_LINK_LIBS lepton) - target_compile_options(colvars PRIVATE -DLEPTON) - target_include_directories(colvars PUBLIC ${LEPTON_DIR}/include) - endif() +# Build Lepton by default +option(COLVARS_LEPTON "Build and link the Lepton library" ON) +if(COLVARS_LEPTON) + set(LEPTON_DIR ${LAMMPS_LIB_SOURCE_DIR}/colvars/lepton) + file(GLOB LEPTON_SOURCES ${LEPTON_DIR}/src/[^.]*.cpp) + add_library(lepton STATIC ${LEPTON_SOURCES}) + # Change the define below to LEPTON_BUILDING_SHARED_LIBRARY when linking Lepton as a DLL with MSVC + target_compile_definitions(lepton PRIVATE -DLEPTON_BUILDING_STATIC_LIBRARY) + set_target_properties(lepton PROPERTIES OUTPUT_NAME lammps_lepton${LAMMPS_MACHINE}) + target_include_directories(lepton PRIVATE ${LEPTON_DIR}/include) +endif() + +add_library(colvars STATIC ${COLVARS_SOURCES}) +target_compile_definitions(colvars PRIVATE -DLAMMPS_${LAMMPS_SIZES}) +set_target_properties(colvars PROPERTIES OUTPUT_NAME lammps_colvars${LAMMPS_MACHINE}) +target_include_directories(colvars PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/colvars) +target_link_libraries(lammps PRIVATE colvars) + +if(COLVARS_LEPTON) + target_link_libraries(lammps PRIVATE lepton) + target_compile_definitions(colvars PRIVATE -DLEPTON) + # Disable the line below when linking Lepton as a DLL with MSVC + target_compile_definitions(colvars PRIVATE -DLEPTON_USE_STATIC_LIBRARIES) + target_include_directories(colvars PUBLIC ${LEPTON_DIR}/include) endif() diff --git a/cmake/Modules/Packages/USER-H5MD.cmake b/cmake/Modules/Packages/USER-H5MD.cmake index 40ea7b7444..6e8e425673 100644 --- a/cmake/Modules/Packages/USER-H5MD.cmake +++ b/cmake/Modules/Packages/USER-H5MD.cmake @@ -1,8 +1,8 @@ -if(PKG_USER-H5MD) - enable_language(C) +enable_language(C) - find_package(HDF5 REQUIRED) - target_link_libraries(h5md ${HDF5_LIBRARIES}) - target_include_directories(h5md PRIVATE ${HDF5_INCLUDE_DIRS}) - include_directories(${HDF5_INCLUDE_DIRS}) +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.12) + cmake_policy(SET CMP0074 NEW) endif() +find_package(HDF5 REQUIRED) +target_link_libraries(h5md PRIVATE ${HDF5_LIBRARIES}) +target_include_directories(h5md PUBLIC ${HDF5_INCLUDE_DIRS}) diff --git a/cmake/Modules/Packages/USER-INTEL.cmake b/cmake/Modules/Packages/USER-INTEL.cmake index 9ae4333ee2..0ae1ec018a 100644 --- a/cmake/Modules/Packages/USER-INTEL.cmake +++ b/cmake/Modules/Packages/USER-INTEL.cmake @@ -1,113 +1,109 @@ -if(PKG_USER-INTEL) - check_include_file_cxx(immintrin.h FOUND_IMMINTRIN) - if(NOT FOUND_IMMINTRIN) - message(FATAL_ERROR "immintrin.h header not found, Intel package won't work without it") - endif() - - add_definitions(-DLMP_USER_INTEL) - - set(INTEL_ARCH "cpu" CACHE STRING "Architectures used by USER-INTEL (cpu or knl)") - set(INTEL_ARCH_VALUES cpu knl) - set_property(CACHE INTEL_ARCH PROPERTY STRINGS ${INTEL_ARCH_VALUES}) - validate_option(INTEL_ARCH INTEL_ARCH_VALUES) - string(TOUPPER ${INTEL_ARCH} INTEL_ARCH) - - find_package(Threads QUIET) - if(Threads_FOUND) - set(INTEL_LRT_MODE "threads" CACHE STRING "Long-range threads mode (none, threads, or c++11)") - else() - set(INTEL_LRT_MODE "none" CACHE STRING "Long-range threads mode (none, threads, or c++11)") - endif() - set(INTEL_LRT_VALUES none threads c++11) - set_property(CACHE INTEL_LRT_MODE PROPERTY STRINGS ${INTEL_LRT_VALUES}) - validate_option(INTEL_LRT_MODE INTEL_LRT_VALUES) - string(TOUPPER ${INTEL_LRT_MODE} INTEL_LRT_MODE) - if(INTEL_LRT_MODE STREQUAL "THREADS") - if(Threads_FOUND) - add_definitions(-DLMP_INTEL_USELRT) - list(APPEND LAMMPS_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) - else() - message(FATAL_ERROR "Must have working threads library for Long-range thread support") - endif() - endif() - if(INTEL_LRT_MODE STREQUAL "C++11") - add_definitions(-DLMP_INTEL_USELRT -DLMP_INTEL_LRT11) - endif() - - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) - message(FATAL_ERROR "USER-INTEL needs at least a 2016 Intel compiler, found ${CMAKE_CXX_COMPILER_VERSION}") - endif() - else() - message(WARNING "USER-INTEL gives best performance with Intel compilers") - endif() - - find_package(TBB QUIET) - if(TBB_FOUND) - list(APPEND LAMMPS_LINK_LIBS ${TBB_MALLOC_LIBRARIES}) - else() - add_definitions(-DLMP_INTEL_NO_TBB) - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - message(WARNING "USER-INTEL with Intel compilers should use TBB malloc libraries") - endif() - endif() - - find_package(MKL QUIET) - if(MKL_FOUND) - add_definitions(-DLMP_USE_MKL_RNG) - list(APPEND LAMMPS_LINK_LIBS ${MKL_LIBRARIES}) - else() - message(STATUS "Pair style dpd/intel will be faster with MKL libraries") - endif() - - if((NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "64") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "128") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "256")) - message(FATAL_ERROR "USER-INTEL only supports memory alignment of 64, 128 or 256 on this platform") - endif() - - if(INTEL_ARCH STREQUAL "KNL") - if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - message(FATAL_ERROR "Must use Intel compiler with USER-INTEL for KNL architecture") - endif() - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -xHost -qopenmp -qoffload") - set(MIC_OPTIONS "-qoffload-option,mic,compiler,\"-fp-model fast=2 -mGLOB_default_function_attrs=\\\"gather_scatter_loop_unroll=4\\\"\"") - add_compile_options(-xMIC-AVX512 -qoffload -fno-alias -ansi-alias -restrict -qoverride-limits ${MIC_OPTIONS}) - add_definitions(-DLMP_INTEL_OFFLOAD) - else() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - include(CheckCXXCompilerFlag) - foreach(_FLAG -O2 -fp-model fast=2 -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) - check_cxx_compiler_flag("${__FLAG}" COMPILER_SUPPORTS${_FLAG}) - if(COMPILER_SUPPORTS${_FLAG}) - add_compile_options(${_FLAG}) - endif() - endforeach() - else() - add_compile_options(-O3 -ffast-math) - endif() - endif() - - # collect sources - set(USER-INTEL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-INTEL) - set(USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/fix_intel.cpp - ${USER-INTEL_SOURCES_DIR}/fix_nh_intel.cpp - ${USER-INTEL_SOURCES_DIR}/intel_buffers.cpp - ${USER-INTEL_SOURCES_DIR}/nbin_intel.cpp - ${USER-INTEL_SOURCES_DIR}/npair_intel.cpp) - - set_property(GLOBAL PROPERTY "USER-INTEL_SOURCES" "${USER-INTEL_SOURCES}") - - # detect styles which have a USER-INTEL version - RegisterStylesExt(${USER-INTEL_SOURCES_DIR} intel USER-INTEL_SOURCES) - RegisterNBinStyle(${USER-INTEL_SOURCES_DIR}/nbin_intel.h) - RegisterNPairStyle(${USER-INTEL_SOURCES_DIR}/npair_intel.h) - RegisterFixStyle(${USER-INTEL_SOURCES_DIR}/fix_intel.h) - - get_property(USER-INTEL_SOURCES GLOBAL PROPERTY USER-INTEL_SOURCES) - if(PKG_KSPACE) - list(APPEND USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.cpp) - RegisterIntegrateStyle(${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.h) - endif() - - list(APPEND LIB_SOURCES ${USER-INTEL_SOURCES}) - include_directories(${USER-INTEL_SOURCES_DIR}) +check_include_file_cxx(immintrin.h FOUND_IMMINTRIN) +if(NOT FOUND_IMMINTRIN) + message(FATAL_ERROR "immintrin.h header not found, Intel package won't work without it") endif() + +target_compile_definitions(lammps PRIVATE -DLMP_USER_INTEL) + +set(INTEL_ARCH "cpu" CACHE STRING "Architectures used by USER-INTEL (cpu or knl)") +set(INTEL_ARCH_VALUES cpu knl) +set_property(CACHE INTEL_ARCH PROPERTY STRINGS ${INTEL_ARCH_VALUES}) +validate_option(INTEL_ARCH INTEL_ARCH_VALUES) +string(TOUPPER ${INTEL_ARCH} INTEL_ARCH) + +find_package(Threads QUIET) +if(Threads_FOUND) + set(INTEL_LRT_MODE "threads" CACHE STRING "Long-range threads mode (none, threads, or c++11)") +else() + set(INTEL_LRT_MODE "none" CACHE STRING "Long-range threads mode (none, threads, or c++11)") +endif() +set(INTEL_LRT_VALUES none threads c++11) +set_property(CACHE INTEL_LRT_MODE PROPERTY STRINGS ${INTEL_LRT_VALUES}) +validate_option(INTEL_LRT_MODE INTEL_LRT_VALUES) +string(TOUPPER ${INTEL_LRT_MODE} INTEL_LRT_MODE) +if(INTEL_LRT_MODE STREQUAL "THREADS") + if(Threads_FOUND) + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_USELRT) + target_link_libraries(lammps PRIVATE Threads::Threads) + else() + message(FATAL_ERROR "Must have working threads library for Long-range thread support") + endif() +endif() +if(INTEL_LRT_MODE STREQUAL "C++11") + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_USELRT -DLMP_INTEL_LRT11) +endif() + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16) + message(FATAL_ERROR "USER-INTEL needs at least a 2016 Intel compiler, found ${CMAKE_CXX_COMPILER_VERSION}") + endif() +else() + message(WARNING "USER-INTEL gives best performance with Intel compilers") +endif() + +find_package(TBB_MALLOC QUIET) +if(TBB_MALLOC_FOUND) + target_link_libraries(lammps PRIVATE TBB::TBB_MALLOC) +else() + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_NO_TBB) + if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + message(WARNING "USER-INTEL with Intel compilers should use TBB malloc libraries") + endif() +endif() + +find_package(MKL QUIET) +if(MKL_FOUND) + target_compile_definitions(lammps PRIVATE -DLMP_USE_MKL_RNG) + target_link_libraries(lammps PRIVATE MKL::MKL) +else() + message(STATUS "Pair style dpd/intel will be faster with MKL libraries") +endif() + +if((NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "64") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "128") AND (NOT ${LAMMPS_MEMALIGN} STREQUAL "256")) + message(FATAL_ERROR "USER-INTEL only supports memory alignment of 64, 128 or 256 on this platform") +endif() + +if(INTEL_ARCH STREQUAL "KNL") + if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + message(FATAL_ERROR "Must use Intel compiler with USER-INTEL for KNL architecture") + endif() + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -xHost -qopenmp -qoffload") + set(MIC_OPTIONS "-qoffload-option,mic,compiler,\"-fp-model fast=2 -mGLOB_default_function_attrs=\\\"gather_scatter_loop_unroll=4\\\"\"") + target_compile_options(lammps PRIVATE -xMIC-AVX512 -qoffload -fno-alias -ansi-alias -restrict -qoverride-limits ${MIC_OPTIONS}) + target_compile_definitions(lammps PRIVATE -DLMP_INTEL_OFFLOAD) +else() + if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + include(CheckCXXCompilerFlag) + foreach(_FLAG -O2 -fp-model fast=2 -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) + check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAG}) + if(COMPILER_SUPPORTS${_FLAG}) + target_compile_options(lammps PRIVATE ${_FLAG}) + endif() + endforeach() + endif() +endif() + +# collect sources +set(USER-INTEL_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-INTEL) +set(USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/fix_intel.cpp + ${USER-INTEL_SOURCES_DIR}/fix_nh_intel.cpp + ${USER-INTEL_SOURCES_DIR}/intel_buffers.cpp + ${USER-INTEL_SOURCES_DIR}/nbin_intel.cpp + ${USER-INTEL_SOURCES_DIR}/npair_intel.cpp) + +set_property(GLOBAL PROPERTY "USER-INTEL_SOURCES" "${USER-INTEL_SOURCES}") + +# detect styles which have a USER-INTEL version +RegisterStylesExt(${USER-INTEL_SOURCES_DIR} intel USER-INTEL_SOURCES) +RegisterNBinStyle(${USER-INTEL_SOURCES_DIR}/nbin_intel.h) +RegisterNPairStyle(${USER-INTEL_SOURCES_DIR}/npair_intel.h) +RegisterFixStyle(${USER-INTEL_SOURCES_DIR}/fix_intel.h) + +get_property(USER-INTEL_SOURCES GLOBAL PROPERTY USER-INTEL_SOURCES) +if(PKG_KSPACE) + list(APPEND USER-INTEL_SOURCES ${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.cpp) + RegisterIntegrateStyle(${USER-INTEL_SOURCES_DIR}/verlet_lrt_intel.h) +endif() + +target_sources(lammps PRIVATE ${USER-INTEL_SOURCES}) +target_include_directories(lammps PRIVATE ${USER-INTEL_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-MOLFILE.cmake b/cmake/Modules/Packages/USER-MOLFILE.cmake index 16ffc34994..1a2df2202b 100644 --- a/cmake/Modules/Packages/USER-MOLFILE.cmake +++ b/cmake/Modules/Packages/USER-MOLFILE.cmake @@ -1,10 +1,8 @@ -if(PKG_USER-MOLFILE) - set(MOLFILE_INCLUDE_DIRS "${LAMMPS_LIB_SOURCE_DIR}/molfile" CACHE STRING "Path to VMD molfile plugin headers") - add_library(molfile INTERFACE) - target_include_directories(molfile INTERFACE ${MOLFILE_INCLUDE_DIRS}) - # no need to link with -ldl on windows - if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - target_link_libraries(molfile INTERFACE ${CMAKE_DL_LIBS}) - endif() - list(APPEND LAMMPS_LINK_LIBS molfile) +set(MOLFILE_INCLUDE_DIRS "${LAMMPS_LIB_SOURCE_DIR}/molfile" CACHE STRING "Path to VMD molfile plugin headers") +add_library(molfile INTERFACE) +target_include_directories(molfile INTERFACE ${MOLFILE_INCLUDE_DIRS}) +# no need to link with -ldl on windows +if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + target_link_libraries(molfile INTERFACE ${CMAKE_DL_LIBS}) endif() +target_link_libraries(lammps PRIVATE molfile) diff --git a/cmake/Modules/Packages/USER-NETCDF.cmake b/cmake/Modules/Packages/USER-NETCDF.cmake index 921156f1e0..1bcbcadc35 100644 --- a/cmake/Modules/Packages/USER-NETCDF.cmake +++ b/cmake/Modules/Packages/USER-NETCDF.cmake @@ -1,24 +1,27 @@ -if(PKG_USER-NETCDF) - # USER-NETCDF can use NetCDF, Parallel NetCDF (PNetCDF), or both. At least one necessary. - # NetCDF library enables dump style "netcdf", while PNetCDF enables dump style "netcdf/mpiio" +# USER-NETCDF can use NetCDF, Parallel NetCDF (PNetCDF), or both. At least one necessary. +# NetCDF library enables dump style "netcdf", while PNetCDF enables dump style "netcdf/mpiio" +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.12) + cmake_policy(SET CMP0074 NEW) +endif() +if(NOT BUILD_MPI) + find_package(NetCDF REQUIRED) +else() find_package(NetCDF) if(NETCDF_FOUND) find_package(PNetCDF) - else(NETCDF_FOUND) + else() find_package(PNetCDF REQUIRED) - endif(NETCDF_FOUND) - - if(NETCDF_FOUND) - include_directories(${NETCDF_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${NETCDF_LIBRARIES}) - add_definitions(-DLMP_HAS_NETCDF) - endif(NETCDF_FOUND) - - if(PNETCDF_FOUND) - include_directories(${PNETCDF_INCLUDES}) - list(APPEND LAMMPS_LINK_LIBS ${PNETCDF_LIBRARIES}) - add_definitions(-DLMP_HAS_PNETCDF) - endif(PNETCDF_FOUND) - - add_definitions(-DNC_64BIT_DATA=0x0020) + endif() endif() + +if(NETCDF_FOUND) + target_link_libraries(lammps PRIVATE NetCDF::NetCDF) + target_compile_definitions(lammps PRIVATE -DLMP_HAS_NETCDF) +endif(NETCDF_FOUND) + +if(PNETCDF_FOUND) + target_link_libraries(lammps PRIVATE PNetCDF::PNetCDF) + target_compile_definitions(lammps PRIVATE -DLMP_HAS_PNETCDF) +endif(PNETCDF_FOUND) + +target_compile_definitions(lammps PRIVATE -DNC_64BIT_DATA=0x0020) diff --git a/cmake/Modules/Packages/USER-OMP.cmake b/cmake/Modules/Packages/USER-OMP.cmake index 668f42f10a..a1053ad355 100644 --- a/cmake/Modules/Packages/USER-OMP.cmake +++ b/cmake/Modules/Packages/USER-OMP.cmake @@ -1,42 +1,40 @@ -if(PKG_USER-OMP) - set(USER-OMP_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-OMP) - set(USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/thr_data.cpp - ${USER-OMP_SOURCES_DIR}/thr_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_nh_omp.cpp - ${USER-OMP_SOURCES_DIR}/fix_nh_sphere_omp.cpp - ${USER-OMP_SOURCES_DIR}/domain_omp.cpp) - add_definitions(-DLMP_USER_OMP) - set_property(GLOBAL PROPERTY "OMP_SOURCES" "${USER-OMP_SOURCES}") + set(USER-OMP_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-OMP) + set(USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/thr_data.cpp + ${USER-OMP_SOURCES_DIR}/thr_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_nh_omp.cpp + ${USER-OMP_SOURCES_DIR}/fix_nh_sphere_omp.cpp + ${USER-OMP_SOURCES_DIR}/domain_omp.cpp) + target_compile_definitions(lammps PRIVATE -DLMP_USER_OMP) + set_property(GLOBAL PROPERTY "OMP_SOURCES" "${USER-OMP_SOURCES}") - # detects styles which have USER-OMP version - RegisterStylesExt(${USER-OMP_SOURCES_DIR} omp OMP_SOURCES) - RegisterFixStyle(${USER-OMP_SOURCES_DIR}/fix_omp.h) + # detects styles which have USER-OMP version + RegisterStylesExt(${USER-OMP_SOURCES_DIR} omp OMP_SOURCES) + RegisterFixStyle(${USER-OMP_SOURCES_DIR}/fix_omp.h) - get_property(USER-OMP_SOURCES GLOBAL PROPERTY OMP_SOURCES) + get_property(USER-OMP_SOURCES GLOBAL PROPERTY OMP_SOURCES) - # manually add package dependent source files from USER-OMP that do not provide styles + # manually add package dependent source files from USER-OMP that do not provide styles - if(PKG_ASPHERE) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_nh_asphere_omp.cpp) - endif() + if(PKG_ASPHERE) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_nh_asphere_omp.cpp) + endif() - if(PKG_RIGID) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_rigid_nh_omp.cpp) - endif() + if(PKG_RIGID) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/fix_rigid_nh_omp.cpp) + endif() - if(PKG_USER-REAXC) - list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/reaxc_bond_orders_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_hydrogen_bonds_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_nonbonded_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_bonds_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_init_md_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_torsion_angles_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_forces_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_multi_body_omp.cpp - ${USER-OMP_SOURCES_DIR}/reaxc_valence_angles_omp.cpp) - endif() + if(PKG_USER-REAXC) + list(APPEND USER-OMP_SOURCES ${USER-OMP_SOURCES_DIR}/reaxc_bond_orders_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_hydrogen_bonds_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_nonbonded_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_bonds_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_init_md_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_torsion_angles_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_forces_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_multi_body_omp.cpp + ${USER-OMP_SOURCES_DIR}/reaxc_valence_angles_omp.cpp) + endif() - list(APPEND LIB_SOURCES ${USER-OMP_SOURCES}) - include_directories(${USER-OMP_SOURCES_DIR}) -endif() + target_sources(lammps PRIVATE ${USER-OMP_SOURCES}) + target_include_directories(lammps PRIVATE ${USER-OMP_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-PLUMED.cmake b/cmake/Modules/Packages/USER-PLUMED.cmake index 426ae2df2a..39148bdf3d 100644 --- a/cmake/Modules/Packages/USER-PLUMED.cmake +++ b/cmake/Modules/Packages/USER-PLUMED.cmake @@ -1,96 +1,98 @@ -if(PKG_USER-PLUMED) - set(PLUMED_MODE "static" CACHE STRING "Linkage mode for Plumed2 library") - set(PLUMED_MODE_VALUES static shared runtime) - set_property(CACHE PLUMED_MODE PROPERTY STRINGS ${PLUMED_MODE_VALUES}) - validate_option(PLUMED_MODE PLUMED_MODE_VALUES) - string(TOUPPER ${PLUMED_MODE} PLUMED_MODE) +set(PLUMED_MODE "static" CACHE STRING "Linkage mode for Plumed2 library") +set(PLUMED_MODE_VALUES static shared runtime) +set_property(CACHE PLUMED_MODE PROPERTY STRINGS ${PLUMED_MODE_VALUES}) +validate_option(PLUMED_MODE PLUMED_MODE_VALUES) +string(TOUPPER ${PLUMED_MODE} PLUMED_MODE) - set(PLUMED_LINK_LIBS "") - if(PLUMED_MODE STREQUAL "STATIC") - find_package(LAPACK REQUIRED) - find_package(BLAS REQUIRED) - find_package(GSL REQUIRED) - list(APPEND LAPACK_LIBRARIES ${BLAS_LIBRARIES}) - list(APPEND PLUMED_LINK_LIBS ${LAPACK_LIBRARIES} ${GSL_LIBRARIES}) - find_package(ZLIB QUIET) - if(ZLIB_FOUND) - list(APPEND PLUMED_LINK_LIBS ${ZLIB_LIBRARIES}) - endif() +set(PLUMED_LINK_LIBS) +if(PLUMED_MODE STREQUAL "STATIC") + find_package(LAPACK REQUIRED) + find_package(BLAS REQUIRED) + find_package(GSL REQUIRED) + list(APPEND PLUMED_LINK_LIBS ${LAPACK_LIBRARIES} ${BLAS_LIBRARIES} GSL::gsl) + find_package(ZLIB QUIET) + if(ZLIB_FOUND) + list(APPEND PLUMED_LINK_LIBS ZLIB::ZLIB) endif() - - find_package(PkgConfig QUIET) - set(DOWNLOAD_PLUMED_DEFAULT ON) - if(PKG_CONFIG_FOUND) - pkg_check_modules(PLUMED QUIET plumed) - if(PLUMED_FOUND) - set(DOWNLOAD_PLUMED_DEFAULT OFF) - endif() + find_package(FFTW3 QUIET) + if(FFTW3_FOUND) + list(APPEND PLUMED_LINK_LIBS FFTW3::FFTW3) endif() - - option(DOWNLOAD_PLUMED "Download Plumed package instead of using an already installed one" ${DOWNLOAD_PLUMED_DEFAULT}) - if(DOWNLOAD_PLUMED) - if(BUILD_MPI) - set(PLUMED_CONFIG_MPI "--enable-mpi") - set(PLUMED_CONFIG_CC ${CMAKE_MPI_C_COMPILER}) - set(PLUMED_CONFIG_CXX ${CMAKE_MPI_CXX_COMPILER}) - else() - set(PLUMED_CONFIG_MPI "--disable-mpi") - set(PLUMED_CONFIG_CC ${CMAKE_C_COMPILER}) - set(PLUMED_CONFIG_CXX ${CMAKE_CXX_COMPILER}) - endif() - if(BUILD_OMP) - set(PLUMED_CONFIG_OMP "--enable-openmp") - else() - set(PLUMED_CONFIG_OMP "--disable-openmp") - endif() - message(STATUS "PLUMED download requested - we will build our own") - if(PLUMED_MODE STREQUAL "STATIC") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed.a") - elseif(PLUMED_MODE STREQUAL "SHARED") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX};/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") - elseif(PLUMED_MODE STREQUAL "RUNTIME") - set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumedWrapper.a") - endif() - include(ExternalProject) - ExternalProject_Add(plumed_build - URL https://github.com/plumed/plumed2/releases/download/v2.6.0/plumed-src-2.6.0.tgz - URL_MD5 204d2edae58d9b10ba3ad460cad64191 - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix= - ${CONFIGURE_REQUEST_PIC} - --enable-modules=all - ${PLUMED_CONFIG_MPI} - ${PLUMED_CONFIG_OMP} - CXX=${PLUMED_CONFIG_CXX} - CC=${PLUMED_CONFIG_CC} - BUILD_BYPRODUCTS ${PLUMED_BUILD_BYPRODUCTS} - ) - ExternalProject_get_property(plumed_build INSTALL_DIR) - set(PLUMED_INSTALL_DIR ${INSTALL_DIR}) - list(APPEND LAMMPS_DEPS plumed_build) - if(PLUMED_MODE STREQUAL "STATIC") - add_definitions(-D__PLUMED_WRAPPER_CXX=1) - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumed.a ${PLUMED_LINK_LIBS} ${CMAKE_DL_LIBS}) - elseif(PLUMED_MODE STREQUAL "SHARED") - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX} ${PLUMED_INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_DL_LIBS}) - elseif(PLUMED_MODE STREQUAL "RUNTIME") - add_definitions(-D__PLUMED_HAS_DLOPEN=1 -D__PLUMED_DEFAULT_KERNEL=${PLUMED_INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}) - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_INSTALL_DIR}/lib/libplumedWrapper.a -rdynamic ${CMAKE_DL_LIBS}) - endif() - set(PLUMED_INCLUDE_DIRS "${PLUMED_INSTALL_DIR}/include") - else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(PLUMED REQUIRED plumed) - if(PLUMED_MODE STREQUAL "STATIC") - add_definitions(-D__PLUMED_WRAPPER_CXX=1) - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.static) - elseif(PLUMED_MODE STREQUAL "SHARED") - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.shared) - elseif(PLUMED_MODE STREQUAL "RUNTIME") - add_definitions(-D__PLUMED_HAS_DLOPEN=1 -D__PLUMED_DEFAULT_KERNEL=${PLUMED_LIBDIR}/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}) - include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.runtime) - endif() - list(APPEND LAMMPS_LINK_LIBS ${PLUMED_LOAD}) - endif() - include_directories(${PLUMED_INCLUDE_DIRS}) endif() + +find_package(PkgConfig QUIET) +set(DOWNLOAD_PLUMED_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(PLUMED QUIET plumed) + if(PLUMED_FOUND) + set(DOWNLOAD_PLUMED_DEFAULT OFF) + endif() +endif() + +option(DOWNLOAD_PLUMED "Download Plumed package instead of using an already installed one" ${DOWNLOAD_PLUMED_DEFAULT}) +if(DOWNLOAD_PLUMED) + if(BUILD_MPI) + set(PLUMED_CONFIG_MPI "--enable-mpi") + set(PLUMED_CONFIG_CC ${CMAKE_MPI_C_COMPILER}) + set(PLUMED_CONFIG_CXX ${CMAKE_MPI_CXX_COMPILER}) + else() + set(PLUMED_CONFIG_MPI "--disable-mpi") + set(PLUMED_CONFIG_CC ${CMAKE_C_COMPILER}) + set(PLUMED_CONFIG_CXX ${CMAKE_CXX_COMPILER}) + endif() + if(BUILD_OMP) + set(PLUMED_CONFIG_OMP "--enable-openmp") + else() + set(PLUMED_CONFIG_OMP "--disable-openmp") + endif() + message(STATUS "PLUMED download requested - we will build our own") + if(PLUMED_MODE STREQUAL "STATIC") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed.a") + elseif(PLUMED_MODE STREQUAL "SHARED") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX};/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set(PLUMED_BUILD_BYPRODUCTS "/lib/libplumedWrapper.a") + endif() + include(ExternalProject) + ExternalProject_Add(plumed_build + URL https://github.com/plumed/plumed2/releases/download/v2.6.0/plumed-src-2.6.0.tgz + URL_MD5 204d2edae58d9b10ba3ad460cad64191 + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND /configure --prefix= + ${CONFIGURE_REQUEST_PIC} + --enable-modules=all + ${PLUMED_CONFIG_MPI} + ${PLUMED_CONFIG_OMP} + CXX=${PLUMED_CONFIG_CXX} + CC=${PLUMED_CONFIG_CC} + BUILD_BYPRODUCTS ${PLUMED_BUILD_BYPRODUCTS} + ) + ExternalProject_get_property(plumed_build INSTALL_DIR) + add_library(LAMMPS::PLUMED UNKNOWN IMPORTED) + add_dependencies(LAMMPS::PLUMED plumed_build) + if(PLUMED_MODE STREQUAL "STATIC") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumed.a INTERFACE_LINK_LIBRARIES "${PLUMED_LINK_LIBS};${CMAKE_DL_LIBS}") + elseif(PLUMED_MODE STREQUAL "SHARED") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumed${CMAKE_SHARED_LIBRARY_SUFFIX} INTERFACE_LINK_LIBRARIES "${INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX};${CMAKE_DL_LIBS}") + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_DEFAULT_KERNEL=${INSTALL_DIR}/lib/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + set_target_properties(LAMMPS::PLUMED PROPERTIES IMPORTED_LOCATION ${INSTALL_DIR}/lib/libplumedWrapper.a INTERFACE_LINK_LIBRARIES "${CMAKE_DL_LIBS}") + endif() + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(PLUMED REQUIRED plumed) + add_library(LAMMPS::PLUMED INTERFACE IMPORTED) + if(PLUMED_MODE STREQUAL "STATIC") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.static) + elseif(PLUMED_MODE STREQUAL "SHARED") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.shared) + elseif(PLUMED_MODE STREQUAL "RUNTIME") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_COMPILE_DEFINITIONS "__PLUMED_DEFAULT_KERNEL=${PLUMED_LIBDIR}/libplumedKernel${CMAKE_SHARED_LIBRARY_SUFFIX}") + include(${PLUMED_LIBDIR}/plumed/src/lib/Plumed.cmake.runtime) + endif() + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_LINK_LIBRARIES "${PLUMED_LOAD}") + set_target_properties(LAMMPS::PLUMED PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${PLUMED_INCLUDE_DIRS}") +endif() +target_link_libraries(lammps PRIVATE LAMMPS::PLUMED) diff --git a/cmake/Modules/Packages/USER-QMMM.cmake b/cmake/Modules/Packages/USER-QMMM.cmake index 544455868e..c613067349 100644 --- a/cmake/Modules/Packages/USER-QMMM.cmake +++ b/cmake/Modules/Packages/USER-QMMM.cmake @@ -1,13 +1,6 @@ -if(PKG_USER-QMMM) - enable_language(C) +enable_language(C) - if(NOT BUILD_LIB) - message(FATAL_ERROR "Building a QM/MM executable with USER-QMMM requires BUILD_LIB=yes") - endif() - if(NOT BUILD_SHARED_LIBS) - message(WARNING "It is recommended to use BUILD_SHARED_LIBS=yes with USER-QMMM") - endif() - add_library(qmmm STATIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm/libqmmm.c) - list(APPEND LAMMPS_LINK_LIBS qmmm) - target_include_directories(qmmm PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm) -endif() +add_library(qmmm STATIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm/libqmmm.c) +set_target_properties(qmmm PROPERTIES OUTPUT_NAME lammps_qmmm${LAMMPS_MACHINE}) +target_link_libraries(lammps PRIVATE qmmm) +target_include_directories(qmmm PUBLIC ${LAMMPS_LIB_SOURCE_DIR}/qmmm) diff --git a/cmake/Modules/Packages/USER-QUIP.cmake b/cmake/Modules/Packages/USER-QUIP.cmake index 93096a2f54..790f27def7 100644 --- a/cmake/Modules/Packages/USER-QUIP.cmake +++ b/cmake/Modules/Packages/USER-QUIP.cmake @@ -1,5 +1,3 @@ -if(PKG_USER-QUIP) - enable_language(Fortran) - find_package(QUIP REQUIRED) - list(APPEND LAMMPS_LINK_LIBS ${QUIP_LIBRARIES} ${LAPACK_LIBRARIES}) -endif() +enable_language(Fortran) +find_package(QUIP REQUIRED) +target_link_libraries(lammps PRIVATE QUIP::QUIP ${LAPACK_LIBRARIES}) diff --git a/cmake/Modules/Packages/USER-SCAFACOS.cmake b/cmake/Modules/Packages/USER-SCAFACOS.cmake index 8bb9e63605..d92a83f7c1 100644 --- a/cmake/Modules/Packages/USER-SCAFACOS.cmake +++ b/cmake/Modules/Packages/USER-SCAFACOS.cmake @@ -1,76 +1,59 @@ -if(PKG_USER-SCAFACOS) - enable_language(Fortran) - enable_language(C) +enable_language(Fortran) +enable_language(C) - find_package(GSL REQUIRED) - find_package(PkgConfig QUIET) - find_package(MPI REQUIRED) - set(DOWNLOAD_SCAFACOS_DEFAULT ON) - if(PKG_CONFIG_FOUND) - pkg_check_modules(SCAFACOS QUIET scafacos) - if(SCAFACOS_FOUND) - set(DOWNLOAD_SCAFACOS_DEFAULT OFF) - endif() +find_package(GSL REQUIRED) +find_package(PkgConfig QUIET) +find_package(MPI REQUIRED) +set(DOWNLOAD_SCAFACOS_DEFAULT ON) +if(PKG_CONFIG_FOUND) + pkg_check_modules(SCAFACOS QUIET scafacos) + if(SCAFACOS_FOUND) + set(DOWNLOAD_SCAFACOS_DEFAULT OFF) endif() - option(DOWNLOAD_SCAFACOS "Download ScaFaCoS library instead of using an already installed one" ${DOWNLOAD_SCAFACOS_DEFAULT}) - if(DOWNLOAD_SCAFACOS) - message(STATUS "ScaFaCoS download requested - we will build our own") - include(ExternalProject) - ExternalProject_Add(scafacos_build - URL https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz - URL_MD5 bd46d74e3296bd8a444d731bb10c1738 - CONFIGURE_COMMAND /configure --prefix= --disable-doc - --enable-fcs-solvers=fmm,p2nfft,direct,ewald,p3m - --with-internal-fftw --with-internal-pfft - --with-internal-pnfft ${CONFIGURE_REQUEST_PIC} - FC=${CMAKE_MPI_Fortran_COMPILER} - CXX=${CMAKE_MPI_CXX_COMPILER} - CC=${CMAKE_MPI_C_COMPILER} - F77= - BUILD_BYPRODUCTS - /lib/libfcs.a - /lib/libfcs_direct.a - /lib/libfcs_ewald.a - /lib/libfcs_fmm.a - /lib/libfcs_p2nfft.a - /lib/libfcs_p3m.a - /lib/libfcs_near.a - /lib/libfcs_gridsort.a - /lib/libfcs_resort.a - /lib/libfcs_redist.a - /lib/libfcs_common.a - /lib/libfcs_pnfft.a - /lib/libfcs_pfft.a - /lib/libfcs_fftw3_mpi.a - /lib/libfcs_fftw3.a - ) - ExternalProject_get_property(scafacos_build INSTALL_DIR) - set(SCAFACOS_BUILD_DIR ${INSTALL_DIR}) - set(SCAFACOS_INCLUDE_DIRS ${SCAFACOS_BUILD_DIR}/include) - list(APPEND LAMMPS_DEPS scafacos_build) - # list and order from pkg_config file of ScaFaCoS build - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_direct.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_ewald.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fmm.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_p2nfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_p3m.a) - list(APPEND LAMMPS_LINK_LIBS ${GSL_LIBRARIES}) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_near.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_gridsort.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_resort.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_redist.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_common.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_pnfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_pfft.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fftw3_mpi.a) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_BUILD_DIR}/lib/libfcs_fftw3.a) - list(APPEND LAMMPS_LINK_LIBS ${MPI_Fortran_LIBRARIES}) - list(APPEND LAMMPS_LINK_LIBS ${MPI_C_LIBRARIES}) - else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(SCAFACOS REQUIRED scafacos) - list(APPEND LAMMPS_LINK_LIBS ${SCAFACOS_LDFLAGS}) - endif() - include_directories(${SCAFACOS_INCLUDE_DIRS}) +endif() +option(DOWNLOAD_SCAFACOS "Download ScaFaCoS library instead of using an already installed one" ${DOWNLOAD_SCAFACOS_DEFAULT}) +if(DOWNLOAD_SCAFACOS) + message(STATUS "ScaFaCoS download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(scafacos_build + URL https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz + URL_MD5 bd46d74e3296bd8a444d731bb10c1738 + CONFIGURE_COMMAND /configure --prefix= --disable-doc + --enable-fcs-solvers=fmm,p2nfft,direct,ewald,p3m + --with-internal-fftw --with-internal-pfft + --with-internal-pnfft ${CONFIGURE_REQUEST_PIC} + FC=${CMAKE_MPI_Fortran_COMPILER} + CXX=${CMAKE_MPI_CXX_COMPILER} + CC=${CMAKE_MPI_C_COMPILER} + F77= + BUILD_BYPRODUCTS + /lib/libfcs.a + /lib/libfcs_direct.a + /lib/libfcs_ewald.a + /lib/libfcs_fmm.a + /lib/libfcs_p2nfft.a + /lib/libfcs_p3m.a + /lib/libfcs_near.a + /lib/libfcs_gridsort.a + /lib/libfcs_resort.a + /lib/libfcs_redist.a + /lib/libfcs_common.a + /lib/libfcs_pnfft.a + /lib/libfcs_pfft.a + /lib/libfcs_fftw3_mpi.a + /lib/libfcs_fftw3.a + ) + ExternalProject_get_property(scafacos_build INSTALL_DIR) + file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + add_library(LAMMPS::SCAFACOS UNKNOWN IMPORTED) + set_target_properties(LAMMPS::SCAFACOS PROPERTIES + IMPORTED_LOCATION "${INSTALL_DIR}/lib/libfcs.a" + INTERFACE_INCLUDE_DIRECTORIES "${INSTALL_DIR}/include" + INTERFACE_LINK_LIBRARIES "${INSTALL_DIR}/lib/libfcs.a;${INSTALL_DIR}/lib/libfcs_direct.a;${INSTALL_DIR}/lib/libfcs_ewald.a;${INSTALL_DIR}/lib/libfcs_fmm.a;${INSTALL_DIR}/lib/libfcs_p2nfft.a;${INSTALL_DIR}/lib/libfcs_p3m.a;GSL::gsl;${INSTALL_DIR}/lib/libfcs_near.a;${INSTALL_DIR}/lib/libfcs_gridsort.a;${INSTALL_DIR}/lib/libfcs_resort.a;${INSTALL_DIR}/lib/libfcs_redist.a;${INSTALL_DIR}/lib/libfcs_common.a;${INSTALL_DIR}/lib/libfcs_pnfft.a;${INSTALL_DIR}/lib/libfcs_pfft.a;${INSTALL_DIR}/lib/libfcs_fftw3_mpi.a;${INSTALL_DIR}/lib/libfcs_fftw3.a;MPI::MPI_Fortran;MPI::MPI_C") + target_link_libraries(lammps PRIVATE LAMMPS::SCAFACOS) + add_dependencies(LAMMPS::SCAFACOS scafacos_build) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(SCAFACOS REQUIRED IMPORTED_TARGET scafacos) + target_link_libraries(lammps PRIVATE PkgConfig::SCAFACOS) endif() diff --git a/cmake/Modules/Packages/USER-SDPD.cmake b/cmake/Modules/Packages/USER-SDPD.cmake index 530dcf2bd9..c9f419d918 100644 --- a/cmake/Modules/Packages/USER-SDPD.cmake +++ b/cmake/Modules/Packages/USER-SDPD.cmake @@ -1,13 +1,13 @@ # Fix rigid/meso requires RIGID to be installed -if(PKG_USER-SDPD) - set(USER-SDPD_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-SDPD) +set(USER-SDPD_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/USER-SDPD) - get_property(hlist GLOBAL PROPERTY FIX) - if(NOT PKG_RIGID) - list(REMOVE_ITEM hlist ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.h) - list(REMOVE_ITEM LIB_SOURCES ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.cpp) - endif() - set_property(GLOBAL PROPERTY FIX "${hlist}") - - include_directories(${USER-SDPD_SOURCES_DIR}) +get_property(hlist GLOBAL PROPERTY FIX) +if(NOT PKG_RIGID) + list(REMOVE_ITEM hlist ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.h) + get_target_property(LAMMPS_SOURCES lammps SOURCES) + list(REMOVE_ITEM LAMMPS_SOURCES ${USER-SDPD_SOURCES_DIR}/fix_rigid_meso.cpp) + set_property(TARGET lammps PROPERTY SOURCES ${LAMMPS_SOURCES}) endif() +set_property(GLOBAL PROPERTY FIX "${hlist}") + +target_include_directories(lammps PRIVATE ${USER-SDPD_SOURCES_DIR}) diff --git a/cmake/Modules/Packages/USER-SMD.cmake b/cmake/Modules/Packages/USER-SMD.cmake index a868918e37..6984b24d52 100644 --- a/cmake/Modules/Packages/USER-SMD.cmake +++ b/cmake/Modules/Packages/USER-SMD.cmake @@ -1,28 +1,28 @@ -if(PKG_USER-SMD) - find_package(Eigen3 NO_MODULE) - if(EIGEN3_FOUND) - set(DOWNLOAD_EIGEN3_DEFAULT OFF) - else() - set(DOWNLOAD_EIGEN3_DEFAULT ON) - endif() - option(DOWNLOAD_EIGEN3 "Download Eigen3 instead of using an already installed one)" ${DOWNLOAD_EIGEN3_DEFAULT}) - if(DOWNLOAD_EIGEN3) - message(STATUS "Eigen3 download requested - we will build our own") - include(ExternalProject) - ExternalProject_Add(Eigen3_build - URL http://bitbucket.org/eigen/eigen/get/3.3.7.tar.gz - URL_MD5 f2a417d083fe8ca4b8ed2bc613d20f07 - CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - ) - ExternalProject_get_property(Eigen3_build SOURCE_DIR) - set(EIGEN3_INCLUDE_DIR ${SOURCE_DIR}) - list(APPEND LAMMPS_DEPS Eigen3_build) - else() - find_package(Eigen3 NO_MODULE) - mark_as_advanced(Eigen3_DIR) - if(NOT EIGEN3_FOUND) - message(FATAL_ERROR "Eigen3 not found, help CMake to find it by setting EIGEN3_INCLUDE_DIR, or set DOWNLOAD_EIGEN3=ON to download it") - endif() - endif() - include_directories(${EIGEN3_INCLUDE_DIR}) +find_package(Eigen3 NO_MODULE) +if(EIGEN3_FOUND) + set(DOWNLOAD_EIGEN3_DEFAULT OFF) +else() + set(DOWNLOAD_EIGEN3_DEFAULT ON) +endif() +option(DOWNLOAD_EIGEN3 "Download Eigen3 instead of using an already installed one)" ${DOWNLOAD_EIGEN3_DEFAULT}) +if(DOWNLOAD_EIGEN3) + message(STATUS "Eigen3 download requested - we will build our own") + include(ExternalProject) + ExternalProject_Add(Eigen3_build + URL http://bitbucket.org/eigen/eigen/get/3.3.7.tar.gz + URL_MD5 f2a417d083fe8ca4b8ed2bc613d20f07 + CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" + ) + ExternalProject_get_property(Eigen3_build SOURCE_DIR) + add_library(LAMMPS::EIGEN3 INTERFACE IMPORTED) + set_target_properties(LAMMPS::EIGEN3 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}") + target_link_libraries(lammps PRIVATE LAMMPS::EIGEN3) + add_dependencies(LAMMPS::EIGEN3 Eigen3_build) +else() + find_package(Eigen3 NO_MODULE) + mark_as_advanced(Eigen3_DIR) + if(NOT EIGEN3_FOUND) + message(FATAL_ERROR "Eigen3 not found, help CMake to find it by setting EIGEN3_INCLUDE_DIR, or set DOWNLOAD_EIGEN3=ON to download it") + endif() + target_link_libraries(lammps PRIVATE Eigen3::Eigen) endif() diff --git a/cmake/Modules/Packages/USER-VTK.cmake b/cmake/Modules/Packages/USER-VTK.cmake index d264577ca2..8c4445167a 100644 --- a/cmake/Modules/Packages/USER-VTK.cmake +++ b/cmake/Modules/Packages/USER-VTK.cmake @@ -1,6 +1,4 @@ -if(PKG_USER-VTK) - find_package(VTK REQUIRED NO_MODULE) - include(${VTK_USE_FILE}) - add_definitions(-DLAMMPS_VTK) - list(APPEND LAMMPS_LINK_LIBS ${VTK_LIBRARIES}) -endif() +find_package(VTK REQUIRED NO_MODULE) +include(${VTK_USE_FILE}) +target_compile_definitions(lammps PRIVATE -DLAMMPS_VTK) +target_link_libraries(lammps PRIVATE ${VTK_LIBRARIES}) diff --git a/cmake/Modules/Packages/VORONOI.cmake b/cmake/Modules/Packages/VORONOI.cmake index 5418132034..1d6893a978 100644 --- a/cmake/Modules/Packages/VORONOI.cmake +++ b/cmake/Modules/Packages/VORONOI.cmake @@ -1,43 +1,48 @@ -if(PKG_VORONOI) - find_package(VORO) - if(VORO_FOUND) - set(DOWNLOAD_VORO_DEFAULT OFF) - else() - set(DOWNLOAD_VORO_DEFAULT ON) - endif() - option(DOWNLOAD_VORO "Download and compile the Voro++ library instead of using an already installed one" ${DOWNLOAD_VORO_DEFAULT}) - if(DOWNLOAD_VORO) - message(STATUS "Voro++ download requested - we will build our own") - include(ExternalProject) - - if(BUILD_SHARED_LIBS) - set(VORO_BUILD_CFLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") - else() - set(VORO_BUILD_CFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") - endif() - if(APPLE) - get_filename_component(VORO_CXX ${CMAKE_CXX_COMPILER} NAME_WE) - set(VORO_BUILD_OPTIONS CXX=${VORO_CXX} CFLAGS=${VORO_BUILD_CFLAGS}) - else() - set(VORO_BUILD_OPTIONS CXX=${CMAKE_CXX_COMPILER} CFLAGS=${VORO_BUILD_CFLAGS}) - endif() - - ExternalProject_Add(voro_build - URL https://download.lammps.org/thirdparty/voro++-0.4.6.tar.gz - URL_MD5 2338b824c3b7b25590e18e8df5d68af9 - CONFIGURE_COMMAND "" BUILD_COMMAND make ${VORO_BUILD_OPTIONS} BUILD_IN_SOURCE 1 INSTALL_COMMAND "" - BUILD_BYPRODUCTS /src/libvoro++.a - ) - ExternalProject_get_property(voro_build SOURCE_DIR) - set(VORO_LIBRARIES ${SOURCE_DIR}/src/libvoro++.a) - set(VORO_INCLUDE_DIRS ${SOURCE_DIR}/src) - list(APPEND LAMMPS_DEPS voro_build) - else() - find_package(VORO) - if(NOT VORO_FOUND) - message(FATAL_ERROR "Voro++ library not found. Help CMake to find it by setting VORO_LIBRARY and VORO_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") - endif() - endif() - include_directories(${VORO_INCLUDE_DIRS}) - list(APPEND LAMMPS_LINK_LIBS ${VORO_LIBRARIES}) +find_package(VORO) +if(VORO_FOUND) + set(DOWNLOAD_VORO_DEFAULT OFF) +else() + set(DOWNLOAD_VORO_DEFAULT ON) +endif() +option(DOWNLOAD_VORO "Download and compile the Voro++ library instead of using an already installed one" ${DOWNLOAD_VORO_DEFAULT}) +if(DOWNLOAD_VORO) + message(STATUS "Voro++ download requested - we will build our own") + include(ExternalProject) + + if(BUILD_SHARED_LIBS) + set(VORO_BUILD_CFLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") + else() + set(VORO_BUILD_CFLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BTYPE}}") + endif() + if(APPLE) + get_filename_component(VORO_CXX ${CMAKE_CXX_COMPILER} NAME_WE) + set(VORO_BUILD_OPTIONS CXX=${VORO_CXX} CFLAGS=${VORO_BUILD_CFLAGS}) + else() + set(VORO_BUILD_OPTIONS CXX=${CMAKE_CXX_COMPILER} CFLAGS=${VORO_BUILD_CFLAGS}) + endif() + + ExternalProject_Add(voro_build + URL https://download.lammps.org/thirdparty/voro++-0.4.6.tar.gz + URL_MD5 2338b824c3b7b25590e18e8df5d68af9 + PATCH_COMMAND patch -b -p0 < ${LAMMPS_LIB_SOURCE_DIR}/voronoi/voro-make.patch + CONFIGURE_COMMAND "" + BUILD_COMMAND make ${VORO_BUILD_OPTIONS} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "" + BUILD_BYPRODUCTS /src/libvoro++.a + ) + ExternalProject_get_property(voro_build SOURCE_DIR) + file(MAKE_DIRECTORY ${SOURCE_DIR}/src) + add_library(LAMMPS::VORO UNKNOWN IMPORTED) + set_target_properties(LAMMPS::VORO PROPERTIES + IMPORTED_LOCATION "${SOURCE_DIR}/src/libvoro++.a" + INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}/src") + target_link_libraries(lammps PRIVATE LAMMPS::VORO) + add_dependencies(LAMMPS::VORO voro_build) +else() + find_package(VORO) + if(NOT VORO_FOUND) + message(FATAL_ERROR "Voro++ library not found. Help CMake to find it by setting VORO_LIBRARY and VORO_INCLUDE_DIR, or set DOWNLOAD_VORO=ON to download it") + endif() + target_link_libraries(lammps PRIVATE VORO::VORO) endif() diff --git a/cmake/Modules/Testing.cmake b/cmake/Modules/Testing.cmake index 0eeef00fe9..f0a5f4519d 100644 --- a/cmake/Modules/Testing.cmake +++ b/cmake/Modules/Testing.cmake @@ -2,7 +2,7 @@ # Testing ############################################################################### option(ENABLE_TESTING "Enable testing" OFF) -if(ENABLE_TESTING AND BUILD_EXE) +if(ENABLE_TESTING) enable_testing() option(LAMMPS_TESTING_SOURCE_DIR "Location of lammps-testing source directory" "") option(LAMMPS_TESTING_GIT_TAG "Git tag of lammps-testing" "master") @@ -28,7 +28,7 @@ if(ENABLE_TESTING AND BUILD_EXE) "https://github.com/lammps/lammps-testing in LAMMPS_TESTING_SOURCE_DIR") endif() - add_test(ShowHelp ${CMAKE_BINARY_DIR}/${LAMMPS_BINARY} -help) + add_test(NAME ShowHelp COMMAND $ -help) if(EXISTS ${LAMMPS_TESTING_SOURCE_DIR}) message(STATUS "Running test discovery...") @@ -42,8 +42,7 @@ if(ENABLE_TESTING AND BUILD_EXE) string(REPLACE "-" "_" TEST_NAME ${TEST_NAME}) string(REPLACE "+" "_" TEST_NAME ${TEST_NAME}) set(TEST_NAME "test_core_${TEST_NAME}_serial") - add_test(${TEST_NAME} ${CMAKE_BINARY_DIR}/${LAMMPS_BINARY} -in ${SCRIPT_NAME}) - set_tests_properties(${TEST_NAME} PROPERTIES WORKING_DIRECTORY ${PARENT_DIR}) + add_test(NAME ${TEST_NAME} COMMAND $ -in ${SCRIPT_NAME} WORKING_DIRECTORY ${PARENT_DIR}) endforeach() list(LENGTH TEST_SCRIPTS NUM_TESTS) diff --git a/cmake/Modules/generate_lmpgitversion.cmake b/cmake/Modules/generate_lmpgitversion.cmake index a4aa59b262..afd4447867 100644 --- a/cmake/Modules/generate_lmpgitversion.cmake +++ b/cmake/Modules/generate_lmpgitversion.cmake @@ -3,17 +3,19 @@ set(temp_git_commit "(unknown)") set(temp_git_branch "(unknown)") set(temp_git_describe "(unknown)") set(temp_git_info "false") -if(GIT_FOUND AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../.git) + +message(STATUS "Git Directory: ${LAMMPS_DIR}/.git") +if(GIT_FOUND AND EXISTS ${LAMMPS_DIR}/.git) set(temp_git_info "true") - execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR}/.. rev-parse HEAD + execute_process(COMMAND ${GIT_EXECUTABLE} -C ${LAMMPS_DIR} rev-parse HEAD OUTPUT_VARIABLE temp_git_commit ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR}/.. rev-parse --abbrev-ref HEAD + execute_process(COMMAND ${GIT_EXECUTABLE} -C ${LAMMPS_DIR} rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE temp_git_branch ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR}/.. describe --dirty=-modified + execute_process(COMMAND ${GIT_EXECUTABLE} -C ${LAMMPS_DIR} describe --dirty=-modified OUTPUT_VARIABLE temp_git_describe ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/cmake/README.md b/cmake/README.md deleted file mode 100644 index b9dd6d4373..0000000000 --- a/cmake/README.md +++ /dev/null @@ -1,2320 +0,0 @@ -# Building LAMMPS using CMake - -LAMMPS recently acquired support for building with CMake thanks to the efforts -of Christoph Junghans (LANL) and Richard Berger (Temple U). One of the key -strengths of CMake is that it can generate the necessary build system files of -your own personal preference. It also enables using common development IDEs such -as Eclipse, Visual Studio, QtCreator, Xcode and many more for LAMMPS -development. - -CMake can both be used as a command-line (CLI) utility `cmake` or through one of -its GUIs. `ccmake` is a text-based ui to configure and build CMake projects. -`cmake-gui` is a graphical frontend for running CMake operations that works on -Linux desktop environments, Windows and MacOS X. - -The following is a tutorial-style introduction in using the CMake system. It -should give you the necessary foundation to understand how to do the most common -tasks, act as a reference and provide examples of typical use cases. - -## Table of Contents - - * [Quick Start for the Impatient](#quick-start-for-the-impatient) - * [Building LAMMPS using cmake](#building-lammps-using-cmake-1) - * [Prerequisites](#prerequisites) - * [Build directory vs. Source Directory](#build-directory-vs-source-directory) - * [Defining and using presets](#defining-and-using-presets) - * [Reference](#reference) - * [Common CMake Configuration Options](#common-cmake-configuration-options) - * [LAMMPS Configuration Options](#lammps-configuration-options) - * [Parallelization and Accelerator Packages](#parallelization-and-accelerator-packages) - * [Default Packages](#default-packages) - * [Other Packages](#other-packages) - * [User Packages](#user-packages) - * [Package-Specific Configuration Options](#package-specific-configuration-options) - * [KSPACE Package](#kspace-package) - * [MKL](#mkl) - * [FFTW3](#fftw3) - * [BLAS](#blas) - * [LAPACK](#lapack) - * [PYTHON Package](#python-package) - * [GPU Package](#gpu-package) - * [MESSAGE Package](#message-package) - * [MSCG Package](#mscg-package) - * [VORONOI Package](#voronoi-package) - * [USER-LATTE Package](#user-latte-package) - * [USER-PLUMED Package](#user-plumed-package) - * [USER-SCAFACOS Package](#user-scafacos-package) - * [USER-SMD Package](#user-smd-package) - * [Optional Features](#optional-features) - * [zlib support](#zlib-support) - * [JPEG support](#jpeg-support) - * [PNG support](#png-support) - * [GZIP support](#gzip-support) - * [FFMPEG support](#ffmpeg-support) - * [Compilers](#compilers) - * [Building with GNU Compilers](#building-with-gnu-compilers) - * [Building with Intel Compilers](#building-with-intel-compilers) - * [Building with LLVM/Clang Compilers](#building-with-llvmclang-compilers) - -## Quick Start for the Impatient -If you want to skip ahead and just run the compilation using `cmake`, please -find a minimal example below. Together with the options reference below, this -should get you started. - -```bash -git clone https://github.com/lammps/lammps.git -mkdir lammps/build -cd lammps/build -cmake [-D OPTION_A=VALUE_A -D OPTION_B=VALUE_B ...] ../cmake -make -``` - -# Building LAMMPS using `cmake` - -## Prerequisites -This tutorial assumes you are running in a command-line environment using a -shell like Bash. - -* Linux: any terminal window will work -* MacOS X: launch the Terminal app -* Windows 10: install and run "Bash on Windows" (aka Ubuntu on Windows) - -Before we start, please download the latest and greatest version of LAMMPS from -GitHub. You can either download it as a tarball or ZIP file, or via git. When -you start with a fresh lammps directory, the contents should look like this: - -```bash -$ ls -bench doc lib potentials README tools -cmake examples LICENSE python src -``` - -## Build directory vs. Source Directory - -By using CMake we separate building LAMMPS into multiple phases: - -1. **Configuration**: define which features we want to enable/disable and how it should be compiled -2. **Compilation**: compile each source file and generate binaries and libraries based on the configuration -3. **Installation** (Optional): finally we can install the generated binaries on our system - -In the GNU Make based build system of LAMMPS, configuration occurs by running -special make targets like `make yes-MOLECULAR`. These targets modify the -**source directory** (`src/`) directory by copying package files and patching -Makefile. In some cases you are force to manually edit Makefiles to add compiler -options and/or correct include directory and library paths. - -These edits and copy operations are no longer necessary when compiling with -CMake. The source directory stays untouched, so you compile LAMMPS in many -different variants using the same source code checkout. It enables true -**out-of-source** builds. - -When using Cmake, you can compile in **any** folder outside of the source -directory. Any working directory you choose becomes a so-called **build -directory**. All configuration files and compilation results are stored in this -folder. We recommend calling it something like `build/`. - -Let's have a look a quick example, where we get the greatest and latest version -of LAMMPS from GitHub via git: -```bash -git clone https://github.com/lammps/lammps.git -``` - -We then create a new `build` folder and make it our current working directory: -``` -mkdir lammps/build -cd lammps/build -``` - -To configure LAMMPS we run `cmake` inside of this folder. However it requires at -least one argument. `cmake` needs to read the LAMMPS `CMakeLists.txt` file to -know what to do. This file is located in the `cmake/` subdirectory of the -lammps checkout. To run `cmake` add the relative or absolute path to the `cmake/` -directory as first argument. - -E.g., if the current working directory is `lammps/build` you can specify the -relative path to `lammps/cmake` as follows: -``` -cmake ../cmake -``` - -You could also specify the absolute path: -``` -cmake /path/to/my/lammps/folder/cmake -``` - -Please note: **This does NOT compile the code!** Running cmake only configures -the next build. It generates the necessary files to compile the code. On -Unix/Linux it defaults to generating Makefiles. You can also choose other output -formats to generate files for Eclipse, Xcode or Visual Studio which are -supported on other platorms. - -To compile LAMMPS once the Makefiles are generated, simply type `make` in the -build directory. - -``` -make -``` -# Defining and using presets - -The CMake build exposes a lot of different options. In the old build system -some of the package selections were possible by using special make target like -`make yes-std` or `make no-lib`. Achieving a similar result with cmake requires -specifying all options manually. This can quickly become a very long command -line that is hard to handle. While these could be stored in a simple script -file, there is another way of defining "presets" to compile LAMMPS in a certain -way. Since the cmake build process - contrary to the conventional build system - -includes the compilation of the bundled libraries into the standard build process, -the grouping of those presets is somewhat different. - -A preset is a regular CMake script file that can use constructs such as -variables, lists and for-loops to manipulate configuration options and create -an [*initial cache*](https://cmake.org/cmake/help/v3.12/manual/cmake.1.html). -Options must be set with the `CACHE` and `FORCE` flag to ensure they are -considered even during a second cmake run. - -Such a file can then be passed to cmake via the `-C` flag. Several examples of -presets can be found in the `cmake/presets` folder. - -```bash -# build LAMMPS with all packages enabled which don't use external libraries and enable GPU package -mkdir build -cd build -cmake -C ../cmake/presets/all_on.cmake -C ../cmake/presets/nolib.cmake -D PKG_GPU=on ../cmake -``` - -# Reference - -## Common CMake Configuration Options - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
CMAKE_INSTALL_PREFIXInstall location where LAMMPS files will be copied to. In the Unix/Linux case with Makefiles this controls what `make install` will do. - Default setting is $HOME/.local. -
CMAKE_BUILD_TYPEControls if debugging symbols are added to the generated binaries -
-
RelWithDebInfo (default)
-
Release
-
Debug
-
MinSizeRel
-
-
CMAKE_VERBOSE_MAKEFILEEnable verbose output from Makefile builds (useful for debugging), the same can be achieved by adding `VERBOSE=1` to the `make` call. -
-
off (default)
-
on
-
-
- - -## LAMMPS Configuration Options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
LAMMPS_SIZE_LIMITControls the integer sizes used by LAMMPS internally -
-
LAMMPS_SMALLBIG (default)
-
32bit , 64bit
-
LAMMPS_SMALLSMALL
-
32bit , 32bit
-
LAMMPS_BIGBIG
-
64bit , 64bit
-
-
LAMMPS_LONGLONG_TO_LONGWorkaround if your system or MPI version does not recognize long long data types -
-
off (default)
-
on
-
-
LAMMPS_MEMALIGNcontrols the alignment of blocks of memory allocated by LAMMPS -
-
64 (default)
-
-
LAMMPS_EXCEPTIONScontrols whether LAMMPS dies after an error or throws a C++ exception. This is particularly useful when running through the C library interface, since an error - in LAMMPS then doesn't kill the parent process -
-
off (default)
-
on
-
-
LAMMPS_MACHINEallows appending a machine suffix to the generated LAMMPS binary -
-
*none* (default)
-
-
LAMMPS_LIB_SUFFIXallows appending a suffix to the generated LAMMPS library -
-
*none* (default)
-
-
BUILD_LIBcontrol whether to build LAMMPS as a library -
-
off (default)
-
on
-
-
BUILD_EXEcontrol whether to build LAMMPS executable -
-
on (default)
-
off
-
-
BUILD_SHARED_LIBScontrol whether to build LAMMPS as a shared-library -
-
off (default)
-
on
-
-
BUILD_DOCcontrol whether to build LAMMPS documentation -
-
off (default)
-
on
-
-
BUILD_TOOLScontrol whether to build LAMMPS tools -
-
off (default)
-
on
-
-
- -## Parallelization and Accelerator Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
BUILD_MPIcontrol whether to build LAMMPS with MPI support. This will look for - `mpicxx` in your path and use this MPI implementation. -
-
on (default, if found)
-
off
-
-
BUILD_OMPcontrol whether to build LAMMPS with OpenMP support. -
-
on (default, if found)
-
off
-
-
PKG_OPT - A handful of pair styles which are optimized for improved CPU performance on - single or multiple cores. These include EAM, LJ, CHARMM, and Morse potentials. - -
-
off (default)
-
on
-
-
PKG_USER-OMP - Hundreds of pair, fix, compute, bond, angle, dihedral, improper, and kspace - styles which are altered to enable threading on many-core CPUs via OpenMP - directives. - -
-
off (default)
-
on
-
-
PKG_USER-INTEL - Dozens of pair, fix, bond, angle, dihedral, improper, and kspace styles which - are optimized for Intel CPUs and KNLs (Knights Landing). - -
-
off (default)
-
on
-
-
PKG_GPU - Dozens of pair styles and a version of the PPPM long-range Coulombic solver - optimized for GPUs. All such styles have a “gpu” as a suffix in their style - name. The GPU code can be compiled with either CUDA or OpenCL, however the - OpenCL variants are no longer actively maintained and only the CUDA versions - are regularly tested. - -
-
off (default)
-
on
-
-
PKG_KOKKOSDozens of atom, pair, bond, angle, dihedral, improper, fix, compute styles adapted to compile using the Kokkos library which can convert them to OpenMP or CUDA code so that they run efficiently on multicore CPUs, KNLs, or GPUs. -
-
off (default)
-
on
-
-
- -## Default Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_ASPHEREComputes, time-integration fixes, and pair styles for aspherical particle models including ellipsoids, 2d lines, and 3d triangles. -
-
off (default)
-
on
-
-
PKG_BODYBody-style particles with internal structure. Computes, time-integration fixes, pair styles, as well as the body styles themselves. -
-
off (default)
-
on
-
-
PKG_CLASS2Bond, angle, dihedral, improper, and pair styles for the COMPASS CLASS2 molecular force field. -
-
off (default)
-
on
-
-
PKG_COLLOIDCoarse-grained finite-size colloidal particles. Pair styles and fix wall styles for colloidal interactions. Includes the Fast Lubrication Dynamics (FLD) method for hydrodynamic interactions, which is a simplified approximation to Stokesian dynamics. -
-
off (default)
-
on
-
-
PKG_COMPRESSCompressed output of dump files via the zlib compression library, using dump styles with a “gz” in their style name. -
-
off (default)
-
on
-
-
PKG_CORESHELLCompute and pair styles that implement the adiabatic core/shell model for polarizability. The pair styles augment Born, Buckingham, and Lennard-Jones styles with core/shell capabilities. The compute temp/cs command calculates the temperature of a system with core/shell particles. -
-
off (default)
-
on
-
-
PKG_DIPOLEAn atom style and several pair styles for point dipole models with short-range or long-range interactions. -
-
off (default)
-
on
-
-
PKG_GRANULARPair styles and fixes for finite-size granular particles, which interact with each other and boundaries via frictional and dissipative potentials. -
-
off (default)
-
on
-
-
PKG_KSPACEA variety of long-range Coulombic solvers, as well as pair styles which compute the corresponding short-range pairwise Coulombic interactions. These include Ewald, particle-particle particle-mesh (PPPM), and multilevel summation method (MSM) solvers. -
-
off (default)
-
on
-
-
PKG_MANYBODY - A variety of manybody and bond-order potentials. These include (AI)REBO, BOP, - EAM, EIM, Stillinger-Weber, and Tersoff potentials. - -
-
off (default)
-
on
-
-
PKG_MC - Several fixes and a pair style that have Monte Carlo (MC) or MC-like - attributes. These include fixes for creating, breaking, and swapping bonds, - for performing atomic swaps, and performing grand-canonical MC (GCMC) in - conjunction with dynamics. - -
-
off (default)
-
on
-
-
PKG_MISC - A variety of compute, fix, pair, dump styles with specialized capabilities that - don’t align with other packages. - -
-
off (default)
-
on
-
-
PKG_MOLECULE - A large number of atom, pair, bond, angle, dihedral, improper styles that are - used to model molecular systems with fixed covalent bonds. The pair styles - include the Dreiding (hydrogen-bonding) and CHARMM force fields, and a TIP4P - water model. - -
-
off (default)
-
on
-
-
PKG_PERI - An atom style, several pair styles which implement different Peridynamics - materials models, and several computes which calculate diagnostics. - Peridynamics is a a particle-based meshless continuum model. - -
-
off (default)
-
on
-
-
PKG_QEQ - Several fixes for performing charge equilibration (QEq) via different - algorithms. These can be used with pair styles that perform QEq as part of - their formulation. - -
-
off (default)
-
on
-
-
PKG_REPLICA - A collection of multi-replica methods which can be used when running multiple - LAMMPS simulations (replicas). See Section 6.5 for an overview of how to run - multi-replica simulations in LAMMPS. Methods in the package include nudged - elastic band (NEB), parallel replica dynamics (PRD), temperature accelerated - dynamics (TAD), parallel tempering, and a verlet/split algorithm for - performing long-range Coulombics on one set of processors, and the remainder - of the force field calcalation on another set. - -
-
off (default)
-
on
-
-
PKG_RIGID - Fixes which enforce rigid constraints on collections of atoms or particles. - This includes SHAKE and RATTLE, as well as varous rigid-body integrators for a - few large bodies or many small bodies. Also several computes which calculate - properties of rigid bodies. - -
-
off (default)
-
on
-
-
PKG_SHOCK - Fixes for running impact simulations where a shock-wave passes through a - material. - -
-
off (default)
-
on
-
-
PKG_SPINModel atomic magnetic spins classically, coupled to atoms moving in the usual manner via MD. Various pair, fix, and compute styles. -
-
off (default)
-
on
-
-
PKG_SNAP - A pair style for the spectral neighbor analysis potential (SNAP). SNAP is - methodology for deriving a highly accurate classical potential fit to a large - archive of quantum mechanical (DFT) data. Also several computes which analyze - attributes of the potential. - -
-
off (default)
-
on
-
-
PKG_SRD - A pair of fixes which implement the Stochastic Rotation Dynamics (SRD) method - for coarse-graining of a solvent, typically around large colloidal particles. - -
-
off (default)
-
on
-
-
- -## Other Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_KIMA pair_style kim command which is a wrapper on the Knowledge Base for Interatomic Models (KIM) repository of interatomic potentials, enabling any of them to be used in LAMMPS simulations. -
-
off (default)
-
on
-
-
PKG_PYTHONEnable support for Python scripting inside of LAMMPS. -
-
off (default)
-
on
-
-
PKG_MESSAGECommands to use LAMMPS as either a client or server and couple it to another application. -
-
off (default)
-
on
-
-
PKG_MSCG - A fix mscg command which can parameterize a Multi-Scale Coarse-Graining (MSCG) - model using the open-source MS-CG library. - -
-
off (default)
-
on
-
-
PKG_MPIIO - Support for parallel output/input of dump and restart files via the MPIIO library. - -
-
off (default)
-
on
-
-
PKG_POEMS - A fix that wraps the Parallelizable Open source Efficient Multibody Software - (POEMS) library, which is able to simulate the dynamics of articulated body - systems. These are systems with multiple rigid bodies (collections of - particles) whose motion is coupled by connections at hinge points. - -
-
off (default)
-
on
-
-
PKG_LATTE - A fix command which wraps the LATTE DFTB code, so that molecular dynamics can - be run with LAMMPS using density-functional tight-binding quantum forces - calculated by LATTE. - -
-
off (default)
-
on
-
-
PKG_VORONOI - A compute command which calculates the Voronoi tesselation of a collection of atoms by wrapping the Voro++ library. This can be used to calculate the local volume or each atoms or its near neighbors. - -
-
off (default)
-
on
-
-
- -## User Packages - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
PKG_USER-ADIOSADIOS is a high-performance I/O library. This package implements the dump “atom/adios” and dump “custom/adios” commands to write data using the ADIOS library. -
-
off (default)
-
on
-
-
PKG_USER-ATC - ATC stands for atoms-to-continuum. This package implements a fix atc command - to either couple molecular dynamics with continuum finite element equations or - perform on-the-fly conversion of atomic information to continuum fields. - -
-
off (default)
-
on
-
-
PKG_USER-AWPMD - AWPMD stands for Antisymmetrized Wave Packet Molecular Dynamics. This package - implements an atom, pair, and fix style which allows electrons to be treated - as explicit particles in a classical molecular dynamics model. - -
-
off (default)
-
on
-
-
PKG_USER-BOCS - This package provides fix bocs, a modified version of fix npt which includes the pressure correction to the barostat as outlined in: N. J. H. Dunn and W. G. Noid, “Bottom-up coarse-grained models that accurately describe the structure, pressure, and compressibility of molecular liquids,” J. Chem. Phys. 143, 243148 (2015). - -
-
off (default)
-
on
-
-
PKG_USER-CGDNA - Several pair styles, a bond style, and integration fixes for coarse-grained - models of single- and double-stranded DNA based on the oxDNA model of Doye, - Louis and Ouldridge at the University of Oxford. This includes Langevin-type - rigid-body integrators with improved stability. - -
-
off (default)
-
on
-
-
PKG_USER-CGSDK - Several pair styles and an angle style which implement the coarse-grained SDK - model of Shinoda, DeVane, and Klein which enables simulation of ionic liquids, - electrolytes, lipids and charged amino acids. - -
-
off (default)
-
on
-
-
PKG_USER-COLVARS - COLVARS stands for collective variables, which can be used to implement - various enhanced sampling methods, including Adaptive Biasing Force, - Metadynamics, Steered MD, Umbrella Sampling and Restraints. A fix colvars - command is implemented which wraps a COLVARS library, which implements these - methods. simulations. - -
-
off (default)
-
on
-
-
PKG_USER-DIFFRACTION - Two computes and a fix for calculating x-ray and electron diffraction - intensities based on kinematic diffraction theory. - -
-
off (default)
-
on
-
-
PKG_USER-DPD - DPD stands for dissipative particle dynamics. This package implements - coarse-grained DPD-based models for energetic, reactive molecular crystalline - materials. It includes many pair styles specific to these systems, including - for reactive DPD, where each particle has internal state for multiple species - and a coupled set of chemical reaction ODEs are integrated each timestep. - Highly accurate time integrators for isothermal, isoenergetic, isobaric and - isenthalpic conditions are included. These enable long timesteps via the - Shardlow splitting algorithm. - -
-
off (default)
-
on
-
-
PKG_USER-DRUDE - Fixes, pair styles, and a compute to simulate thermalized Drude oscillators as - a model of polarization. - -
-
off (default)
-
on
-
-
PKG_USER-EFF - EFF stands for electron force field which allows a classical MD code to model - electrons as particles of variable radius. This package contains atom, pair, - fix and compute styles which implement the eFF as described in A. - Jaramillo-Botero, J. Su, Q. An, and W.A. Goddard III, JCC, 2010. The eFF - potential was first introduced by Su and Goddard, in 2007. - -
-
off (default)
-
on
-
-
PKG_USER-FEP - FEP stands for free energy perturbation. This package provides methods for - performing FEP simulations by using a fix adapt/fep command with soft-core - pair potentials, which have a “soft” in their style name. - -
-
off (default)
-
on
-
-
PKG_USER-H5MD - H5MD stands for HDF5 for MD. HDF5 is a portable, binary, self-describing file - format, used by many scientific simulations. H5MD is a format for molecular - simulations, built on top of HDF5. This package implements a dump h5md command - to output LAMMPS snapshots in this format. - -
-
off (default)
-
on
-
-
PKG_USER-LB - Fixes which implement a background Lattice-Boltzmann (LB) fluid, which can be - used to model MD particles influenced by hydrodynamic forces. - -
-
off (default)
-
on
-
-
PKG_USER-MANIFOLD - Several fixes and a “manifold” class which enable simulations of particles - constrained to a manifold (a 2D surface within the 3D simulation box). This is - done by applying the RATTLE constraint algorithm to formulate single-particle - constraint functions g(xi,yi,zi) = 0 and their derivative (i.e. the normal of - the manifold) n = grad(g). - -
-
off (default)
-
on
-
-
PKG_USER-MEAMC - A pair style for the modified embedded atom (MEAM) potential translated from - the Fortran version in the MEAM package to plain C++. In contrast to the MEAM - package, no library needs to be compiled and the pair style can be - instantiated multiple times. - -
-
off (default)
-
on
-
-
PKG_USER-MESO - Several extensions of the the dissipative particle dynamics (DPD) method. - Specifically, energy-conserving DPD (eDPD) that can model non-isothermal - processes, many-body DPD (mDPD) for simulating vapor-liquid coexistence, and - transport DPD (tDPD) for modeling advection-diffusion-reaction systems. The - equations of motion of these DPD extensions are integrated through a modified - velocity-Verlet (MVV) algorithm. - -
-
off (default)
-
on
-
-
PKG_USER-MGPT - A pair style which provides a fast implementation of the quantum-based MGPT - multi-ion potentials. The MGPT or model GPT method derives from - first-principles DFT-based generalized pseudopotential theory (GPT) through a - series of systematic approximations valid for mid-period transition metals - with nearly half-filled d bands. The MGPT method was originally developed by - John Moriarty at LLNL. The pair style in this package calculates forces and - energies using an optimized matrix-MGPT algorithm due to Tomas Oppelstrup at - LLNL. - -
-
off (default)
-
on
-
-
PKG_USER-MISC - A potpourri of (mostly) unrelated features contributed to LAMMPS by users. - Each feature is a single fix, compute, pair, bond, angle, dihedral, improper, - or command style. - -
-
off (default)
-
on
-
-
PKG_USER-MOFFF - Pair, angle and improper styles needed to employ the MOF-FF force field by - Schmid and coworkers with LAMMPS. MOF-FF is a first principles derived force - field with the primary aim to simulate MOFs and related porous framework - materials, using spherical Gaussian charges. It is described in S. Bureekaew - et al., Phys. Stat. Sol. B 2013, 250, 1128-1141. For the usage of MOF-FF see - the example in the example directory as well as the MOF+ website. - -
-
off (default)
-
on
-
-
PKG_USER-MOLFILE - A dump molfile command which uses molfile plugins that are bundled with the - VMD molecular visualization and analysis program, to enable LAMMPS to dump - snapshots in formats compatible with various molecular simulation tools. - -
-
off (default)
-
on
-
-
PKG_USER-NETCDF - Dump styles for writing NetCDF formatted dump files. NetCDF is a portable, - binary, self-describing file format developed on top of HDF5. The file - contents follow the AMBER NetCDF trajectory conventions - (http://ambermd.org/netcdf/nctraj.xhtml), but include extensions. - -
-
off (default)
-
on
-
-
PKG_USER-PHONON - A fix phonon command that calculates dynamical matrices, which can then be - used to compute phonon dispersion relations, directly from molecular dynamics - simulations. - -
-
off (default)
-
on
-
-
PKG_USER-PLUMED - The fix plumed command allows you to use the PLUMED free energy plugin for molecular dynamics to analyze and bias your LAMMPS trajectory on the fly. The PLUMED library is called from within the LAMMPS input script by using the fix plumed command. - -
-
off (default)
-
on
-
-
PKG_USER-PTM - A compute ptm/atom command that calculates local structure characterization using the Polyhedral Template Matching methodology. - -
-
off (default)
-
on
-
-
PKG_USER-QTB - Two fixes which provide a self-consistent quantum treatment of vibrational modes in a classical molecular dynamics simulation. By coupling the MD simulation to a colored thermostat, it introduces zero point energy into the system, altering the energy power spectrum and the heat capacity to account for their quantum nature. This is useful when modeling systems at temperatures lower than their classical limits or when temperatures ramp across the classical limits in a simulation. - -
-
off (default)
-
on
-
-
PKG_USER-QUIP - A pair_style quip command which wraps the QUIP libAtoms library, which - includes a variety of interatomic potentials, including Gaussian Approximation - Potential (GAP) models developed by the Cambridge University group. - -
-
off (default)
-
on
-
-
PKG_USER-QMMM - A fix qmmm command which allows LAMMPS to be used in a QM/MM simulation, - currently only in combination with the Quantum ESPRESSO package. - -
-
off (default)
-
on
-
-
PKG_USER-REAXC - A pair style which implements the ReaxFF potential in C/C++ (in contrast to - the REAX package and its Fortran library). ReaxFF is universal reactive force - field. See the src/USER-REAXC/README file for more info on differences between - the two packages. Also two fixes for monitoring molecules as bonds are created - and destroyed. - -
-
off (default)
-
on
-
-
PKG_USER-SCAFACOS - A KSpace style which wraps the ScaFaCoS Coulomb solver library to compute long-range Coulombic interactions. - -
-
off (default)
-
on
-
-
PKG_USER-SDPD - A pair style for smoothed dissipative particle dynamics (SDPD), which is an - extension of smoothed particle hydrodynamics (SPH) to mesoscale where thermal - fluctuations are important (see the USER-SPH package). Also two fixes for - moving and rigid body integration of SPH/SDPD particles (particles of - atom_style meso). -
-
off (default)
-
on
-
-
PKG_USER-SMD - An atom style, fixes, computes, and several pair styles which implements - smoothed Mach dynamics (SMD) for solids, which is a model related to smoothed - particle hydrodynamics (SPH) for liquids (see the USER-SPH package). - -
-
off (default)
-
on
-
-
PKG_USER-SMTBQ - A pair style which implements a Second Moment Tight Binding model with QEq - charge equilibration (SMTBQ) potential for the description of ionocovalent - bonds in oxides. - -
-
off (default)
-
on
-
-
PKG_USER-SPH - An atom style, fixes, computes, and several pair styles which implements - smoothed particle hydrodynamics (SPH) for liquids. See the related USER-SMD - package package for smooth Mach dynamics (SMD) for solids. - -
-
off (default)
-
on
-
-
PKG_USER-TALLY - Several compute styles that can be called when pairwise interactions are - calculated to tally information (forces, heat flux, energy, stress, etc) about - individual interactions. - -
-
off (default)
-
on
-
-
PKG_USER-UEF - A fix style for the integration of the equations of motion under extensional - flow with proper boundary conditions, as well as several supporting compute - styles and an output option. - -
-
off (default)
-
on
-
-
PKG_USER-VTK - A dump vtk command which outputs snapshot info in the VTK format, enabling - visualization by Paraview or other visualization packages. - -
-
off (default)
-
on
-
-
PKG_USER-YAFF - Some potentials that are also implemented in the Yet Another Force Field (YAFF) code. - The expressions and their use are discussed in the following papers: - - -
-
off (default)
-
on
-
-
- -## Package-Specific Configuration Options - -### KSPACE Package - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
FFT -

FFT library for KSPACE package

-

If either MKL or FFTW is selected cmake will try to locate - these libraries automatically. To control which one should be used please see - the options below for each FFT library. Otherwise it will default to KISS - FFT.

- 		-
-
FFTW3
-
FFTW2
-
MKL
-
KISS (default)
-
-
FFT_SINGLEUse single-precision floating-point in FFT -
-
off (default = double precision)
-
on
-
-
FFT_PACKOptimization for FFT -
-
array (default)
-
pointer
-
memcpy
-
-
- -### FFTW3 - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
FFTW3_INCLUDE_DIRSpath to FFTW3 include files -
FFTW3_LIBRARIESlist of paths to FFTW3 libraries -
- -### MKL - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
MKL_INCLUDE_DIRSpath to MKL include files -
MKL_LIBRARIESlist of paths to MKL libraries -
- -### BLAS - -See [FindBLAS documentation](https://cmake.org/cmake/help/latest/module/FindBLAS.html) - -### LAPACK - -See [FindLAPACK documentation](https://cmake.org/cmake/help/latest/module/FindLAPACK.html) - -### PYTHON Package - -See [FindPYTHON documentation](https://cmake.org/cmake/help/latest/module/FindPython.html) - -### USER-INTEL Package - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
INTEL_ARCHTarget architecture for USER-INTEL package -
-
cpu (default)
-
knl
-
-
INTEL_LRT_MODEHow to support Long-range thread mode in Verlet integration -
-
threads (default, if pthreads available)
-
none (default, if pthreads not available)
-
c++11
-
-
- -### GPU Package -The GPU package builds a support library which can either use OpenCL or CUDA as -target API. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
GPU_APIAPI used by GPU package -
-
opencl (default)
-
cuda
-
-
GPU_PRECPrecision size used by GPU package kernels -
-
mixed (default)
-
single
-
double
-
-
OCL_TUNE (OpenCL only)Tuning target for OpenCL driver code -
-
generic (default)
-
intel (Intel CPU)
-
phi (Intel Xeon Phi)
-
fermi (NVIDIA)
-
kepler (NVIDIA)
-
cypress (AMD)
-
-
GPU_ARCH (CUDA only)CUDA SM architecture targeted by GPU package -
-
sm_20 (Fermi)
-
sm_30 (Kepler) (default)
-
sm_50 (Maxwell)
-
sm_60 (Pascal)
-
sm_70 (Volta)
-
sm_75 (Turing)
-
-
CUDPP_OPT (CUDA only)Enable CUDA Performance Primitives Optimizations -
-
on (default)
-
off
-
-
CUDA_MPS_SUPPORT (CUDA only)Enable tweaks for running with Nvidia CUDA Multi-process services daemon -
-
on
-
off (default)
-
-
BIN2C (CUDA only)Path to bin2c executable, will automatically pick up the first one in your $PATH.(automatic)
- -### KIM Package - -Requires installation of the KIM library with API v2 - -If `DOWNLOAD_KIM` is set, the KIM library will be downloaded and built inside -the CMake build directory. If the KIM library is already on your system (in a -location CMake cannot find it), set the `PKG_CONFIG_PATH` environment variable -so that `libkim-api` can be found. - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_KIMDownload KIM API v2 and compile it as part of the build. -
-
off (default)
-
on
-
-
- -### MESSAGE Package - -This package can optionally include support for messaging via sockets, using the open-source [ZeroMQ library](http://zeromq.org/), which must be installed on your system. - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
MESSAGE_ZMQBuild with ZeroMQ support -
-
off (default)
-
on
-
-
ZMQ_LIBRARY - ZMQ library file (only needed if at custom location) - -
ZMG_INCLUDE_DIR - Provide include directory of existing ZMQ installation (only needed if at custom location) - -
- -### MSCG Package - -Requires installation of the MSCG library - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_MSCGDownload MSCG and compile it as part of the build -
-
off (default)
-
on
-
-
MSCG_LIBRARY - MSCG library file (only needed if at custom location) - -
MSCG_INCLUDE_DIR - Provide include directory of existing MSCG installation (only needed if at custom location) - -
- -### VORONOI Package - -Requires installation of the Voro++ library - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_VORODownload Voro++ and compile it as part of the build -
-
off (default)
-
on
-
-
VORO_LIBRARY - Voro++ library file (only needed if at custom location) - -
VORO_INCLUDE_DIR - Provide include directory of existing Voro++ installation (only needed if at custom location) - -
- -### USER-COLVARS Package - -Requires a C++11 compiler to compile with the Lepton library included. - - - - - - - - - - - - - - - -
OptionDescriptionValues
COLVARS_LEPTONEnable the use of the Lepton library inside the Colvars library. - -
-
on (default)
-
off
-
-
- -### USER-LATTE Package - -Requires installation of the LATTE library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_LATTEDownload LATTE and compile it as part of the build -
-
off (default)
-
on
-
-
LATTE_LIBRARY - LATTE library file (only needed if at custom location) - -
- -### USER-PLUMED Package - -Requires installation of the PLUMED library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_PLUMEDDownload PLUMED and compile it as part of the build -
-
off (default)
-
on
-
-
PLUMED_MODE - Determines the linkage mode for the PLUMED library. - -
-
static (default)
-
shared
-
runtime
-
-
- -### USER-LATTE Package - -Requires installation of the LATTE library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_LATTEDownload LATTE and compile it as part of the build -
-
off (default)
-
on
-
-
LATTE_LIBRARY - LATTE library file (only needed if at custom location) - -
- -### USER-SMD Package - -Requires installation of the Eigen3 library - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_EIGEN3Download Eigen3 and compile it as part of the build -
-
off (default)
-
on
-
-
EIGEN3_INCLUDE_DIR - Provide include directory of existing Eigen3 installation (only needed if at custom location) - -
- - -### USER-SCAFACOS Package - -To build with this package, you must download and build the [ScaFaCoS Coulomb solver library](http://www.scafacos.de/) - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
DOWNLOAD_SCAFACOSDownload SCAFACOS and compile it as part of the build -
-
off (default)
-
on
-
-
SCAFACOS_LIBRARY - SCAFACOS library file (only needed if at custom location) - -
SCAFACOS_INCLUDE_DIR - SCAFACOS include directory (only needed if at custom location) - -
- -## Optional Features - -### zlib support - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
ZLIB_INCLUDE_DIR -
ZLIB_LIBRARIES -
- -### JPEG support - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_JPEGEnables/Disable JPEG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
JPEG_INCLUDE_DIR -
JPEG_LIBRARIES -
- -### PNG support -(requires zlib support) - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_PNGEnables/Disable PNG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
PNG_INCLUDE_DIR -
PNG_LIBRARIES -
- -### GZIP support - -requires `gzip` to be in your `PATH` - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_GZIPEnables/Disable GZIP support in LAMMPS -
-
yes (default, if found)
-
no
-
-
GZIP_EXECUTABLEPath to gzip executable, will automatically pick up the first one in your $PATH.(automatic)
- -### FFMPEG support - -requires `ffmpeg` to be in your `PATH` - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
WITH_FFMPEGEnables/Disable FFMPEG support in LAMMPS -
-
yes (default, if found)
-
no
-
-
FFMPEG_EXECUTABLEPath to ffmpeg executable, will automatically pick up the first one in your $PATH.(automatic)
- - -## Compilers - -By default, `cmake` will use your environment C/C++/Fortran compilers for a -build. It uses the `CC`, `CXX` and `FC` environment variables to detect which -compilers should be used. However, these values will be cached after the first -run of `cmake`. Subsequent runs of `cmake` will ignore changes in these -environment variables. To ensure the correct values are used you avoid the -cache by setting the `CMAKE_C_COMPILER`, `CMAKE_CXX_COMPILER`, -`CMAKE_Fortran_COMPILER` options directly. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionDefault
CMAKE_C_COMPILERC Compiler which should be used by CMakevalue of `CC` environment variable at first `cmake` run
CMAKE_CXX_COMPILERC++ compiler which should be used by CMake - value of `CXX` environment variable at first `cmake` run -
CMAKE_Fortran_COMPILERC++ compiler which should be used by CMake - value of `FC` environment variable at first `cmake` run -
CXX_COMPILER_LAUNCHERCMake will run this tool and pass the compiler and its arguments to the tool. Some example tools are distcc and ccache. - (empty) -
- -### Building with GNU Compilers - -```bash -cmake -D CMAKE_C_COMPILER=gcc -D CMAKE_CXX_COMPILER=g++ -D CMAKE_Fortran_COMPILER=gfortran ../cmake -``` - -### Building with Intel Compilers - -```bash -cmake -D CMAKE_C_COMPILER=icc -D CMAKE_CXX_COMPILER=icpc -D CMAKE_Fortran_COMPILER=ifort ../cmake -``` - - -### Building with LLVM/Clang Compilers - -```bash -cmake -D CMAKE_C_COMPILER=clang -D CMAKE_CXX_COMPILER=clang++ -D CMAKE_Fortran_COMPILER=flang ../cmake -``` - -## LAMMPS Developer Options - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescriptionValues
ENABLE_TESTINGControl wheather to add tests via CTest -
-
off (default)
-
on
-
-
LAMMPS_TESTING_SOURCE_DIRCustom location of lammps-testing repository (optional). If not specified it will download it via Git -
LAMMPS_TESTING_GIT_TAGIf lammps-testing repository is cloned, this is the tag/commit that will be checked out -
-
master (default)
-
-
ENABLE_COVERAGEEnables code coverage support via gcov and adds a gcovr build target to generate a coverage report. -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_ADDRESSEnables Address Sanitizer support when compiling using GCC or Clang for detecting memory leaks in binaries while running them. See https://clang.llvm.org/docs/AddressSanitizer.html -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_UNDEFINEDEnables Undefined Behavior Sanitizer support when compiling using GCC or Clang for detecting code that is running into undefined behavior of the language. See https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html -
-
off (default)
-
on
-
-
ENABLE_SANITIZE_THREADEnables Thread Sanitizer support when compiling using GCC or Clang for detecting data races in binaries while running them. See https://clang.llvm.org/docs/ThreadSanitizer.html -
-
off (default)
-
on
-
-
diff --git a/cmake/pkgconfig/liblammps.pc.in b/cmake/pkgconfig/liblammps.pc.in index a8710ca224..ca2c6191b7 100644 --- a/cmake/pkgconfig/liblammps.pc.in +++ b/cmake/pkgconfig/liblammps.pc.in @@ -18,12 +18,6 @@ # myapp_CFLAGS = $(LAMMPS_CFLAGS) # myapp_LDADD = $(LAMMPS_LIBS) -# Use this in CMake: -# CMakeLists.txt: -# find_package(PkgConfig) -# pkg_check_modules(LAMMPS IMPORTED_TARGET lammps) -# target_link_libraries( PkgConfig::LAMMPS) - prefix=@CMAKE_INSTALL_PREFIX@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ @@ -31,8 +25,8 @@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: liblammps@LAMMPS_MACHINE@ Description: Large-scale Atomic/Molecular Massively Parallel Simulator Library URL: http://lammps.sandia.gov -Version: @LAMMPS_VERSION@ +Version: @PROJECT_VERSION@ Requires: -Libs: -L${libdir} -llammps@LAMMPS_LIB_SUFFIX@ +Libs: -L${libdir} -llammps@LAMMPS_MACHINE@ Libs.private: -lm Cflags: -I${includedir} @LAMMPS_API_DEFINES@ diff --git a/cmake/presets/clang.cmake b/cmake/presets/clang.cmake index e2b1e02cc9..bfc355669d 100644 --- a/cmake/presets/clang.cmake +++ b/cmake/presets/clang.cmake @@ -2,7 +2,7 @@ set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE) -set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2 -DNDEBG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) set(MPI_CXX "clang++" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) unset(HAVE_OMP_H_INCLUDE CACHE) diff --git a/cmake/presets/hip.cmake b/cmake/presets/hip.cmake new file mode 100644 index 0000000000..047c8e8eb0 --- /dev/null +++ b/cmake/presets/hip.cmake @@ -0,0 +1,12 @@ +# preset that will enable hipcc plus gcc with support for MPI and OpenMP (on Linux boxes) + +set(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "" FORCE) +set(CMAKE_C_COMPILER "gcc" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +unset(HAVE_OMP_H_INCLUDE CACHE) + +set(OpenMP_CXX "hipcc" CACHE STRING "" FORCE) +set(OpenMP_CXX_FLAGS "-fopenmp" CACHE STRING "" FORCE) +set(OpenMP_CXX_LIB_NAMES "omp" CACHE STRING "" FORCE) +set(OpenMP_omp_LIBRARY "libomp.so" CACHE PATH "" FORCE) + diff --git a/cmake/presets/intel.cmake b/cmake/presets/intel.cmake index 869bf7bca2..099d744ad3 100644 --- a/cmake/presets/intel.cmake +++ b/cmake/presets/intel.cmake @@ -2,7 +2,6 @@ set(CMAKE_CXX_COMPILER "icpc" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE) -set(CMAKE_CXX_FLAGS "-O3 -DNDEBG" CACHE STRING "" FORCE) set(MPI_CXX "icpc" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) unset(HAVE_OMP_H_INCLUDE CACHE) diff --git a/cmake/presets/kokkos-cuda.cmake b/cmake/presets/kokkos-cuda.cmake new file mode 100644 index 0000000000..36d099479d --- /dev/null +++ b/cmake/presets/kokkos-cuda.cmake @@ -0,0 +1,11 @@ +# preset that enables KOKKOS and selects CUDA compilation with OpenMP +# enabled as well. This preselects CC 5.0 as default GPU arch, since +# that is compatible with all higher CC, but not the default CC 3.5 +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA ON CACHE BOOL "" FORCE) +set(Kokkos_ARCH_MAXWELL50 on CACHE BOOL "" FORCE) +set(BUILD_OMP ON CACHE BOOL "" FORCE) +get_filename_component(NVCC_WRAPPER_CMD ${CMAKE_CURRENT_SOURCE_DIR}/../lib/kokkos/bin/nvcc_wrapper ABSOLUTE) +set(CMAKE_CXX_COMPILER ${NVCC_WRAPPER_CMD} CACHE FILEPATH "" FORCE) diff --git a/cmake/presets/kokkos-openmp.cmake b/cmake/presets/kokkos-openmp.cmake new file mode 100644 index 0000000000..27d09f62cf --- /dev/null +++ b/cmake/presets/kokkos-openmp.cmake @@ -0,0 +1,6 @@ +# preset that enables KOKKOS and selects OpenMP (only) compilation +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "" FORCE) +set(BUILD_OMP ON CACHE BOOL "" FORCE) diff --git a/cmake/presets/kokkos-serial.cmake b/cmake/presets/kokkos-serial.cmake new file mode 100644 index 0000000000..0208d2ee3a --- /dev/null +++ b/cmake/presets/kokkos-serial.cmake @@ -0,0 +1,5 @@ +# preset that enables KOKKOS and selects serial compilation only +set(PKG_KOKKOS ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "" FORCE) +set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "" FORCE) diff --git a/cmake/presets/mingw-cross.cmake b/cmake/presets/mingw-cross.cmake index a733142455..61c4bbc45b 100644 --- a/cmake/presets/mingw-cross.cmake +++ b/cmake/presets/mingw-cross.cmake @@ -23,6 +23,7 @@ set(DOWNLOAD_VORO ON CACHE BOOL "" FORCE) set(DOWNLOAD_EIGEN3 ON CACHE BOOL "" FORCE) set(LAMMPS_MEMALIGN "0" CACHE STRING "" FORCE) set(CMAKE_TUNE_FLAGS "-Wno-missing-include-dirs" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "--enable-stdcall-fixup" CACHE STRING "" FORCE) +set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-stdcall-fixup" CACHE STRING "" FORCE) +set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--enable-stdcall-fixup" CACHE STRING "" FORCE) set(BUILD_TOOLS ON CACHE BOOL "" FORCE) set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lammps-installer") diff --git a/cmake/presets/most.cmake b/cmake/presets/most.cmake index 2f5b517526..aa5b60d146 100644 --- a/cmake/presets/most.cmake +++ b/cmake/presets/most.cmake @@ -5,10 +5,10 @@ set(ALL_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE GRANULAR KSPACE MANYBODY MC MISC MOLECULE OPT PERI POEMS PYTHON QEQ REPLICA RIGID SHOCK SNAP SPIN SRD VORONOI - USER-CGDNA USER-CGSDK USER-COLVARS USER-DIFFRACTION - USER-DPD USER-DRUDE USER-FEP USER-MEAMC USER-MESODPD + USER-BOCS USER-CGDNA USER-CGSDK USER-COLVARS USER-DIFFRACTION + USER-DPD USER-DRUDE USER-EFF USER-FEP USER-MEAMC USER-MESODPD USER-MISC USER-MOFFF USER-OMP USER-PHONON USER-REACTION - USER-REAXC USER-SPH USER-SMD USER-UEF USER-YAFF) + USER-REAXC USER-SDPD USER-SPH USER-SMD USER-UEF USER-YAFF) foreach(PKG ${ALL_PACKAGES}) set(PKG_${PKG} ON CACHE BOOL "" FORCE) diff --git a/doc/Makefile b/doc/Makefile index ff56226e83..c1f299a2f7 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -12,6 +12,7 @@ PYTHON = $(shell which python3) VIRTUALENV = virtualenv HAS_PYTHON3 = NO HAS_VIRTUALENV = NO +HAS_PDFLATEX = NO ifeq ($(shell which python3 >/dev/null 2>&1; echo $$?), 0) HAS_PYTHON3 = YES @@ -27,6 +28,11 @@ VIRTUALENV = virtualenv HAS_VIRTUALENV = YES endif +ifeq ($(shell which pdflatex >/dev/null 2>&1; echo $$?), 0) +HAS_PDFLATEX = YES +endif + + SPHINXEXTRA = -j $(shell $(PYTHON) -c 'import multiprocessing;print(multiprocessing.cpu_count())') .PHONY: help clean-all clean clean-spelling epub mobi rst html pdf spelling anchor_check style_check @@ -51,7 +57,7 @@ help: # ------------------------------------------ clean-all: clean - rm -rf $(BUILDDIR)/docenv $(BUILDDIR)/doctrees $(BUILDDIR)/mathjax + rm -rf $(BUILDDIR)/docenv $(BUILDDIR)/doctrees $(BUILDDIR)/mathjax Manual.pdf Developer.pdf clean: clean-spelling rm -rf html epub latex @@ -77,7 +83,7 @@ html: $(ANCHORCHECK) $(MATHJAX) @rm -rf html/JPG @cp -r src/PDF html/PDF @mkdir -p html/JPG - @cp `grep -A2 '\.\. .*image::' src/*.rst | grep ':target:' | sed -e 's,.*:target: JPG/,src/JPG/,' | sort | uniq` html/JPG/ + @cp `grep -A2 '\.\. .*\(image\|figure\)::' src/*.rst | grep ':target:' | sed -e 's,.*:target: JPG/,src/JPG/,' | sort | uniq` html/JPG/ @rm -rf html/PDF/.[sg]* @mkdir -p html/_static/mathjax @cp -r $(MATHJAX)/es5 html/_static/mathjax/ @@ -86,8 +92,7 @@ html: $(ANCHORCHECK) $(MATHJAX) spelling: $(VENV) utils/sphinx-config/false_positives.txt @(\ . $(VENV)/bin/activate ;\ - pip install sphinxcontrib-spelling ;\ - cp utils/sphinx-config/false_positives.txt $(RSTDIR)/ ;\ + cp utils/sphinx-config/false_positives.txt $(RSTDIR)/ ; env PYTHONWARNINGS= \ sphinx-build -b spelling -c utils/sphinx-config -d $(BUILDDIR)/doctrees $(RSTDIR) spelling ;\ deactivate ;\ ) @@ -113,6 +118,7 @@ mobi: epub @echo "Conversion finished. The MOBI manual file is created." pdf: $(ANCHORCHECK) + @if [ "$(HAS_PDFLATEX)" == "NO" ] ; then echo "PDFLaTeX was not found! Please check README.md for further instructions" 1>&2; exit 1; fi @(\ cd src/Developer; \ pdflatex developer; \ @@ -187,7 +193,10 @@ $(VENV): @( \ $(VIRTUALENV) -p $(PYTHON) $(VENV); \ . $(VENV)/bin/activate; \ + pip install --upgrade pip; \ pip install Sphinx; \ + pip install sphinxcontrib-spelling ;\ + pip install breathe; \ deactivate;\ ) diff --git a/doc/README b/doc/README index 1416584eed..cbc0c425dc 100644 --- a/doc/README +++ b/doc/README @@ -93,12 +93,18 @@ support for PDFLaTeX. Also the following LaTeX packages need to be installed (e.g. from texlive): - amsmath - babel +- capt-of - cmap - fncychap +- framed - geometry - hyperref - hypcap +- needspace - times +- tabulary +- upquote +- wrapfig ---------------- Installing prerequisites for epub build diff --git a/doc/include-file-conventions.md b/doc/include-file-conventions.md index 132f2ec4eb..1b8e79d71e 100644 --- a/doc/include-file-conventions.md +++ b/doc/include-file-conventions.md @@ -49,22 +49,15 @@ include files provided with LAMMPS are included with double quotes For headers declaring functions of the C-library, the corresponding C++ versions should be included (examples: `#include ` or -`#include `). However, these includes are limited to those defined -in the C++98 standard. Some files thus must use the older style until -the minimum C++ standard requirement of LAMMPS is lifted to C++11 or -even beyond (examples: `#include ` versus `#include ` -or `#include ` versus `#include `). +`#include ` instead of `#include ` or +`#include` ). ### C++ Standard Compliance -LAMMPS core files currently correspond to the C++98 standard. Files -requiring C++11 or later are only permitted in (optional) packages -and particularly packages that are not part of the list of commonly -used packages such as MOLECULE, KSPACE, MANYBODY, or RIGID. - -Also, LAMMPS uses the C-style stdio library for I/O instead of iostreams. -Since using both at the same time can cause problems, iostreams should -be avoided where possible. +LAMMPS core files use standard conforming C++ compatible with the +C++11 standard, unless explicitly noted. Also, LAMMPS uses the C-style +stdio library for I/O instead of iostreams. Since using both at the +same time can cause problems, iostreams should be avoided where possible. ### Lean Header Files diff --git a/doc/lammps.1 b/doc/lammps.1 index d2851603c6..80d07cbf0a 100644 --- a/doc/lammps.1 +++ b/doc/lammps.1 @@ -1,4 +1,4 @@ -.TH LAMMPS "19 March 2020" "2020-03-19" +.TH LAMMPS "5 May 2020" "2020-05-5" .SH NAME .B LAMMPS \- Molecular Dynamics Simulator. diff --git a/doc/src/.gitignore b/doc/src/.gitignore index e0b9693457..afe24cc10c 100644 --- a/doc/src/.gitignore +++ b/doc/src/.gitignore @@ -1,3 +1 @@ -/Eqs -/JPG /false_positives.txt diff --git a/doc/src/Build.rst b/doc/src/Build.rst index c96fb0b5b9..441af79009 100644 --- a/doc/src/Build.rst +++ b/doc/src/Build.rst @@ -1,12 +1,14 @@ Build LAMMPS ************ -LAMMPS can be built as an executable or library from source code via -either traditional makefiles (which may require manual editing) -for use with GNU make or gmake, or a build environment generated by CMake -(Unix Makefiles, Xcode, Visual Studio, KDevelop or more). As an -alternative you can download a package with pre-built executables -as described on the :doc:`Install ` doc page. +LAMMPS is built as a library and an executable from source code using +either traditional makefiles for use with GNU make (which may require +manual editing), or using a build environment generated by CMake (Unix +Makefiles, Ninja, Xcode, Visual Studio, KDevelop, CodeBlocks and more). + +As an alternative you can download a package with pre-built executables +or automated build trees as described on the :doc:`Install ` +doc page. .. toctree:: :maxdepth: 1 diff --git a/doc/src/Build_basics.rst b/doc/src/Build_basics.rst index 8ad48b0810..335c1a5189 100644 --- a/doc/src/Build_basics.rst +++ b/doc/src/Build_basics.rst @@ -6,7 +6,8 @@ CMake and make: * :ref:`Serial vs parallel build ` * :ref:`Choice of compiler and compile/link options ` -* :ref:`Build LAMMPS as an executable or a library ` +* :ref:`Build the LAMMPS executable and library ` +* :ref:`Including and removing debug support ` * :ref:`Build the LAMMPS documentation ` * :ref:`Install LAMMPS after a build ` @@ -15,7 +16,7 @@ CMake and make: .. _serial: Serial vs parallel build -------------------------------------- +------------------------ LAMMPS is written to use the ubiquitous `MPI (Message Passing Interface) `_ library API @@ -28,8 +29,8 @@ MPI STUBS library. Independent of the distributed memory MPI parallelization, parts of LAMMPS are also written with support for shared memory parallelization -using the OpenMP threading standard. A more detailed discussion of that -is below. +using the `OpenMP `_ threading +standard. A more detailed discussion of that is below. **CMake build**\ : @@ -41,7 +42,7 @@ is below. # no default value The executable created by CMake (after running make) is named ``lmp`` unless -the LAMMPS_MACHINE option is set. When setting ``LAMMPS_MACHINE=name`` +the ``LAMMPS_MACHINE`` option is set. When setting ``LAMMPS_MACHINE=name`` the executable will be called ``lmp_name``. Using ``BUILD_MPI=no`` will enforce building a serial executable using the MPI STUBS library. @@ -55,18 +56,20 @@ The build with traditional makefiles has to be done inside the source folder ``s make serial # serial build, produces lmp_serial using Makefile/serial make mybox # uses Makefile.mybox to produce lmp_mybox -Any "make machine" command will look up the make settings from a file -Makefile.machine, create a folder Obj_machine with all objects and -generated files and an executable called ``lmp_machine``\ . The standard -parallel build with ``make mpi`` assumes a standard MPI installation with -MPI compiler wrappers where all necessary compiler and linker flags to -get access and link with the suitable MPI headers and libraries are set -by the wrapper programs. For other cases or the serial build, you have -to adjust the make file variables ``MPI_INC``, ``MPI_PATH``, ``MPI_LIB`` -as well as ``CC`` and ``LINK``\ . To enable OpenMP threading usually -a compiler specific flag needs to be added to the compile and link -commands. For the GNU compilers, this is ``-fopenmp``\ , which can be -added to the ``CC`` and ``LINK`` makefile variables. +Any ``make machine`` command will look up the make settings from a file +``Makefile.machine`` in the folder ``src/MAKE`` or one of its +sub-directories ``MINE``, ``MACHINES``, or ``OPTIONS``, create a folder +``Obj_machine`` with all objects and generated files and an executable +called ``lmp_machine``\ . The standard parallel build with ``make mpi`` +assumes a standard MPI installation with MPI compiler wrappers where all +necessary compiler and linker flags to get access and link with the +suitable MPI headers and libraries are set by the wrapper programs. For +other cases or the serial build, you have to adjust the make file +variables ``MPI_INC``, ``MPI_PATH``, ``MPI_LIB`` as well as ``CC`` and +``LINK``\ . To enable OpenMP threading usually a compiler specific flag +needs to be added to the compile and link commands. For the GNU +compilers, this is ``-fopenmp``\ , which can be added to the ``CC`` and +``LINK`` makefile variables. For the serial build the following make variables are set (see src/MAKE/Makefile.serial): @@ -80,18 +83,19 @@ For the serial build the following make variables are set (see src/MAKE/Makefile You also need to build the STUBS library for your platform before making LAMMPS itself. A ``make serial`` build does this for you automatically, -otherwise, type ``make mpi-stubs`` from the src directory, or ``make`` from -the src/STUBS dir. If the build fails, you will need to edit the -STUBS/Makefile for your platform. The stubs library does not provide -MPI/IO functions required by some LAMMPS packages, e.g. MPIIO or USER-LB, -and thus is not compatible with those packages. +otherwise, type ``make mpi-stubs`` from the src directory, or ``make`` +from the ``src/STUBS`` dir. If the build fails, you may need to edit +the ``STUBS/Makefile`` for your platform. The stubs library does not +provide MPI/IO functions required by some LAMMPS packages, +e.g. ``MPIIO`` or ``USER-LB``, and thus is not compatible with those +packages. .. note:: The file ``src/STUBS/mpi.c`` provides a CPU timer function called - MPI_Wtime() that calls gettimeofday() . If your operating system - does not support gettimeofday() , you will need to insert code to - call another timer. Note that the ANSI-standard function clock() + ``MPI_Wtime()`` that calls ``gettimeofday()``. If your operating system + does not support ``gettimeofday()``, you will need to insert code to + call another timer. Note that the ANSI-standard function ``clock()`` rolls over after an hour or so, and is therefore insufficient for timing long LAMMPS simulations. @@ -111,20 +115,22 @@ self-installed MPICH or OpenMPI, so you should study the provided documentation to find out how to build and link with it. The majority of OpenMP (threading) support in LAMMPS is provided by the -USER-OMP package; see the :doc:`Speed omp ` doc page for -details. The USER-INTEL package also includes OpenMP threading (it is -compatible with USER-OMP) and adds vectorization support when compiled -with compatible compilers, in particular the Intel compilers on top of -OpenMP. Also, the KOKKOS package can be compiled to include OpenMP -threading. +``USER-OMP`` package; see the :doc:`Speed omp ` doc page for +details. The ``USER-INTEL`` package also includes OpenMP threading (it +is compatible with ``USER-OMP`` and will usually fall back on styles +from that package, if a ``USER-INTEL`` does not exist) and adds +vectorization support when compiled with compatible compilers, in +particular the Intel compilers on top of OpenMP. Also, the ``KOKKOS`` +package can be compiled to include OpenMP threading. In addition, there are a few commands in LAMMPS that have native OpenMP -support included as well. These are commands in the MPIIO, SNAP, -USER-DIFFRACTION, and USER-DPD packages. In addition some packages -support OpenMP threading indirectly through the libraries they interface -to: e.g. LATTE and USER-COLVARS. See the :doc:`Packages details -` doc page for more info on these packages and the doc -pages for their respective commands for OpenMP threading info. +support included as well. These are commands in the ``MPIIO``, +``SNAP``, ``USER-DIFFRACTION``, and ``USER-DPD`` packages. In addition +some packages support OpenMP threading indirectly through the libraries +they interface to: e.g. ``LATTE``, ``KSPACE``, and ``USER-COLVARS``. +See the :doc:`Packages details ` doc page for more +info on these packages and the doc pages for their respective commands +for OpenMP threading info. For CMake, if you use ``BUILD_OMP=yes``, you can use these packages and turn on their native OpenMP support and turn on their native OpenMP @@ -142,14 +148,15 @@ please refer to its documentation. **OpenMP Compiler compatibility info**\ : Some compilers do not fully support the ``default(none)`` directive -and others (e.g. GCC version 9 and beyond) may implement OpenMP 4.0 -semantics, which are incompatible with the OpenMP 3.1 semantics used -in LAMMPS (for maximal compatibility with compiler versions in use). -In those case, all ``default(none)`` directives (which aid in detecting -incorrect and unwanted sharing) can be replaced with ``default(shared)`` -while dropping all ``shared()`` directives. The script -'src/USER-OMP/hack_openmp_for_pgi_gcc9.sh' can be used to automate -this conversion. +and others (e.g. GCC version 9 and beyond, Clang version 10 and later) +may implement strict OpenMP 4.0 and later semantics, which are incompatible +with the OpenMP 3.1 semantics used in LAMMPS for maximal compatibility +with compiler versions in use. If compilation with OpenMP enabled fails +because of your compiler requiring strict OpenMP 4.0 semantic, you can +change the behavior by adding ``-D LAMMPS_OMP_COMPAT=4`` to the ``LMP_INC`` +variable in your makefile, or add it to the command line while configuring +with CMake. CMake will detect the suitable setting for the GNU, Clang, +and Intel compilers. ---------- @@ -158,15 +165,18 @@ this conversion. Choice of compiler and compile/link options --------------------------------------------------------- -The choice of compiler and compiler flags can be important for +The choice of compiler and compiler flags can be important for maximum performance. Vendor provided compilers for a specific hardware can produce faster code than open-source compilers like the GNU compilers. -On x86 hardware most popular compilers are quite similar in performance -of C/C++ code at high optimization levels. When using the USER-INTEL -package, there is a distinct advantage in using the `Intel C++ compiler -`_ due to much improved vectorization through SSE and AVX -instructions on compatible hardware as the source code includes changes -and compiler directives to enable high degrees of vectorization. +On the most common x86 hardware most popular C++ compilers are quite +similar in performance of C/C++ code at high optimization levels. When +using the ``USER-INTEL`` package, there is a distinct advantage in using +the `Intel C++ compiler `_ due to much improved vectorization +through SSE and AVX instructions on compatible hardware as the source +code includes changes and Intel compiler specific directives to enable +high degrees of vectorization. This may change over time as equivalent +vectorization directives are included into OpenMP standard revisions and +other compilers adopt them. .. _intel: https://software.intel.com/en-us/intel-compilers @@ -177,17 +187,20 @@ LAMMPS. **CMake build**\ : -By default CMake will use a compiler it finds and it will add -optimization flags appropriate to that compiler and any -:doc:`accelerator packages ` you have included in the -build. +By default CMake will use a compiler it finds according to internal +preferences and it will add optimization flags appropriate to that +compiler and any :doc:`accelerator packages ` you have +included in the build. -You can tell CMake to look for a specific compiler with these variable -settings. Likewise you can specify the corresponding ``CMAKE_*_FLAGS`` -variables if you want to experiment with alternate optimization flags. -You should specify all 3 compilers, so that the small number of LAMMPS -source files written in C or Fortran are built with a compiler consistent -with the one used for all the C++ files: +You can tell CMake to look for a specific compiler with setting CMake +variable during configuration. For a few common choices, there are also +presets in the ``cmake/presets`` folder. For convenience, there is a +``CMAKE_TUNE_FLAGS`` variable that can be set to apply global compiler +options. More on that below, but you can also specify the corresponding +``CMAKE_*_FLAGS`` variables individually if you want to experiment with +alternate optimization flags. You should specify all 3 compilers, so +that the (few) LAMMPS source files written in C or Fortran are built +with a compiler consistent with the one used for the C++ files: .. code-block:: bash @@ -210,16 +223,17 @@ A few example command lines are: # Building with LLVM/Clang Compilers: cmake ../cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_Fortran_COMPILER=flang -For compiling with the Clang/LLVM compilers a special CMake preset is -included that can be loaded with `-C ../cmake/presets/clang.cmake`. +For compiling with the Clang/LLVM compilers a CMake preset is provided that +can be loaded with `-C ../cmake/presets/clang.cmake`. Similarly, +`-C ../cmake/presets/intel.cmake` should switch the -In addition you can set ``CMAKE_TUNE_FLAGS`` to specifically add compiler -flags to tune for optimal performance on given hosts. By default these are -initialized to some compiler specific flags, where known, to optimize the -LAMMPS executable with optimizations and instructions available on the host -where LAMMPS is compiled. For example, for Intel compilers this would be -``-xHost`` and for GNU compilers this would be ``-march=native``. To turn -these flags off, set ``-D CMAKE_TUNE_FLAGS=``. +In addition you can set ``CMAKE_TUNE_FLAGS`` to specifically add +compiler flags to tune for optimal performance on given hosts. By +default these are initialized to some compiler specific flags, to +optimize the LAMMPS executable with optimizations and instructions +available on the host where LAMMPS is compiled. For example, for Intel +compilers this would be ``-xHost`` and for GNU compilers this would be +``-march=native``. To turn these flags off, do ``-D CMAKE_TUNE_FLAGS=``. .. note:: @@ -235,11 +249,11 @@ these flags off, set ``-D CMAKE_TUNE_FLAGS=``. The "compiler/linker settings" section of a Makefile.machine lists compiler and linker settings for your C++ compiler, including optimization flags. For a parallel build it is recommended to use -mpicxx or mpiCC, since these compiler wrappers will include a variety of -settings appropriate for your MPI installation and thus avoiding the -guesswork of finding the right flags. +``mpicxx`` or ``mpiCC``, since these compiler wrappers will include a +variety of settings appropriate for your MPI installation and thus +avoiding the guesswork of finding the right flags. -Parallel build (see src/MAKE/Makefile.mpi): +Parallel build (see ``src/MAKE/Makefile.mpi``): .. code-block:: bash @@ -248,7 +262,7 @@ Parallel build (see src/MAKE/Makefile.mpi): LINK = mpicxx LINKFLAGS = -g -O -Serial build (see src/MAKE/Makefile.serial): +Serial build with GNU gcc (see ``src/MAKE/Makefile.serial``): .. code-block:: make @@ -257,17 +271,35 @@ Serial build (see src/MAKE/Makefile.serial): LINK = g++ LINKFLAGS = -g -O + .. note:: - If you build LAMMPS with any :doc:`accelerator packages ` - included, there may be specific optimization flags that are either - required or recommended to enable required features and to achieve - optimal performance. You need to include these in the CCFLAGS and - LINKFLAGS settings above. For details, see the individual package - doc pages listed on the :doc:`Speed packages ` doc - page. Or examine these files in the src/MAKE/OPTIONS directory. - They correspond to each of the 5 accelerator packages and their - hardware variants: + If compilation stops with a message like the following: + + .. code-block:: + + g++ -g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I../STUBS -c ../main.cpp + In file included from ../pointers.h:24:0, + from ../input.h:17, + from ../main.cpp:16: + ../lmptype.h:34:2: error: #error LAMMPS requires a C++11 (or later) compliant compiler. Enable C++11 compatibility or upgrade the compiler. + + then you have either an unsupported (old) compiler or you have to + turn on C++11 mode. The latter applies to GCC 4.8.x shipped with + RHEL 7.x and CentOS 7.x. For those compilers, you need to add the + ``-std=c++11`` flag. Otherwise, you would have to install a newer + compiler that supports C++11; either as a binary package or through + compiling from source. + +If you build LAMMPS with any :doc:`accelerator packages +` included, there may be specific optimization flags +that are either required or recommended to enable required features and +to achieve optimal performance. You need to include these in the +CCFLAGS and LINKFLAGS settings above. For details, see the individual +package doc pages listed on the :doc:`Speed packages ` +doc page. Or examine these files in the src/MAKE/OPTIONS directory. +They correspond to each of the 5 accelerator packages and their hardware +variants: .. code-block:: bash @@ -284,69 +316,74 @@ Serial build (see src/MAKE/Makefile.serial): .. _exe: -Build LAMMPS as an executable or a library ----------------------------------------------------- +Build the LAMMPS executable and library +--------------------------------------- -LAMMPS can be built as either an executable or as a static or shared -library. The LAMMPS library can be called from another application or -a scripting language. See the :doc:`Howto couple ` doc +LAMMPS is always built as a library of C++ classes plus an executable. +The executable is a simple ``main()`` function that sets up MPI and then +creates a LAMMPS class instance from the LAMMPS library, which +will then process commands provided via a file or from the console +input. The LAMMPS library can also be called from another application +or a scripting language. See the :doc:`Howto couple ` doc page for more info on coupling LAMMPS to other codes. See the :doc:`Python ` doc page for more info on wrapping and running LAMMPS from Python via its library interface. **CMake build**\ : -For CMake builds, you can select through setting CMake variables which -files the compilation produces during the configuration step. If none -are set, defaults are applied. +For CMake builds, you can select through setting CMake variables between +building a shared or a static LAMMPS library and what kind of suffix is +added to them (in case you want to concurrently install multiple variants +of binaries with different settings). If none are set, defaults are applied. .. code-block:: bash - -D BUILD_EXE=value # yes (default) or no - -D BUILD_LIB=value # yes or no (default) -D BUILD_SHARED_LIBS=value # yes or no (default) - -D LAMMPS_LIB_SUFFIX=name # name = mpi, serial, mybox, titan, laptop, etc + -D LAMMPS_MACHINE=name # name = mpi, serial, mybox, titan, laptop, etc # no default value -Setting ``BUILD_EXE=no`` will not produce an executable. Setting -``BUILD_LIB=yes`` will produce a static library named ``liblammps.a``\ . -Setting both ``BUILD_LIB=yes`` and ``BUILD_SHARED_LIBS=yes`` will produce a -shared library named ``liblammps.so`` instead. If ``LAMMPS_LIB_SUFFIX=name`` -is set in addition, the name of the generated libraries will be changed to -either ``liblammps_name.a`` or ``liblammps_name.so``\ , respectively. +The compilation will always produce a LAMMPS library and an executable +linked to it. By default this will be a static library named +``liblammps.a`` and an executable named ``lmp`` Setting +``BUILD_SHARED_LIBS=yes`` will instead produce a shared library called +``liblammps.so`` (or ``liblammps.dylib`` or ``liblammps.dll`` depending +on the platform) If ``LAMMPS_MACHINE=name`` is set in addition, the name +of the generated libraries will be changed to either +``liblammps_name.a`` or ``liblammps_name.so``\ , respectively and the +executable will be called ``lmp_name``. **Traditional make**\ : With the traditional makefile based build process, the choice of the generated executable or library depends on the "mode" setting. -Several options are available and ``mode=exe`` is the default. +Several options are available and ``mode=static`` is the default. .. code-block:: bash make machine # build LAMMPS executable lmp_machine - make mode=exe machine # same as "make machine" - make mode=lib machine # build LAMMPS static lib liblammps_machine.a - make mode=shlib machine # build LAMMPS shared lib liblammps_machine.so - make mode=shexe machine # same as "mode=exe" but uses objects from "mode=shlib" + make mode=static machine # same as "make machine" + make mode=shared machine # build LAMMPS shared lib liblammps_machine.so instead -The two "exe" builds will generate and executable ``lmp_machine``\ , -while the two library builds will create a file ``liblammps_machine.a`` -or ``liblammps_machine.so``\ . They will also create generic soft links, -named ``liblammps.a`` and ``liblammps.so``\ , which point to the specific +The "static" build will generate a static library called +``liblammps_machine.a`` and an executable named ``lmp_machine``\ , while +the "shared" build will generate a shared library +``liblammps_machine.so`` instead and ``lmp_machine`` will be linked to +it. The build step will also create generic soft links, named +``liblammps.a`` and ``liblammps.so``\ , which point to the specific ``liblammps_machine.a/so`` files. **CMake and make info**\ : -Note that for a shared library to be usable by a calling program, all -the auxiliary libraries it depends on must also exist as shared -libraries. This will be the case for libraries included with LAMMPS, -such as the dummy MPI library in src/STUBS or any package libraries in -the lib/packages directory, since they are always built in a shared -library compatible way using the ``-fPIC`` switch. However, if a library -like MPI or FFTW does not exist as a shared library, the shared library -build may generate an error. This means you will need to install a -shared library version of the auxiliary library. The build instructions -for the library should tell you how to do this. +Note that for creating a shared library, all the libraries it depends on +must be compiled to be compatible with shared libraries. This should be +the case for libraries included with LAMMPS, such as the dummy MPI +library in ``src/STUBS`` or any package libraries in the ``lib`` +directory, since they are always built in a shared library compatible +way using the ``-fPIC`` compiler switch. However, if an auxiliary +library (like MPI or FFTW) does not exist as a compatible format, the +shared library linking step may generate an error. This means you will +need to install a compatible version of the auxiliary library. The +build instructions for that library should tell you how to do this. As an example, here is how to build and install the `MPICH library `_, a popular open-source version of MPI, as a shared library @@ -360,10 +397,12 @@ in the default /usr/local/lib location: make make install -You may need to use ``sudo make install`` in place of the last line if you -do not have write privileges for ``/usr/local/lib``. The end result should -be the file ``/usr/local/lib/libmpich.so``. On many Linux installations the -folder ``${HOME}/.local`` is an alternative to using ``/usr/local`` and does +You may need to use ``sudo make install`` in place of the last line if +you do not have write privileges for ``/usr/local/lib`` or use the +``--prefix`` configuration option to select an installation folder, +where you do have write access. The end result should be the file +``/usr/local/lib/libmpich.so``. On many Linux installations the folder +``${HOME}/.local`` is an alternative to using ``/usr/local`` and does not require superuser or sudo access. In that case the configuration step becomes: @@ -371,12 +410,41 @@ step becomes: ./configure --enable-shared --prefix=${HOME}/.local -Avoiding using "sudo" for custom software installation (i.e. from source +Avoiding to use "sudo" for custom software installation (i.e. from source and not through a package manager tool provided by the OS) is generally recommended to ensure the integrity of the system software installation. ---------- +.. _debug: + +Excluding or removing debug support +----------------------------------- + +By default the compilation settings will include the *-g* flag which +instructs the compiler to include debug information (e.g. which line of +source code a particular instruction correspond to). This can be +extremely useful in case LAMMPS crashes and can help to provide crucial +information in :doc:`tracking down the origin of a crash ` +and help the LAMMPS developers fix bugs in the source code. However, +this increases the storage requirements for object files, libraries, and +the executable 3-5 fold. + +If this is a concern, you can change the compilation settings or remove +the debug information from the LAMMPS executable: + +- **Traditional make**: edit your ``Makefile.`` to remove the + *-g* flag from the ``CCFLAGS`` and ``LINKFLAGS`` definitions +- **CMake**: use ``-D CMAKE_BUILD_TYPE=Release`` or explicitly reset + the applicable compiler flags (best done using the text mode or + graphical user interface). +- **Remove debug info**: If you are only concerned about the executable + being too large, you can use the ``strip`` tool (e.g. ``strip + lmp_serial``) to remove the debug information from the executable file. + Do not strip libraries or object files, as that will render them unusable. + +---------- + .. _doc: Build the LAMMPS documentation @@ -427,7 +495,8 @@ It is also possible to create the HTML version of the manual within the :doc:`CMake build directory `. The reason for this option is to include the installation of the HTML manual pages into the "install" step when installing LAMMPS after the CMake build via -``make install``. +``make install``. The documentation build is included in the default +build target, but can also be requested independently with ``make doc``. .. code-block:: bash diff --git a/doc/src/Build_cmake.rst b/doc/src/Build_cmake.rst index 7a60ffa437..3473d83143 100644 --- a/doc/src/Build_cmake.rst +++ b/doc/src/Build_cmake.rst @@ -1,205 +1,158 @@ Build LAMMPS with CMake ======================= -This page is a short summary of how to use CMake to build LAMMPS. -Details on CMake variables that enable specific LAMMPS build options -are given on the pages linked to from the :doc:`Build ` doc page. +This page describes how to use `CMake `_ in general +to build LAMMPS. Details for specific compile time settings and options +to enable and configure add-on packages are discussed with those +packages. Links to those pages on the :doc:`Build overview ` +page. -Richard Berger (Temple U) has also written a `more comprehensive guide `_ -for how to use CMake to build LAMMPS. If you are new to CMake it is a -good place to start. +The following text assumes some familiarity with CMake and focuses on +using the command line tool ``cmake`` and what settings are supported +for building LAMMPS. A more detailed tutorial on how to use ``cmake`` +itself, the text mode or graphical user interface, change the generated +output files for different build tools and development environments is +on a :doc:`separate page `. ----------- +.. note:: + + LAMMPS currently requires that CMake version 3.10 or later is available; + version 3.12 or later is preferred. + +.. warning:: + + You must not mix the :doc:`traditional make based ` + LAMMPS build procedure with using CMake. Thus no packages may be + installed or a build been previously attempted in the LAMMPS source + directory by using ``make ``. CMake will detect if this is + the case and generate an error. To remove conflicting files from the + ``src`` you can use the command ``make no-all purge`` which will + un-install all packages and delete all auto-generated files. + + +Advantages of using CMake +^^^^^^^^^^^^^^^^^^^^^^^^^ + +CMake is an alternative to compiling LAMMPS in the traditional way +through :doc:`(manually customized) makefiles ` and a recent +addition to LAMMPS thanks to the efforts of Christoph Junghans (LANL) +and Richard Berger (Temple U). Using CMake has multiple advantages that +are specifically helpful for people with limited experience in compiling +software or for people that want to modify or extend LAMMPS. + +- CMake can detect available hardware, tools, features, and libraries + and adapt the LAMMPS default build configuration accordingly. +- CMake can generate files for different build tools and integrated + development environments (IDE). +- CMake supports customization of settings with a text mode or graphical + user interface. No knowledge of file formats or and complex command + line syntax required. +- All enabled components are compiled in a single build operation. +- Automated dependency tracking for all files and configuration options. +- Support for true out-of-source compilation. Multiple configurations + and settings with different choices of LAMMPS packages, settings, or + compilers can be configured and built concurrently from the same + source tree. +- Simplified packaging of LAMMPS for Linux distributions, environment + modules, or automated build tools like `Homebrew `_. +- Integration of automated regression testing (the LAMMPS side for that + is still under development). + +.. _cmake_build: + +Getting started +^^^^^^^^^^^^^^^ Building LAMMPS with CMake is a two-step process. First you use CMake -to create a build environment in a new directory. On Linux systems, -this will be by default based on Unix-style makefiles for use with make. -Then you use the *make* command to build LAMMPS, which uses the created -Makefile(s). Example: +to generate a build environment in a new directory. For that purpose +you can use either the command-line utility ``cmake`` (or ``cmake3``), +the text-mode UI utility ``ccmake`` (or ``ccmake3``) or the graphical +utility ``cmake-gui``, or use them interchangeably. The second step is +then the compilation and linking of all objects, libraries, and +executables. Here is a minimal example using the command line version of +CMake to build LAMMPS with no add-on packages enabled and no +customization: .. code-block:: bash - cd lammps # change to the LAMMPS distribution directory - mkdir build; cd build # create a new directory (folder) for build - cmake [options ...] ../cmake # configuration with (command-line) cmake - cmake --build . # compilation (or type "make") + cd lammps # change to the LAMMPS distribution directory + mkdir build; cd build # create and use a build directory + cmake ../cmake # configuration reading CMake scripts from ../cmake + cmake --build . # compilation (or type "make") -The ``cmake`` command will detect available features, enable selected -packages and options, and will generate the build environment. By default -this build environment will be created for "Unix Makefiles" on most -platforms and particularly on Linux. However, alternate build tools -(e.g. Ninja) and project files for Integrated Development Environments -(IDEs) like Eclipse, CodeBlocks, or Kate can be generated, too. This is -selected via the ``-G`` command line flag. Further details about features -and settings for CMake are in the `CMake online documentation `_ +This will create and change into a folder called ``build``, then run the +configuration step to generate build files for the default build command +and then launch that build command to compile LAMMPS. During the +configuration step CMake will try to detect whether support for MPI, +OpenMP, FFTW, gzip, JPEG, PNG, and ffmpeg are available and enable the +corresponding configuration settings. The progress of this +configuration can be followed on the screen and a summary of selected +options and settings will be printed at the end. The ``cmake --build +.`` command will launch the compilation, which, if successful, will +ultimately produce a library ``liblammps.a`` and the LAMMPS executable +``lmp`` inside the ``build`` folder. -.. _cmake_doc: https://cmake.org/documentation/ +Compilation can take a long time, since LAMMPS is a large project with +many features. If your machine has multiple CPU cores (most do these +days), you can speed this up by compiling sources in parallel with +``make -j N`` (with N being the maximum number of concurrently executed +tasks). Also installation of the `ccache `_ (= +Compiler Cache) software may speed up repeated compilation even more, +e.g. during code development. -For the rest of the documentation -we will assume that the build environment is generated for "Unix Makefiles" -and thus the ``make`` command will be used to compile and link LAMMPS as -indicated above, producing (by default) an executable called ``lmp`` and -a library called ``liblammps.a`` in the ``build`` folder. - -If your machine has multiple CPU cores (most do these days), you can -compile sources in parallel with a command like ``make -j N`` (with N -being the maximum number of concurrently executed tasks). Also -installation of the ``ccache`` (= Compiler Cache) software may speed -up repeated compilation, e.g. during code development, significantly. +After the initial build, whenever you edit LAMMPS source files, enable +or disable packages, change compiler flags or build options, you must +re-compile and relink the LAMMPS executable with ``cmake --build .`` (or +``make``). If the compilation fails for some reason, try running +``cmake .`` and then compile again. The included dependency tracking +should make certain that only the necessary subset of files are +re-compiled. You can also delete compiled objects, libraries and +executables with ``cmake --build . clean`` (or ``make clean``). After compilation, you may optionally install the LAMMPS executable into your system with: .. code-block:: bash - make install # optional, copy LAMMPS executable & library elsewhere + make install # optional, copy compiled files into installation location -This will install the lammps executable and library (if requested), some -tools (if configured) and additional files like library API headers, -manpages, potential and force field files. The location of the installation -tree is set by the CMake variable "CMAKE_INSTALL_PREFIX" which defaults -to ${HOME}/.local +This will install the LAMMPS executable and library, some tools (if +configured) and additional files like LAMMPS API headers, manpages, +potential and force field files. The location of the installation tree +defaults to ``${HOME}/.local``. ----------- +.. _cmake_options: -.. _cmake_build: +Configuration and build options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are 3 variants of the CMake command itself: a command-line version -(``cmake`` or ``cmake3``), a text mode UI version (``ccmake`` or ``ccmake3``), -and a graphical GUI version (``cmake-gui``). You can use any of them -interchangeably to configure and create the LAMMPS build environment. -On Linux all the versions produce a Makefile as their output by default. -See more details on each below. +The CMake commands have one mandatory argument: a folder containing a +file called ``CMakeLists.txt`` (for LAMMPS it is located in the +``cmake`` folder) or a build folder containing a file called +``CMakeCache.txt``, which is generated at the end of the CMake +configuration step. The cache file contains all current CMake settings. -You can specify a variety of options with any of the 3 versions, which -affect how the build is performed and what is included in the LAMMPS -executable. Links to pages explaining all the options are listed on -the :doc:`Build ` doc page. +To modify settings, enable or disable features, you need to set *variables* +with either the *-D* command line flag (``-D VARIABLE1_NAME=value``) or +change them in the text mode of graphical user interface. The *-D* flag +can be used several times in one command. -You must perform the CMake build system generation and compilation in -a new directory you create. It can be anywhere on your local machine. -In these Build pages we assume that you are building in a directory -called ``lammps/build``. You can perform separate builds independently -with different options, so long as you perform each of them in a -separate directory you create. All the auxiliary files created by one -build process (executable, object files, log files, etc) are stored in -this directory or sub-directories within it that CMake creates. +For your convenience we provide :ref:`CMake presets ` +that combine multiple settings to enable optional LAMMPS packages or use +a different compiler tool chain. Those are loaded with the *-C* flag +(``-C ../cmake/presets/minimal.cmake``). This step would only be needed +once, as the settings from the preset files are stored in the +``CMakeCache.txt`` file. It is also possible to customize the build +by adding one or more *-D* flags to the CMake command line. -.. note:: +Generating files for alternate build tools (e.g. Ninja) and project files +for IDEs like Eclipse, CodeBlocks, or Kate can be selected using the *-G* +command line flag. A list of available generator settings for your +specific CMake version is given when running ``cmake --help``. - To perform a CMake build, no packages can be installed or a build - been previously attempted in the LAMMPS src directory by using ``make`` - commands to :doc:`perform a conventional LAMMPS build `. - CMake detects if this is the case and generates an error, telling you - to type ``make no-all purge`` in the src directory to un-install all - packages. The purge removes all the \*.h files auto-generated by - make. -You must have CMake version 3.10 or later on your system to build -LAMMPS. Installation instructions for CMake are below. - -After the initial build, if you edit LAMMPS source files, or add your -own new files to the source directory, you can just re-type make from -your build directory and it will re-compile only the files that have -changed. If you want to change CMake options you can run cmake (or -ccmake or cmake-gui) again from the same build directory and alter -various options; see details below. Or you can remove the entire build -folder, recreate the directory and start over. - ----------- - -**Command-line version of CMake**\ : - -.. code-block:: bash - - cmake [options ...] /path/to/lammps/cmake # build from any dir - cmake [options ...] ../cmake # build from lammps/build - cmake3 [options ...] ../cmake # build from lammps/build - -The cmake command takes one required argument, which is the LAMMPS -cmake directory which contains the CMakeLists.txt file. - -The argument can be prefixed or followed by various CMake -command-line options. Several useful ones are: - -.. code-block:: bash - - -D CMAKE_INSTALL_PREFIX=path # where to install LAMMPS executable/lib if desired - -D CMAKE_BUILD_TYPE=type # type = RelWithDebInfo (default), Release, MinSizeRel, or Debug - -G output # style of output CMake generates (e.g. "Unix Makefiles" or "Ninja") - -D CMAKE_MAKE_PROGRAM=builder # name of the builder executable (e.g. when using "gmake" instead of "make") - -DVARIABLE=value # setting for a LAMMPS feature to enable - -D VARIABLE=value # ditto, but cannot come after CMakeLists.txt dir - -C path/to/preset/file # load some CMake settings before configuring - -All the LAMMPS-specific -D variables that a LAMMPS build supports are -described on the pages linked to from the :doc:`Build ` doc page. -All of these variable names are upper-case and their values are -lower-case, e.g. -D LAMMPS_SIZES=smallbig. For boolean values, any of -these forms can be used: yes/no, on/off, 1/0. - -On Unix/Linux machines, CMake generates a Makefile by default to -perform the LAMMPS build. Alternate forms of build info can be -generated via the -G switch, e.g. Visual Studio on a Windows machine, -Xcode on MacOS, or KDevelop on Linux. Type ``cmake --help`` to see the -"Generator" styles of output your system supports. - -.. note:: - - When CMake runs, it prints configuration info to the screen. - You should review this to verify all the features you requested were - enabled, including packages. You can also see what compilers and - compile options will be used for the build. Any errors in CMake - variable syntax will also be flagged, e.g. mis-typed variable names or - variable values. - -CMake creates a CMakeCache.txt file when it runs. This stores all the -settings, so that when running CMake again you can use the current -folder '.' instead of the path to the LAMMPS cmake folder as the -required argument to the CMake command. Either way the existing -settings will be inherited unless the CMakeCache.txt file is removed. - -If you later want to change a setting you can rerun cmake in the build -directory with different setting. Please note that some automatically -detected variables will not change their value when you rerun cmake. -In these cases it is usually better to first remove all the -files/directories in the build directory, or start with a fresh build -directory. - ----------- - -**Curses version (terminal-style menu) of CMake**\ : - -.. code-block:: bash - - ccmake ../cmake - -You initiate the configuration and build environment generation steps -separately. For the first you have to type **c**\ , for the second you -have to type **g**\ . You may need to type **c** multiple times, and may be -required to edit some of the entries of CMake configuration variables -in between. Please see the `ccmake manual `_ for -more information. - ----------- - -**GUI version of CMake**\ : - -.. code-block:: bash - - cmake-gui ../cmake - -You initiate the configuration and build environment generation steps -separately. For the first you have to click on the **Configure** button, -for the second you have to click on the **Generate** button. You may -need to click on **Configure** multiple times, and may be required to -edit some of the entries of CMake configuration variables in between. -Please see the `cmake-gui manual `_ -for more information. - ----------- - -**Installing CMake** +Installing CMake +^^^^^^^^^^^^^^^^ Check if your machine already has CMake installed: @@ -216,11 +169,12 @@ software packages, do this: module list # is a module for cmake already loaded? module avail # is a module for cmake available? - module load cmake3 # load cmake module with appropriate name + module load cmake # load cmake module with appropriate name -Most Linux distributions offer pre-compiled cmake packages through -their package management system. If you do not have CMake or a new -enough version, you can download the latest version at -`https://cmake.org/download/ `_. -Instructions on how to install it on various platforms can be found -`on this page `_. +Most Linux distributions offer pre-compiled cmake packages through their +package management system. If you do not have CMake or a recent enough +version (Note: for CentOS 7.x you need to enable the EPEL repository), +you can download the latest version from `https://cmake.org/download/ +`_. Instructions on how to install it on +various platforms can be found `on this page +`_. diff --git a/doc/src/Build_development.rst b/doc/src/Build_development.rst index 21e99d69be..474e1fb833 100644 --- a/doc/src/Build_development.rst +++ b/doc/src/Build_development.rst @@ -1,15 +1,15 @@ Development build options (CMake only) ====================================== -The CMake build of LAMMPS has a few extra options which are useful during -development, testing or debugging. +The CMake build procedure of LAMMPS offers a few extra options which are +useful during development, testing or debugging. ---------- .. _compilation: Verify compilation flags ------------------------------------------- +------------------------ Sometimes it is necessary to verify the complete sequence of compilation flags generated by the CMake build. To enable a more verbose output during @@ -30,7 +30,7 @@ Another way of doing this without reconfiguration is calling make with variable .. _sanitizer: Address, Undefined Behavior, and Thread Sanitizer Support -------------------------------------------------------------------------- +--------------------------------------------------------- Compilers such as GCC and Clang support generating instrumented binaries which use different sanitizer libraries to detect problems in code @@ -41,10 +41,11 @@ during run-time. They can detect issues like: - `data races `_ Please note that this kind of instrumentation usually comes with a small -performance hit (much less than using tools like `Valgrind `_). -The to enable these features additional compiler flags need to be added -to the compilation and linking stages. This is most easily done through -setting the ``CMAKE_TUNE_FLAGS`` variable during configuration. Examples: +performance hit (much less than using tools like `Valgrind +`_). The to enable these features additional +compiler flags need to be added to the compilation and linking stages. +This is most easily done through setting the ``CMAKE_TUNE_FLAGS`` +variable during configuration. Examples: .. code-block:: bash @@ -52,8 +53,6 @@ setting the ``CMAKE_TUNE_FLAGS`` variable during configuration. Examples: -D CMAKE_TUNE_FLAGS=-fsanitize=undefined # enable undefined behavior sanitizer -D CMAKE_TUNE_FLAGS=-fsanitize=thread # enable thread sanitizer -.. _valgrind: https://valgrind.org - ---------- .. _testing: @@ -75,24 +74,26 @@ developers can run the tests directly on their workstation. -D LAMMPS_TESTING_SOURCE_DIR=path # path to lammps-testing repository (option if in custom location) -D LAMMPS_TESTING_GIT_TAG=value # version of lammps-testing repository that should be used, value = master (default) or custom git commit or tag -If you enable testing in the CMake build it will create an additional target called "test". You can run them with: +If you enable testing in the CMake build it will create an additional +target called "test". You can run them with: .. code-block:: bash - make test + cmake --build . test The test cases used come from the lammps-testing repository. They are -derivatives of the examples folder with some modifications to make the run -faster. +derivatives of the examples folder with some modifications to make the +run faster. -You can also collect code coverage metrics while running the tests by enabling -coverage support during building. +You can also collect code coverage metrics while running the tests by +enabling coverage support during building. .. code-block:: bash -D ENABLE_COVERAGE=value # enable coverage measurements, value = no (default) or yes -This will also add the following targets to generate coverage reports after running the LAMMPS executable: +This will also add the following targets to generate coverage reports +after running the LAMMPS executable: .. code-block:: bash @@ -100,7 +101,8 @@ This will also add the following targets to generate coverage reports after runn make gen_coverage_html # generate coverage report in HTML format make gen_coverage_xml # generate coverage report in XML format -These reports require GCOVR to be installed. The easiest way to do this to install it via pip: +These reports require GCOVR to be installed. The easiest way to do this +to install it via pip: .. code-block:: bash diff --git a/doc/src/Build_extras.rst b/doc/src/Build_extras.rst index 4589015e35..3d8da39640 100644 --- a/doc/src/Build_extras.rst +++ b/doc/src/Build_extras.rst @@ -87,27 +87,33 @@ GPU package --------------------- To build with this package, you must choose options for precision and -which GPU hardware to build for. +which GPU hardware to build for. The GPU package currently supports +three different types of backends: OpenCL, CUDA and HIP. **CMake build**\ : .. code-block:: bash - -D GPU_API=value # value = opencl (default) or cuda - -D GPU_PREC=value # precision setting - # value = double or mixed (default) or single - -D OCL_TUNE=value # hardware choice for GPU_API=opencl - # generic (default) or intel (Intel CPU) or fermi, kepler, cypress (NVIDIA) - -D GPU_ARCH=value # primary GPU hardware choice for GPU_API=cuda - # value = sm_XX, see below - # default is sm_30 - -D CUDPP_OPT=value # optimization setting for GPU_API=cuda - # enables CUDA Performance Primitives Optimizations - # value = yes (default) or no - -D CUDA_MPS_SUPPORT=value # enables some tweaks required to run with active nvidia-cuda-mps daemon - # value = yes or no (default) + -D GPU_API=value # value = opencl (default) or cuda or hip + -D GPU_PREC=value # precision setting + # value = double or mixed (default) or single + -D OCL_TUNE=value # hardware choice for GPU_API=opencl + # generic (default) or intel (Intel CPU) or fermi, kepler, cypress (NVIDIA) + -D GPU_ARCH=value # primary GPU hardware choice for GPU_API=cuda + # value = sm_XX, see below + # default is sm_30 + -D HIP_ARCH=value # primary GPU hardware choice for GPU_API=hip + # value depends on selected HIP_PLATFORM + # default is 'gfx906' for HIP_PLATFORM=hcc and 'sm_30' for HIP_PLATFORM=nvcc + -D HIP_USE_DEVICE_SORT=value # enables GPU sorting + # value = yes (default) or no + -D CUDPP_OPT=value # optimization setting for GPU_API=cuda + # enables CUDA Performance Primitives Optimizations + # value = yes (default) or no + -D CUDA_MPS_SUPPORT=value # enables some tweaks required to run with active nvidia-cuda-mps daemon + # value = yes or no (default) -GPU_ARCH settings for different GPU hardware is as follows: +:code:`GPU_ARCH` settings for different GPU hardware is as follows: * sm_12 or sm_13 for GT200 (supported by CUDA 3.2 until CUDA 6.5) * sm_20 or sm_21 for Fermi (supported by CUDA 3.2 until CUDA 7.5) @@ -126,6 +132,28 @@ Thus the GPU_ARCH setting is merely an optimization, to have code for the preferred GPU architecture directly included rather than having to wait for the JIT compiler of the CUDA driver to translate it. +If you are compiling with HIP, note that before running CMake you will have to +set appropriate environment variables. Some variables such as +:code:`HCC_AMDGPU_TARGET` or :code:`CUDA_PATH` are necessary for :code:`hipcc` +and the linker to work correctly. + +.. code:: bash + + # AMDGPU target + export HIP_PLATFORM=hcc + export HCC_AMDGPU_TARGET=gfx906 + cmake -D PKG_GPU=on -D GPU_API=HIP -D HIP_ARCH=gfx906 -D CMAKE_CXX_COMPILER=hipcc .. + make -j 4 + +.. code:: bash + + # CUDA target + # !!! DO NOT set CMAKE_CXX_COMPILER !!! + export HIP_PLATFORM=nvcc + export CUDA_PATH=/usr/local/cuda + cmake -D PKG_GPU=on -D GPU_API=HIP -D HIP_ARCH=sm_70 .. + make -j 4 + **Traditional make**\ : Before building LAMMPS, you must build the GPU library in ``lib/gpu``\ . @@ -189,13 +217,27 @@ KIM package --------------------- To build with this package, the KIM library with API v2 must be downloaded -and built on your system. It must include the KIM models that you want to -use with LAMMPS. If you want to use the :doc:`kim_query ` +and built on your system. It must include the KIM models that you want to +use with LAMMPS. + +If you would like to use the :doc:`kim_query ` command, you also need to have libcurl installed with the matching development headers and the curl-config tool. -See the `Obtaining KIM Models `_ -web page to +If you would like to use the :doc:`kim_property ` +command, you need to build LAMMPS with the Python 3.6 or later package +installed. See the :doc:`Python ` doc page for more info on building +LAMMPS with the version of Python on your system. +After successfully building LAMMPS with Python, you need to +install the kim-property Python package, which can be easily done using +*pip* as ``pip install kim-property``, or from the *conda-forge* channel as +``conda install kim-property`` if LAMMPS is built in Conda. More detailed +information is available at: +`kim-property installation `_. + +In addition to installing the KIM API, it is also necessary to install the +library of KIM models (interatomic potentials). +See `Obtaining KIM Models `_ to learn how to install a pre-build binary of the OpenKIM Repository of Models. See the list of all KIM models here: https://openkim.org/browse/models @@ -265,95 +307,209 @@ using. For example: .. _kokkos: KOKKOS package ---------------------------- +-------------- -To build with this package, you must choose which hardware you want to -build for, either CPUs (multi-threading via OpenMP) or KNLs (OpenMP) -or GPUs (NVIDIA Cuda). +Using the KOKKOS package requires choosing several settings. You have +to select whether you want to compile with parallelization on the host +and whether you want to include offloading of calculations to a device +(e.g. a GPU). The default setting is to have no host parallelization +and no device offloading. In addition, you can select the hardware +architecture to select the instruction set. Since most hardware is +backward compatible, you may choose settings for an older architecture +to have an executable that will run on this and newer architectures. -For a CMake or make build, these are the possible choices for the -``KOKKOS_ARCH`` settings described below. Note that for CMake, these are -really Kokkos variables, not LAMMPS variables. Hence you must use -case-sensitive values, e.g. BDW, not bdw. +.. note:: -* AMDAVX = AMD 64-bit x86 CPUs -* EPYC = AMD EPYC Zen class CPUs -* ARMv80 = ARMv8.0 Compatible CPU -* ARMv81 = ARMv8.1 Compatible CPU -* ARMv8-ThunderX = ARMv8 Cavium ThunderX CPU -* ARMv8-TX2 = ARMv8 Cavium ThunderX2 CPU -* WSM = Intel Westmere CPUs -* SNB = Intel Sandy/Ivy Bridge CPUs -* HSW = Intel Haswell CPUs -* BDW = Intel Broadwell Xeon E-class CPUs -* SKX = Intel Sky Lake Xeon E-class HPC CPUs (AVX512) -* KNC = Intel Knights Corner Xeon Phi -* KNL = Intel Knights Landing Xeon Phi -* BGQ = IBM Blue Gene/Q CPUs -* Power7 = IBM POWER8 CPUs -* Power8 = IBM POWER8 CPUs -* Power9 = IBM POWER9 CPUs -* Kepler = NVIDIA Kepler default (generation CC 3.5) -* Kepler30 = NVIDIA Kepler generation CC 3.0 -* Kepler32 = NVIDIA Kepler generation CC 3.2 -* Kepler35 = NVIDIA Kepler generation CC 3.5 -* Kepler37 = NVIDIA Kepler generation CC 3.7 -* Maxwell = NVIDIA Maxwell default (generation CC 5.0) -* Maxwell50 = NVIDIA Maxwell generation CC 5.0 -* Maxwell52 = NVIDIA Maxwell generation CC 5.2 -* Maxwell53 = NVIDIA Maxwell generation CC 5.3 -* Pascal60 = NVIDIA Pascal generation CC 6.0 -* Pascal61 = NVIDIA Pascal generation CC 6.1 -* Volta70 = NVIDIA Volta generation CC 7.0 -* Volta72 = NVIDIA Volta generation CC 7.2 -* Turing75 = NVIDIA Turing generation CC 7.5 + If you run Kokkos on a different GPU architecture than what LAMMPS + was compiled with, there will be a delay during device initialization + while the just-in-time compiler is recompiling all GPU kernels for + the new hardware. This is, however, only supported for GPUs of the + **same** major hardware version and different minor hardware versions, + e.g. 5.0 and 5.2 but not 5.2 and 6.0. LAMMPS will abort with an + error message indicating a mismatch, if that happens. -**CMake build**\ : +The settings discussed below have been tested with LAMMPS and are +confirmed to work. Kokkos is an active project with ongoing improvements +and projects working on including support for additional architectures. +More information on Kokkos can be found on the +`Kokkos GitHub project `_. +Available Architecture settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These are the possible choices for the Kokkos architecture ID. They must +be specified in uppercase. + +.. list-table:: + :header-rows: 0 + :widths: auto + + * - **Arch-ID** + - **HOST or GPU** + - **Description** + * - AMDAVX + - HOST + - AMD 64-bit x86 CPU (AVX 1) + * - EPYC + - HOST + - AMD EPYC Zen class CPU (AVX 2) + * - ARMV80 + - HOST + - ARMv8.0 Compatible CPU + * - ARMV81 + - HOST + - ARMv8.1 Compatible CPU + * - ARMV8_THUNDERX + - HOST + - ARMv8 Cavium ThunderX CPU + * - ARMV8_THUNDERX2 + - HOST + - ARMv8 Cavium ThunderX2 CPU + * - WSM + - HOST + - Intel Westmere CPU (SSE 4.2) + * - SNB + - HOST + - Intel Sandy/Ivy Bridge CPU (AVX 1) + * - HSW + - HOST + - Intel Haswell CPU (AVX 2) + * - BDW + - HOST + - Intel Broadwell Xeon E-class CPU (AVX 2 + transactional mem) + * - SKX + - HOST + - Intel Sky Lake Xeon E-class HPC CPU (AVX512 + transactional mem) + * - KNC + - HOST + - Intel Knights Corner Xeon Phi + * - KNL + - HOST + - Intel Knights Landing Xeon Phi + * - BGQ + - HOST + - IBM Blue Gene/Q CPU + * - POWER7 + - HOST + - IBM POWER7 CPU + * - POWER8 + - HOST + - IBM POWER8 CPU + * - POWER9 + - HOST + - IBM POWER9 CPU + * - KEPLER30 + - GPU + - NVIDIA Kepler generation CC 3.0 GPU + * - KEPLER32 + - GPU + - NVIDIA Kepler generation CC 3.2 GPU + * - KEPLER35 + - GPU + - NVIDIA Kepler generation CC 3.5 GPU + * - KEPLER37 + - GPU + - NVIDIA Kepler generation CC 3.7 GPU + * - MAXWELL50 + - GPU + - NVIDIA Maxwell generation CC 5.0 GPU + * - MAXWELL52 + - GPU + - NVIDIA Maxwell generation CC 5.2 GPU + * - MAXWELL53 + - GPU + - NVIDIA Maxwell generation CC 5.3 GPU + * - PASCAL60 + - GPU + - NVIDIA Pascal generation CC 6.0 GPU + * - PASCAL61 + - GPU + - NVIDIA Pascal generation CC 6.1 GPU + * - VOLTA70 + - GPU + - NVIDIA Volta generation CC 7.0 GPU + * - VOLTA72 + - GPU + - NVIDIA Volta generation CC 7.2 GPU + * - TURING75 + - GPU + - NVIDIA Turing generation CC 7.5 GPU + * - VEGA900 + - GPU + - AMD GPU MI25 GFX900 + * - VEGA906 + - GPU + - AMD GPU MI50/MI60 GFX906 + +Basic CMake build settings: +^^^^^^^^^^^^^^^^^^^^^^^^^^^ For multicore CPUs using OpenMP, set these 2 variables. .. code-block:: bash - -D KOKKOS_ARCH=archCPU # archCPU = CPU from list above - -D KOKKOS_ENABLE_OPENMP=yes + -D Kokkos_ARCH_HOSTARCH=yes # HOSTARCH = HOST from list above + -D Kokkos_ENABLE_OPENMP=yes + -D BUILD_OMP=yes -For Intel KNLs using OpenMP, set these 2 variables: +Please note that enabling OpenMP for KOKKOS requires that OpenMP is +also :ref:`enabled for the rest of LAMMPS `. + +For Intel KNLs using OpenMP, set these variables: .. code-block:: bash - -D KOKKOS_ARCH=KNL - -D KOKKOS_ENABLE_OPENMP=yes + -D Kokkos_ARCH_KNL=yes + -D Kokkos_ENABLE_OPENMP=yes -For NVIDIA GPUs using CUDA, set these 4 variables: +For NVIDIA GPUs using CUDA, set these variables: .. code-block:: bash - -D KOKKOS_ARCH="archCPU;archGPU" # archCPU = CPU from list above that is hosting the GPU - # archGPU = GPU from list above - -D KOKKOS_ENABLE_CUDA=yes - -D KOKKOS_ENABLE_OPENMP=yes - -D CMAKE_CXX_COMPILER=wrapper # wrapper = full path to Cuda nvcc wrapper + -D Kokkos_ARCH_HOSTARCH=yes # HOSTARCH = HOST from list above + -D Kokkos_ARCH_GPUARCH=yes # GPUARCH = GPU from list above + -D Kokkos_ENABLE_CUDA=yes + -D Kokkos_ENABLE_OPENMP=yes + -D CMAKE_CXX_COMPILER=wrapper # wrapper = full path to Cuda nvcc wrapper -The wrapper value is the Cuda nvcc compiler wrapper provided in the -Kokkos library: ``lib/kokkos/bin/nvcc_wrapper``\ . The setting should -include the full path name to the wrapper, e.g. +This will also enable executing FFTs on the GPU, either via the internal +KISSFFT library, or - by preference - with the cuFFT library bundled +with the CUDA toolkit, depending on whether CMake can identify its +location. The *wrapper* value for ``CMAKE_CXX_COMPILER`` variable is +the path to the CUDA nvcc compiler wrapper provided in the Kokkos +library: ``lib/kokkos/bin/nvcc_wrapper``\ . The setting should include +the full path name to the wrapper, e.g. .. code-block:: bash - -D CMAKE_CXX_COMPILER=/home/username/lammps/lib/kokkos/bin/nvcc_wrapper + -D CMAKE_CXX_COMPILER=${HOME}/lammps/lib/kokkos/bin/nvcc_wrapper -**Traditional make**\ : +To simplify the compilation, three preset files are included in the +``cmake/presets`` folder, ``kokkos-serial.cmake``, ``kokkos-openmp.cmake``, +and ``kokkos-cuda.cmake``. They will enable the KOKKOS package and +enable some hardware choice. So to compile with OpenMP host parallelization, +CUDA device parallelization (for GPUs with CC 5.0 and up) with some +common packages enabled, you can do the following: + +.. code-block:: bash + + mkdir build-kokkos-cuda + cd build-kokkos-cuda + cmake -C ../cmake/presets/minimal.cmake -C ../cmake/presets/kokkos-cuda.cmake ../cmake + cmake --build . + +Basic traditional make settings: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Choose which hardware to support in ``Makefile.machine`` via ``KOKKOS_DEVICES`` and ``KOKKOS_ARCH`` settings. See the -``src/MAKE/OPTIONS/Makefile.kokkos\*`` files for examples. +``src/MAKE/OPTIONS/Makefile.kokkos*`` files for examples. For multicore CPUs using OpenMP: .. code-block:: make KOKKOS_DEVICES = OpenMP - KOKKOS_ARCH = archCPU # archCPU = CPU from list above + KOKKOS_ARCH = HOSTARCH # HOSTARCH = HOST from list above For Intel KNLs using OpenMP: @@ -367,22 +523,72 @@ For NVIDIA GPUs using CUDA: .. code-block:: make KOKKOS_DEVICES = Cuda - KOKKOS_ARCH = archCPU,archGPU # archCPU = CPU from list above that is hosting the GPU - # archGPU = GPU from list above - FFT_INC = -DFFT_CUFFT # enable use of cuFFT (optional) - FFT_LIB = -lcufft # link to cuFFT library + KOKKOS_ARCH = HOSTARCH,GPUARCH # HOSTARCH = HOST from list above that is hosting the GPU + KOKKOS_CUDA_OPTIONS = "enable_lambda" + # GPUARCH = GPU from list above + FFT_INC = -DFFT_CUFFT # enable use of cuFFT (optional) + FFT_LIB = -lcufft # link to cuFFT library -For GPUs, you also need the following 2 lines in your Makefile.machine -before the CC line is defined, in this case for use with OpenMPI mpicxx. -The 2 lines define a nvcc wrapper compiler, which will use nvcc for -compiling CUDA files and use a C++ compiler for non-Kokkos, non-CUDA -files. +For GPUs, you also need the following lines in your ``Makefile.machine`` +before the CC line is defined. They tell ``mpicxx`` to use an ``nvcc`` +compiler wrapper, which will use ``nvcc`` for compiling CUDA files and a +C++ compiler for non-Kokkos, non-CUDA files. .. code-block:: make + # For OpenMPI KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) export OMPI_CXX = $(KOKKOS_ABSOLUTE_PATH)/config/nvcc_wrapper - CC = mpicxx + CC = mpicxx + +.. code-block:: make + + # For MPICH and derivatives + KOKKOS_ABSOLUTE_PATH = $(shell cd $(KOKKOS_PATH); pwd) + CC = mpicxx -cxx=$(KOKKOS_ABSOLUTE_PATH)/config/nvcc_wrapper + + +Advanced KOKKOS compilation settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are other allowed options when building with the KOKKOS package +that can improve performance or assist in debugging or profiling. Below +are some examples that may be useful in combination with LAMMPS. For +the full list (which keeps changing as the Kokkos package itself evolves), +please consult the Kokkos library documentation. + +As alternative to using multi-threading via OpenMP +(``-DKokkos_ENABLE_OPENMP=on`` or ``KOKKOS_DEVICES=OpenMP``) it is also +possible to use Posix threads directly (``-DKokkos_ENABLE_PTHREAD=on`` +or ``KOKKOS_DEVICES=Pthread``). While binding of threads to individual +or groups of CPU cores is managed in OpenMP with environment variables, +you need assistance from either the "hwloc" or "libnuma" library for the +Pthread thread parallelization option. To enable use with CMake: +``-DKokkos_ENABLE_HWLOC=on`` or ``-DKokkos_ENABLE_LIBNUMA=on``; and with +conventional make: ``KOKKOS_USE_TPLS=hwloc`` or +``KOKKOS_USE_TPLS=libnuma``. + +The CMake option ``-DKokkos_ENABLE_LIBRT=on`` or the makefile setting +``KOKKOS_USE_TPLS=librt`` enables the use of a more accurate timer +mechanism on many Unix-like platforms for internal profiling. + +The CMake option ``-DKokkos_ENABLE_DEBUG=on`` or the makefile setting +``KOKKOS_DEBUG=yes`` enables printing of run-time +debugging information that can be useful. It also enables runtime +bounds checking on Kokkos data structures. As to be expected, enabling +this option will negatively impact the performance and thus is only +recommended when developing a Kokkos-enabled style in LAMMPS. + +The CMake option ``-DKokkos_ENABLE_CUDA_UVM=on`` or the makefile +setting ``KOKKOS_CUDA_OPTIONS=enable_lambda,force_uvm`` enables the +use of CUDA "Unified Virtual Memory" (UVM) in Kokkos. UVM allows to +transparently use RAM on the host to supplement the memory used on the +GPU (with some performance penalty) and thus enables running larger +problems that would otherwise not fit into the RAM on the GPU. + +Please note, that the LAMMPS KOKKOS package must **always** be compiled +with the *enable_lambda* option when using GPUs. The CMake configuration +will thus always enable it. ---------- @@ -421,7 +627,7 @@ args: $ make lib-latte args="-b" # download and build in lib/latte/LATTE-master $ make lib-latte args="-p $HOME/latte" # use existing LATTE installation in $HOME/latte $ make lib-latte args="-b -m gfortran" # download and build in lib/latte and - # copy Makefile.lammps.gfortran to Makefile.lammps + # copy Makefile.lammps.gfortran to Makefile.lammps Note that 3 symbolic (soft) links, "includelink" and "liblink" and "filelink.o", are created in lib/latte to point into the LATTE home diff --git a/doc/src/Build_link.rst b/doc/src/Build_link.rst index b18dc49e38..39fc4fe40a 100644 --- a/doc/src/Build_link.rst +++ b/doc/src/Build_link.rst @@ -1,33 +1,35 @@ Link LAMMPS as a library to another code ======================================== -LAMMPS can be used as a library by another application, including -Python scripts. The files src/library.cpp and library.h define the +LAMMPS is designed as a library of C++ objects and can thus be +integrated into other applications including Python scripts. +The files ``src/library.cpp`` and ``src/library.h`` define a C-style API for using LAMMPS as a library. See the :doc:`Howto library ` doc page for a description of the interface and how to extend it for your needs. The :doc:`Build basics ` doc page explains how to build -LAMMPS as either a shared or static library. This results in one of -these 2 files: - -.. code-block:: bash - - liblammps.so # shared library - liblammps.a # static library +LAMMPS as either a shared or static library. This results in a file +in the compilation folder called ``liblammps.a`` or ``liblammps_.a`` +in case of building a static library. In case of a shared library +the name is the same only that the suffix is going to be either ``.so`` +or ``.dylib`` or ``.dll`` instead of ``.a`` depending on the OS. +In some cases the ``.so`` file may be a symbolic link to a file with +the suffix ``.so.0`` (or some other number). .. note:: - Care should be taken to use the same MPI library for the calling - code and the LAMMPS library. The library.h file includes mpi.h and - uses definitions from it so those need to be available and - consistent. When LAMMPS is compiled with the MPI STUBS library, - then its mpi.h file needs to be included. While it is technically - possible to use a full MPI library in the calling code and link to - a serial LAMMPS library compiled with MPI STUBS, it is recommended - to use the *same* MPI library for both, and then use MPI_Comm_split() - in the calling code to pass a suitable communicator with a subset - of MPI ranks to the function creating the LAMMPS instance. + Care should be taken to use the same MPI library for the calling code + and the LAMMPS library. The ``library.h`` file includes ``mpi.h`` + and uses definitions from it so those need to be available and + consistent. When LAMMPS is compiled with the included STUBS MPI + library, then its ``mpi.h`` file needs to be included. While it is + technically possible to use a full MPI library in the calling code + and link to a serial LAMMPS library compiled with MPI STUBS, it is + recommended to use the *same* MPI library for both, and then use + ``MPI_Comm_split()`` in the calling code to pass a suitable + communicator with a subset of MPI ranks to the function creating the + LAMMPS instance. ---------- @@ -42,11 +44,11 @@ executable code from the library is copied into the calling executable. *CMake build*\ : -This assumes that LAMMPS has been configured with "-D BUILD_LIB=yes" -and installed with "make install" and the PKG_CONFIG_PATH environment -variable updated to include the *liblammps.pc* file installed into the -configured destination folder, if needed. The commands to compile and -link the coupled executable are then: +This assumes that LAMMPS has been configured without setting a +``LAMMPS_MACHINE`` name, installed with "make install", and the +``PKG_CONFIG_PATH`` environment variable has been updated to include the +``liblammps.pc`` file installed into the configured destination folder. +The commands to compile and link a coupled executable are then: .. code-block:: bash @@ -56,30 +58,35 @@ link the coupled executable are then: *Traditional make*\ : This assumes that LAMMPS has been compiled in the folder -"${HOME}/lammps/src" with "make mode=lib mpi". The commands to compile -and link the coupled executable are then: +``${HOME}/lammps/src`` with "make mpi". The commands to compile and link +a coupled executable are then: .. code-block:: bash mpicc -c -O -I${HOME}/lammps/src caller.c - mpicxx -o caller caller.o -L${HOME}/lammps/src -llammps + mpicxx -o caller caller.o -L${HOME}/lammps/src -llammps_mpi -The *-I* argument is the path to the location of the *library.h* +The *-I* argument is the path to the location of the ``library.h`` header file containing the interface to the LAMMPS C-style library -interface. The *-L* argument is the path to where the *liblammps.a* -file is located. The *-llammps* argument is shorthand for telling the -compiler to link the file *liblammps.a*\ . +interface. The *-L* argument is the path to where the ``liblammps_mpi.a`` +file is located. The *-llammps_mpi* argument is shorthand for telling the +compiler to link the file ``liblammps_mpi.a``. If LAMMPS has been +built as a shared library, then the linker will use ``liblammps_mpi.so`` +instead. If both files are available, the linker will usually prefer +the shared library. In case of a shared library, you may need to update +the ``LD_LIBRARY_PATH`` environment variable or running the ``caller`` +executable will fail since it cannot find the shared library at runtime. However, it is only as simple as shown above for the case of a plain LAMMPS library without any optional packages that depend on libraries -(bundled or external). Otherwise, you need to include all flags, -libraries, and paths for the coupled executable, that are also -required to link the LAMMPS executable. +(bundled or external) or when using a shared library. Otherwise, you +need to include all flags, libraries, and paths for the coupled +executable, that are also required to link the LAMMPS executable. *CMake build*\ : When using CMake, additional libraries with sources in the lib folder -are built, but not included in liblammps.a and (currently) not +are built, but not included in ``liblammps.a`` and (currently) not installed with "make install" and not included in the *pkgconfig* configuration file. They can be found in the top level build folder, but you have to determine the necessary link flags manually. It is @@ -87,23 +94,26 @@ therefore recommended to either use the traditional make procedure to build and link with a static library or build and link with a shared library instead. +.. TODO: this needs to be updated to reflect that latest CMake changes after they are complete. + *Traditional make*\ : After you have compiled a static LAMMPS library using the conventional -build system for example with "make mode=lib serial". And you also -have installed the POEMS package after building its bundled library in -lib/poems. Then the commands to build and link the coupled executable +build system for example with "make mode=static serial". And you also +have installed the ``POEMS`` package after building its bundled library +in ``lib/poems``. Then the commands to build and link the coupled executable change to: .. code-block:: bash gcc -c -O -I${HOME}/lammps/src/STUBS -I${HOME}/lammps/src -caller.c g++ -o caller caller.o -L${HOME}/lammps/lib/poems \ - -L${HOME}/lammps/src/STUBS -L${HOME}/lammps/src -llammps -lpoems -lmpi_stubs + -L${HOME}/lammps/src/STUBS -L${HOME}/lammps/src -llammps_serial -lpoems -lmpi_stubs -Note, that you need to link with "g++" instead of "gcc", since LAMMPS -is C++ code. You can display the currently applied settings for building -LAMMPS for the "serial" machine target by using the command: +Note, that you need to link with "g++" instead of "gcc", since the +LAMMPS library is C++ code. You can display the currently applied +settings for building LAMMPS for the "serial" machine target by using +the command: .. code-block:: bash @@ -113,16 +123,16 @@ Which should output something like: .. code-block:: bash - # Compiler: + # Compiler: CXX=g++ - # Linker: + # Linker: LD=g++ - # Compilation: - CXXFLAGS=-g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I${HOME}/lammps/lib/poems -I${HOME}/lammps/src/STUBS - # Linking: + # Compilation: + CXXFLAGS=-g -O3 -DLAMMPS_GZIP -DLAMMPS_MEMALIGN=64 -I${HOME}/compile/lammps/lib/poems -I${HOME}/compile/lammps/src/STUBS + # Linking: LDFLAGS=-g -O - # Libraries: - LDLIBS=-L${HOME}/lammps/lib/poems -L${HOME}/lammps/src/STUBS -lpoems -lmpi_stubs + # Libraries: + LDLIBS=-L${HOME}/compile/lammps/src -llammps_serial -L${HOME}/compile/lammps/lib/poems -L${HOME}/compile/lammps/src/STUBS -lpoems -lmpi_stubs From this you can gather the necessary paths and flags. With makefiles for other *machine* configurations you need to do the @@ -133,14 +143,13 @@ of the makefile. **Link with LAMMPS as a shared library**\ : -When linking to LAMMPS built as a shared library, the situation -becomes much simpler, as all dependent libraries and objects are -included in the shared library, which is - technically speaking - -effectively a regular LAMMPS executable that is missing the `main()` -function. Thus those libraries need not to be specified when linking -the calling executable. Only the *-I* flags are needed. So the -example case from above of the serial version static LAMMPS library -with the POEMS package installed becomes: +When linking to LAMMPS built as a shared library, the situation becomes +much simpler, as all dependent libraries and objects are either included +in the shared library or registered as a dependent library in the shared +library file. Thus those libraries need not to be specified when +linking the calling executable. Only the *-I* flags are needed. So the +example case from above of the serial version static LAMMPS library with +the POEMS package installed becomes: *CMake build*\ : @@ -155,19 +164,19 @@ build process are the same as for the static library. *Traditional make*\ : The commands with a shared LAMMPS library compiled with the -traditional make build using "make mode=shlib serial" becomes: +traditional make build using "make mode=shared serial" becomes: .. code-block:: bash gcc -c -O -I${HOME}/lammps/src/STUBS -I${HOME}/lammps/src -caller.c - g++ -o caller caller.o -L${HOME}/lammps/src -llammps + g++ -o caller caller.o -L${HOME}/lammps/src -llammps_serial *Locating liblammps.so at runtime*\ : -However, now the `liblammps.so` file is required at runtime and needs +However, now the ``liblammps.so`` file is required at runtime and needs to be in a folder, where the shared linker program of the operating -system can find it. This would be either a folder like "/usr/local/lib64" -or "${HOME}/.local/lib64" or a folder pointed to by the LD_LIBRARY_PATH +system can find it. This would be either a folder like ``/usr/local/lib64`` +or ``${HOME}/.local/lib64`` or a folder pointed to by the ``LD_LIBRARY_PATH`` environment variable. You can type .. code-block:: bash @@ -177,11 +186,11 @@ environment variable. You can type to see what directories are in that list. Or you can add the LAMMPS src directory (or the directory you performed -a CMake style build in) to your LD_LIBRARY_PATH, so that the current +a CMake style build in) to your ``LD_LIBRARY_PATH``, so that the current version of the shared library is always available to programs that use it. For the Bourne or Korn shells (/bin/sh, /bin/ksh, /bin/bash etc.), you -would add something like this to your ~/.profile file: +would add something like this to your ``${HOME}/.profile`` file: .. code-block:: bash @@ -189,14 +198,14 @@ would add something like this to your ~/.profile file: export LD_LIBRARY_PATH For the csh or tcsh shells, you would equivalently add something like this -to your ~/.cshrc file: +to your ``${HOME}/.cshrc`` file: .. code-block:: csh setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${HOME}/lammps/src You can verify whether all required shared libraries are found with the -`ldd` tool. Example: +``ldd`` tool. Example: .. code-block:: bash @@ -229,20 +238,20 @@ If a required library is missing, you would get a 'not found' entry: Either flavor of library (static or shared) allows one or more LAMMPS objects to be instantiated from the calling program. When used from a C++ program, most of the symbols and functions in LAMMPS are wrapped -in a LAMMPS_NS namespace; you can safely use any of its classes and +in a ``LAMMPS_NS`` namespace; you can safely use any of its classes and methods from within the calling code, as needed, and you will not incur conflicts with functions and variables in your code that share the name. This, however, does not extend to all additional libraries bundled with LAMMPS in the lib folder and some of the low-level code of some packages. To be compatible with C, Fortran, Python programs, the library has a simple -C-style interface, provided in src/library.cpp and src/library.h. +C-style interface, provided in ``src/library.cpp`` and ``src/library.h``. See the :doc:`Python library ` doc page for a description of the Python interface to LAMMPS, which wraps the C-style interface from a shared library through the `ctypes python module `_. -See the sample codes in examples/COUPLE/simple for examples of C++ and +See the sample codes in ``examples/COUPLE/simple`` for examples of C++ and C and Fortran codes that invoke LAMMPS through its library interface. Other examples in the COUPLE directory use coupling ideas discussed on the :doc:`Howto couple ` doc page. diff --git a/doc/src/Build_make.rst b/doc/src/Build_make.rst index 0379a8379a..af5e319121 100644 --- a/doc/src/Build_make.rst +++ b/doc/src/Build_make.rst @@ -2,10 +2,14 @@ Build LAMMPS with make ====================== Building LAMMPS with traditional makefiles requires that you have a -Makefile."machine" file appropriate for your system in the src/MAKE, -src/MAKE/MACHINES, src/MAKE/OPTIONS, or src/MAKE/MINE directory (see -below). It can include various options for customizing your LAMMPS -build with a number of global compilation options and features. +``Makefile.`` file appropriate for your system in either the +``src/MAKE``, ``src/MAKE/MACHINES``, ``src/MAKE/OPTIONS``, or +``src/MAKE/MINE`` directory (see below). It can include various options +for customizing your LAMMPS build with a number of global compilation +options and features. + +Requirements +^^^^^^^^^^^^ Those makefiles are written for and tested with GNU make and may not be compatible with other make programs. In most cases, if the "make" @@ -16,44 +20,54 @@ with :doc:`CMake `. The makefiles of the traditional make based build process and the scripts they are calling expect a few additional tools to be available and functioning. - * a Bourne shell compatible "Unix" shell program (often this is bash) - * a few shell utilities: ls, mv, ln, rm, grep, sed, tr, cat, touch, diff, dirname - * python (optional, required for "make lib-XXX" in the src folder) + * a working C/C++ compiler toolchain supporting the C++11 standard; on + Linux these are often the GNU compilers. Some older compilers + require adding flags like ``-std=c++11`` to enable the C++11 mode. + * a Bourne shell compatible "Unix" shell program (often this is ``bash``) + * a few shell utilities: ``ls``, ``mv``, ``ln``, ``rm``, ``grep``, ``sed``, ``tr``, ``cat``, ``touch``, ``diff``, ``dirname`` + * python (optional, required for ``make lib-`` in the src folder). + python scripts are currently tested with python 2.7 and 3.6. The procedure + for :doc:`building the documentation ` requires python 3. -To include LAMMPS packages (i.e. optional commands and styles) you -must enable them first, as discussed on the :doc:`Build package -` doc page. If a packages requires (provided or +Getting started +^^^^^^^^^^^^^^^ + +To include LAMMPS packages (i.e. optional commands and styles) you must +enable (or "install") them first, as discussed on the :doc:`Build +package ` doc page. If a packages requires (provided or external) libraries, you must configure and build those libraries **before** building LAMMPS itself and especially **before** enabling -such a package with "make yes-". Building :doc:`LAMMPS -with CMake ` can automate much of this for many types of +such a package with ``make yes-``. Building :doc:`LAMMPS with +CMake ` can automate much of this for many types of machines, especially workstations, desktops, and laptops, so we suggest you try it first when building LAMMPS in those cases. The commands below perform a default LAMMPS build, producing the LAMMPS -executable lmp_serial and lmp_mpi in lammps/src: +executable ``lmp_serial`` and ``lmp_mpi`` in ``lammps/src``: .. code-block:: bash - cd lammps/src - make serial # build a serial LAMMPS executable + cd lammps/src # change to main LAMMPS source folder + make serial # build a serial LAMMPS executable using GNU g++ make mpi # build a parallel LAMMPS executable with MPI make # see a variety of make options -This initial compilation can take a long time, since LAMMPS is a large -project with many features. If your machine has multiple CPU cores -(most do these days), using a command like "make -jN mpi" (with N = -the number of available CPU cores) can be much faster. If you plan to -do development on LAMMPS or need to re-compile LAMMPS repeatedly, the -installation of the ccache (= Compiler Cache) software may speed up -compilation even more. +Compilation can take a long time, since LAMMPS is a large project with +many features. If your machine has multiple CPU cores (most do these +days), you can speed this up by compiling sources in parallel with +``make -j N`` (with N being the maximum number of concurrently executed +tasks). Also installation of the `ccache `_ (= +Compiler Cache) software may speed up repeated compilation even more, +e.g. during code development. After the initial build, whenever you edit LAMMPS source files, or add or remove new files to the source directory (e.g. by installing or uninstalling packages), you must re-compile and relink the LAMMPS -executable with the same "make" command. This makefiles dependencies -should insure that only the subset of files that need to be are -re-compiled. +executable with the same ``make `` command. The makefile's +dependency tracking should insure that only the necessary subset of +files are re-compiled. If you change settings in the makefile, you have +to recompile *everything*. To delete all objects you can use ``make +clean-``. .. note:: @@ -65,13 +79,15 @@ re-compiled. correctly detect which parts need to be recompiled after changes were made to the sources. ----------- +Customized builds and alternate makefiles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The lammps/src/MAKE tree contains the Makefile.machine files included -in the LAMMPS distribution. Typing "make machine" uses -*Makefile.machine*\ . Thus the "make serial" or "make mpi" lines above -use Makefile.serial and Makefile.mpi, respectively. Other makefiles -are in these directories: +The ``src/MAKE`` directory tree contains the ``Makefile.`` +files included in the LAMMPS distribution. Typing ``make example`` uses +``Makefile.example`` from one of those folders, if available. Thus the +``make serial`` and ``make mpi`` lines above use +``src/MAKE/Makefile.serial`` and ``src/MAKE/Makefile.mpi``, +respectively. Other makefiles are in these directories: .. code-block:: bash @@ -79,11 +95,16 @@ are in these directories: MACHINES # Makefiles for specific machines MINE # customized Makefiles you create (you may need to create this folder) -Typing "make" lists all the available Makefile.machine files. A file -with the same name can appear in multiple folders (not a good idea). -The order the directories are searched is as follows: src/MAKE/MINE, -src/MAKE, src/MAKE/OPTIONS, src/MAKE/MACHINES. This gives preference -to a customized file you put in src/MAKE/MINE. +Simply typing ``make`` lists all the available ``Makefile.`` +files with a single line description toward the end of the output. A +file with the same name can appear in multiple folders (not a good +idea). The order the directories are searched is as follows: +``src/MAKE/MINE``, ``src/MAKE``, ``src/MAKE/OPTIONS``, +``src/MAKE/MACHINES``. This gives preference to a customized file you +put in ``src/MAKE/MINE``. If you create your own custom makefile under +a new name, please edit the first line with the description and machine +name, so you will not confuse yourself, when looking at the machine +summary. Makefiles you may wish to try include these (some require a package first be installed). Many of these include specific compiler flags diff --git a/doc/src/Build_package.rst b/doc/src/Build_package.rst index 2e0bea723a..f2511bb6a4 100644 --- a/doc/src/Build_package.rst +++ b/doc/src/Build_package.rst @@ -130,6 +130,8 @@ src directory. ---------- +.. _cmake_presets: + **CMake shortcuts for installing many packages**\ : Instead of specifying all the CMake options via the command-line, diff --git a/doc/src/Commands_all.rst b/doc/src/Commands_all.rst index 36ae112527..92c0ba107a 100644 --- a/doc/src/Commands_all.rst +++ b/doc/src/Commands_all.rst @@ -70,6 +70,7 @@ An alphabetic list of all general LAMMPS commands. * :doc:`kim_init ` * :doc:`kim_interactions ` * :doc:`kim_param ` + * :doc:`kim_property ` * :doc:`kim_query ` * :doc:`kspace_modify ` * :doc:`kspace_style ` diff --git a/doc/src/Commands_compute.rst b/doc/src/Commands_compute.rst index a17bbe08ba..a54c4b4e03 100644 --- a/doc/src/Commands_compute.rst +++ b/doc/src/Commands_compute.rst @@ -42,7 +42,7 @@ KOKKOS, o = USER-OMP, t = OPT. * :doc:`com ` * :doc:`com/chunk ` * :doc:`contact/atom ` - * :doc:`coord/atom ` + * :doc:`coord/atom (k) ` * :doc:`damage/atom ` * :doc:`dihedral ` * :doc:`dihedral/local ` @@ -79,16 +79,13 @@ KOKKOS, o = USER-OMP, t = OPT. * :doc:`ke/atom/eff ` * :doc:`ke/eff ` * :doc:`ke/rigid ` - * :doc:`meso/e/atom ` - * :doc:`meso/rho/atom ` - * :doc:`meso/t/atom ` * :doc:`mesont ` * :doc:`momentum ` * :doc:`msd ` * :doc:`msd/chunk ` * :doc:`msd/nongauss ` * :doc:`omega/chunk ` - * :doc:`orientorder/atom ` + * :doc:`orientorder/atom (k) ` * :doc:`pair ` * :doc:`pair/local ` * :doc:`pe ` @@ -134,6 +131,9 @@ KOKKOS, o = USER-OMP, t = OPT. * :doc:`sna/atom ` * :doc:`snad/atom ` * :doc:`snav/atom ` + * :doc:`sph/e/atom ` + * :doc:`sph/rho/atom ` + * :doc:`sph/t/atom ` * :doc:`spin ` * :doc:`stress/atom ` * :doc:`stress/mop ` @@ -162,5 +162,6 @@ KOKKOS, o = USER-OMP, t = OPT. * :doc:`torque/chunk ` * :doc:`vacf ` * :doc:`vcm/chunk ` + * :doc:`viscosity/cos ` * :doc:`voronoi/atom ` * :doc:`xrd ` diff --git a/doc/src/Commands_fix.rst b/doc/src/Commands_fix.rst index ce7cade0f6..3df8690af9 100644 --- a/doc/src/Commands_fix.rst +++ b/doc/src/Commands_fix.rst @@ -22,6 +22,7 @@ OPT. .. table_from_list:: :columns: 5 + * :doc:`accelerate/cos ` * :doc:`adapt ` * :doc:`adapt/fep ` * :doc:`addforce ` @@ -94,9 +95,7 @@ OPT. * :doc:`lb/viscous ` * :doc:`lineforce ` * :doc:`manifoldforce ` - * :doc:`meso ` * :doc:`meso/move ` - * :doc:`meso/stationary ` * :doc:`momentum (k) ` * :doc:`move ` * :doc:`mscg ` @@ -201,6 +200,8 @@ OPT. * :doc:`smd/move_tri_surf ` * :doc:`smd/setvel ` * :doc:`smd/wall_surface ` + * :doc:`sph ` + * :doc:`sph/stationary ` * :doc:`spring ` * :doc:`spring/chunk ` * :doc:`spring/rg ` diff --git a/doc/src/Commands_pair.rst b/doc/src/Commands_pair.rst index 4effe71f42..8e1158610e 100644 --- a/doc/src/Commands_pair.rst +++ b/doc/src/Commands_pair.rst @@ -131,6 +131,7 @@ OPT. * :doc:`lj/class2/coul/cut (ko) ` * :doc:`lj/class2/coul/cut/soft ` * :doc:`lj/class2/coul/long (gko) ` + * :doc:`lj/class2/coul/long/cs ` * :doc:`lj/class2/coul/long/soft ` * :doc:`lj/class2/soft ` * :doc:`lj/cubic (go) ` diff --git a/doc/src/Commands_removed.rst b/doc/src/Commands_removed.rst index b749467d13..e137e39a08 100644 --- a/doc/src/Commands_removed.rst +++ b/doc/src/Commands_removed.rst @@ -48,7 +48,7 @@ The USER-CUDA package had been removed, since it had been unmaintained for a long time and had known bugs and problems. Significant parts of the design were transferred to the :ref:`KOKKOS package `, which has similar -performance characteristics on Nvidia GPUs. Both, the KOKKOS +performance characteristics on NVIDIA GPUs. Both, the KOKKOS and the :ref:`GPU package ` are maintained and allow running LAMMPS with GPU acceleration. diff --git a/doc/src/Errors.rst b/doc/src/Errors.rst index 896dfbcbef..48db5f6472 100644 --- a/doc/src/Errors.rst +++ b/doc/src/Errors.rst @@ -12,5 +12,6 @@ additional details for many of them. Errors_common Errors_bugs + Errors_debug Errors_messages Errors_warnings diff --git a/doc/src/Errors_bugs.rst b/doc/src/Errors_bugs.rst index e169b93d81..ff0b155795 100644 --- a/doc/src/Errors_bugs.rst +++ b/doc/src/Errors_bugs.rst @@ -1,7 +1,8 @@ Reporting bugs ============== -If you are confident that you have found a bug in LAMMPS, please follow the steps outlined below: +If you are confident that you have found a bug in LAMMPS, please follow +the steps outlined below: * Check the `New features and bug fixes `_ section of the `LAMMPS WWW site @@ -17,20 +18,22 @@ If you are confident that you have found a bug in LAMMPS, please follow the step * Check the `mailing list archives `_ to see if the issue has been discussed before. -If none of these steps yields any useful information, please file -a new bug report on the `GitHub Issue page `_\ . -The website will offer you to select a suitable template with explanations -and then you should replace those explanations with the information -that you can provide to reproduce your issue. +If none of these steps yields any useful information, please file a new +bug report on the `GitHub Issue page `_. The website will offer +you to select a suitable template with explanations and then you should +replace those explanations with the information that you can provide to +reproduce your issue. The most useful thing you can do to help us verify and fix a bug is to isolate the problem. Run it on the smallest number of atoms and fewest number of processors with the simplest input script that reproduces the -bug. Try to identify what command or combination of commands is -causing the problem and upload the complete input deck as a tar or zip -archive. Please avoid using binary restart files unless the issue requires -it. In the latter case you should also include an input deck to quickly +bug. Try to identify what command or combination of commands is causing +the problem and upload the complete input deck as a tar or zip archive. +Please avoid using binary restart files unless the issue requires it. +In the latter case you should also include an input deck to quickly generate this restart from a data file or a simple additional input. +This input deck can be used with tools like a debugger or `valgrind +`_ to further :doc:`debug the crash `. You may also send an email to the LAMMPS mailing list at "lammps-users at lists.sourceforge.net" describing the problem with the @@ -43,3 +46,4 @@ have looked at it. .. _lws: https://lammps.sandia.gov .. _gip: https://github.com/lammps/issues +.. _valgrind: https://valgrind.org diff --git a/doc/src/Errors_debug.rst b/doc/src/Errors_debug.rst new file mode 100644 index 0000000000..1e7e678602 --- /dev/null +++ b/doc/src/Errors_debug.rst @@ -0,0 +1,237 @@ +Debugging crashes +================= + +If LAMMPS crashes with a "segmentation fault" or a "bus error" or +similar message, then you can use the following two methods to further +narrow down the origin of the issue. This will help the LAMMPS +developers (or yourself) to understand the reason for the crash and +apply a fix (either to the input script or the source code). +This requires that your LAMMPS executable includes the required +:ref:`debug information `. Otherwise it is not possible to +look up the names of functions or variables. + +The following patch will introduce a bug into the code for pair style +:doc:`lj/cut ` when using the ``examples/melt/in.melt`` input. +We use it to show how to identify the origin of a segmentation fault. + +.. code-block:: diff + + --- a/src/pair_lj_cut.cpp + +++ b/src/pair_lj_cut.cpp + @@ -81,6 +81,7 @@ void PairLJCut::compute(int eflag, int vflag) + int nlocal = atom->nlocal; + double *special_lj = force->special_lj; + int newton_pair = force->newton_pair; + + double comx = 0.0; + + inum = list->inum; + ilist = list->ilist; + @@ -134,8 +135,10 @@ void PairLJCut::compute(int eflag, int vflag) + evdwl,0.0,fpair,delx,dely,delz); + } + } + - } + + + comx += atom->rmass[i]*x[i][0]; /* BUG */ + + } + + printf("comx = %g\n",comx); + if (vflag_fdotr) virial_fdotr_compute(); + } + +After recompiling LAMMPS and running the input you should get something like this: + +.. code-block: + + $ ./lmp -in in.melt + LAMMPS (19 Mar 2020) + OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task + Lattice spacing in x,y,z = 1.6796 1.6796 1.6796 + Created orthogonal box = (0 0 0) to (16.796 16.796 16.796) + 1 by 1 by 1 MPI processor grid + Created 4000 atoms + create_atoms CPU = 0.000432253 secs + Neighbor list info ... + update every 20 steps, delay 0 steps, check no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.8 + ghost atom cutoff = 2.8 + binsize = 1.4, bins = 12 12 12 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d/newton + bin: standard + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + Segmentation fault (core dumped) + + +Using the GDB debugger to get a stack trace +------------------------------------------- + +There are two options to use the GDB debugger for identifying the origin +of the segmentation fault or similar crash. The GDB debugger has many +more features and options, as can be seen for example its `online +documentation `_. + +Run LAMMPS from within the debugger +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running LAMMPS under the control of the debugger as shown below only +works for a single MPI rank (for debugging a program running in parallel +you usually need a parallel debugger program). A simple way to launch +GDB is to prefix the LAMMPS command line with ``gdb --args`` and then +type the command "run" at the GDB prompt. This will launch the +debugger, load the LAMMPS executable and its debug info, and then run +it. When it reaches the code causing the segmentation fault, it will +stop with a message why it stopped, print the current line of code, and +drop back to the GDB prompt. + +.. code-block:: + + [...] + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + + Program received signal SIGSEGV, Segmentation fault. + 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x829740, eflag=1, vflag=) at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + 139 comx += atom->rmass[i]*x[i][0]; /* BUG */ + (gdb) + +Now typing the command "where" will show the stack of functions starting from +the current function back to "main()". + +.. code-block:: + + (gdb) where + #0 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x829740, eflag=1, vflag=) at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + #1 0x00000000004cf0a2 in LAMMPS_NS::Verlet::setup (this=0x7e6c90, flag=1) at /home/akohlmey/compile/lammps/src/verlet.cpp:131 + #2 0x000000000049db42 in LAMMPS_NS::Run::command (this=this@entry=0x7fffffffcca0, narg=narg@entry=1, arg=arg@entry=0x7e8750) + at /home/akohlmey/compile/lammps/src/run.cpp:177 + #3 0x000000000041258a in LAMMPS_NS::Input::command_creator (lmp=, narg=1, arg=0x7e8750) + at /home/akohlmey/compile/lammps/src/input.cpp:878 + #4 0x0000000000410ad3 in LAMMPS_NS::Input::execute_command (this=0x7d1410) at /home/akohlmey/compile/lammps/src/input.cpp:864 + #5 0x00000000004111fb in LAMMPS_NS::Input::file (this=0x7d1410) at /home/akohlmey/compile/lammps/src/input.cpp:229 + #6 0x000000000040933a in main (argc=, argv=) at /home/akohlmey/compile/lammps/src/main.cpp:65 + (gdb) + +You can also print the value of variables and see if there is anything +unexpected. Segmentation faults, for example, commonly happen when a +pointer variable is not assigned and still initialized to NULL. + +.. code-block:: + + (gdb) print x + $1 = (double **) 0x7ffff7ca1010 + (gdb) print i + $2 = 0 + (gdb) print x[0] + $3 = (double *) 0x7ffff6d80010 + (gdb) print x[0][0] + $4 = 0 + (gdb) print x[1][0] + $5 = 0.83979809569125363 + (gdb) print atom->rmass + $6 = (double *) 0x0 + (gdb) + + +Inspect a core dump file with the debugger +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When an executable crashes with a "core dumped" message, it creates a +file "core" or "core." which contains the information about the +current state. This file may be located in the folder where you ran +LAMMPS or in some hidden folder managed by the systemd daemon. In the +latter case, you need to "extract" the core file with the ``coredumpctl`` +utility to the current folder. Example: ``coredumpctl -o core dump lmp``. +Now you can launch the debugger to load the executable, its debug info +and the core dump and drop you to a prompt like before. + +.. code-block:: + + $ gdb lmp core + Reading symbols from lmp... + [New LWP 1928535] + [Thread debugging using libthread_db enabled] + Using host libthread_db library "/lib64/libthread_db.so.1". + Core was generated by `./lmp -in in.melt'. + Program terminated with signal SIGSEGV, Segmentation fault. + #0 0x00000000006653ab in LAMMPS_NS::PairLJCut::compute (this=0x1b10740, eflag=1, vflag=) + at /home/akohlmey/compile/lammps/src/pair_lj_cut.cpp:139 + 139 comx += atom->rmass[i]*x[i][0]; /* BUG */ + (gdb) + +From here on, you use the same commands as shown before to get a stack +trace and print current values of (pointer) variables. + + +Using valgrind to get a stack trace +----------------------------------- + +The `valgrind `_ suite of tools allows to closely +inspect the behavior of a compiled program by essentially emulating a +CPU and instrumenting the program while running. This slows down +execution quite significantly, but can also report issues that are not +resulting in a crash. The default valgrind tool is a memory checker and +you can use it by prefixing the normal command line with ``valgrind``. +Unlike GDB, this will also work for parallel execution, but it is +recommended to redirect the valgrind output to a file (e.g. with +``--log-file=crash-%p.txt``, the %p will be substituted with the +process ID) so that the messages of the multiple valgrind instances to +the console are not mixed. + +.. code-block:: + + $ valgrind ./lmp -in in.melt + ==1933642== Memcheck, a memory error detector + ==1933642== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. + ==1933642== Using Valgrind-3.15.0 and LibVEX; rerun with -h for copyright info + ==1933642== Command: ./lmp -in in.melt + ==1933642== + LAMMPS (19 Mar 2020) + OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task + Lattice spacing in x,y,z = 1.6796 1.6796 1.6796 + Created orthogonal box = (0 0 0) to (16.796 16.796 16.796) + 1 by 1 by 1 MPI processor grid + Created 4000 atoms + create_atoms CPU = 0.032964 secs + Neighbor list info ... + update every 20 steps, delay 0 steps, check no + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.8 + ghost atom cutoff = 2.8 + binsize = 1.4, bins = 12 12 12 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d/newton + bin: standard + Setting up Verlet run ... + Unit style : lj + Current step : 0 + Time step : 0.005 + ==1933642== Invalid read of size 8 + ==1933642== at 0x6653AB: LAMMPS_NS::PairLJCut::compute(int, int) (pair_lj_cut.cpp:139) + ==1933642== by 0x4CF0A1: LAMMPS_NS::Verlet::setup(int) (verlet.cpp:131) + ==1933642== by 0x49DB41: LAMMPS_NS::Run::command(int, char**) (run.cpp:177) + ==1933642== by 0x412589: void LAMMPS_NS::Input::command_creator(LAMMPS_NS::LAMMPS*, int, char**) (input.cpp:881) + ==1933642== by 0x410AD2: LAMMPS_NS::Input::execute_command() (input.cpp:864) + ==1933642== by 0x4111FA: LAMMPS_NS::Input::file() (input.cpp:229) + ==1933642== by 0x409339: main (main.cpp:65) + ==1933642== Address 0x0 is not stack'd, malloc'd or (recently) free'd + ==1933642== + +As you can see, the stack trace information is similar to that obtained +from GDB. In addition you get a more specific hint about what cause the +segmentation fault, i.e. that it is a NULL pointer dereference. To find +out which pointer exactly was NULL, you need to use the debugger, though. + diff --git a/doc/src/Errors_messages.rst b/doc/src/Errors_messages.rst index 6970cb4a9a..2140719253 100644 --- a/doc/src/Errors_messages.rst +++ b/doc/src/Errors_messages.rst @@ -2783,7 +2783,7 @@ Doc page with :doc:`WARNING messages ` if this helps. *Did not find all elements in MEAM library file* - The requested elements were not found in the MEAM file. + Some requested elements were not found in the MEAM file. Check spelling etc. *Did not find fix shake partner info* Could not find bond partners implied by fix shake command. This error @@ -3135,6 +3135,9 @@ Doc page with :doc:`WARNING messages ` *Epsilon or sigma reference not set by pair style in ewald/n* The pair style is not providing the needed epsilon or sigma values. +*Error in MEAM parameter file: keyword %s (further information)* + Self-explanatory. Check the parameter file. + *Error in vdw spline: inner radius > outer radius* A pre-tabulated spline is invalid. Likely a problem with the potential parameters. @@ -4611,7 +4614,7 @@ Doc page with :doc:`WARNING messages ` *Incorrect format in COMB3 potential file* Incorrect number of words per line in the potential file. -*Incorrect format in MEAM potential file* +*Incorrect format in MEAM library file* Incorrect number of words per line in the potential file. *Incorrect format in SNAP coefficient file* @@ -5669,6 +5672,9 @@ Doc page with :doc:`WARNING messages ` A fix is referenced incorrectly or a fix that produces per-atom values is used in an equal-style variable formula. +*Mismatched parameter in MEAM library file: z!=lat* + The coordination number and lattice do not match, check that consistent values are given. + *Mismatched variable in variable formula* A variable is referenced incorrectly or an atom-style variable that produces per-atom values is used in an equal-style variable @@ -5845,6 +5851,9 @@ Doc page with :doc:`WARNING messages ` Cannot use the temper command with only one processor partition. Use the -partition command-line option. +*Must not have multiple fixes change box parameter ...* + Self-explanatory. + *Must read Atoms before Angles* The Atoms section of a data file must come before an Angles section. @@ -7764,6 +7773,9 @@ keyword to allow for additional bonds to be formed *Too many atoms to dump sort* Cannot sort when running with more than 2\^31 atoms. +*Too many elements extracted from MEAM library.* + Increase 'maxelt' in meam.h and recompile. + *Too many exponent bits for lookup table* Table size specified via pair_modify command does not work with your machine's floating point representation. @@ -7997,11 +8009,11 @@ keyword to allow for additional bonds to be formed *Unknown unit_style* Self-explanatory. Check the input script or data file. -*Unrecognized lattice type in MEAM file 1* +*Unrecognized lattice type in MEAM library file* The lattice type in an entry of the MEAM library file is not valid. -*Unrecognized lattice type in MEAM file 2* +*Unrecognized lattice type in MEAM parameter file* The lattice type in an entry of the MEAM parameter file is not valid. @@ -8017,6 +8029,9 @@ keyword to allow for additional bonds to be formed *Unsupported order in kspace_style pppm/disp, pair_style %s* Only pair styles with 1/r and 1/r\^6 dependence are currently supported. +*Unsupported parameter in MEAM library file* + Self-explanatory. + *Use cutoff keyword to set cutoff in single mode* Mode is single so cutoff/multi keyword cannot be used. diff --git a/doc/src/Howto.rst b/doc/src/Howto.rst index f4376898b1..0ca955ab86 100644 --- a/doc/src/Howto.rst +++ b/doc/src/Howto.rst @@ -16,6 +16,7 @@ Tutorials howto :name: tutorials :maxdepth: 1 + Howto_cmake Howto_github Howto_pylammps Howto_bash diff --git a/doc/src/Howto_cmake.rst b/doc/src/Howto_cmake.rst new file mode 100644 index 0000000000..fa6038efb5 --- /dev/null +++ b/doc/src/Howto_cmake.rst @@ -0,0 +1,483 @@ +Using CMake with LAMMPS tutorial +================================ + +The support for building LAMMPS with CMake is a recent addition to +LAMMPS thanks to the efforts of Christoph Junghans (LANL) and Richard +Berger (Temple U). One of the key strengths of CMake is that it is not +tied to a specific platform or build system and thus generate the files +necessary to build and develop for different build systems and on +different platforms. Note, that this applies to the build system itself +not the LAMMPS code. In other words, without additional porting effort, +it is not possible - for example - to compile LAMMPS with Visual C++ on +Windows. The build system output can also include support files +necessary to program LAMMPS as a project in integrated development +environments (IDE) like Eclipse, Visual Studio, QtCreator, Xcode, +CodeBlocks, Kate and others. + +A second important feature of CMake is, that it can detect and validate +available libraries, optimal settings, available support tools and so +on, so that by default LAMMPS will take advantage of available tools +without requiring to provide the details about how to enable/integrate +them. + +The downside of this approach is, that there is some complexity +associated with running CMake itself and how to customize the building +of LAMMPS. This tutorial will show how to manage this through some +selected examples. Please see the chapter about :doc:`building LAMMPS +` for descriptions of specific flags and options for LAMMPS in +general and for specific packages. + +CMake can be used through either the command-line interface (CLI) +program ``cmake`` (or ``cmake3``), a text mode interactive user +interface (TUI) program ``ccmake`` (or ``ccmake3``), or a graphical user +interface (GUI) program ``cmake-gui``. All of them are portable +software available on all supported platforms and can be used +interchangeably. The minimum supported CMake version is 3.10 (3.12 or +later is recommended). + +All details about features and settings for CMake are in the `CMake +online documentation `_. We focus +below on the most important aspects with respect to compiling LAMMPS. + +Prerequisites +------------- + +This tutorial assumes that you are operating in a command-line environment +using a shell like Bash. + +- Linux: any Terminal window will work +- MacOS X: launch the Terminal application. +- Windows 10: install and run the :doc:`Windows subsystem for Linux ` + +We also assume that you have downloaded and unpacked a recent LAMMPS source code package +or used Git to create a clone of the LAMMPS sources on your compilation machine. + +You should change into the top level directory of the LAMMPS source tree all +paths mentioned in the tutorial are relative to that. Immediately after downloading +it should look like this: + +.. code-block:: bash + + $ ls + bench doc lib potentials README tools + cmake examples LICENSE python src + +Build versus source directory +----------------------------- + +When using CMake the build procedure is separated into multiple distinct phases: + + #. **Configuration:** detect or define which features and settings + should be enable and used and how LAMMPS should be compiled + #. **Compilation:** generate and compile all necessary source files + and build libraries and executables. + #. **Installation:** copy selected files from the compilation into + your file system, so they can be used without having to keep the + source and build tree around. + +The configuration and compilation of LAMMPS has to happen in a dedicated +*build directory* which must be different from the source directory. +Also the source directory (``src``) must remain pristine, so it is not +allowed to "install" packages using the traditional make process and +after an compilation attempt all created source files must be removed. +This can be achieved with ``make no-all purge``. + +You can pick **any** folder outside the source tree. We recommend to +create a folder ``build`` in the top-level directory, or multiple +folders in case you want to have separate builds of LAMMPS with +different options (``build-parallel``, ``build-serial``) or with +different compilers (``build-gnu``, ``build-clang``, ``build-intel``) +and so on. All the auxiliary files created by one build process +(executable, object files, log files, etc) are stored in this directory +or sub-directories within it that CMake creates. + + +Running CMake +------------- + +CLI version +^^^^^^^^^^^ + +In the (empty) ``build`` directory, we now run the command ``cmake +../cmake``, which will start the configuration phase and you will see +the progress of the configuration printed to the screen followed by a +summary of the enabled features, options and compiler settings. A typical +summary screen will look like this: + +.. code-block:: + + $ cmake ../cmake/ + -- The CXX compiler identification is GNU 8.2.0 + -- Check for working CXX compiler: /opt/tools/gcc-8.2.0/bin/c++ + -- Check for working CXX compiler: /opt/tools/gcc-8.2.0/bin/c++ - works + -- Detecting CXX compiler ABI info + -- Detecting CXX compiler ABI info - done + -- Detecting CXX compile features + -- Detecting CXX compile features - done + -- Found Git: /usr/bin/git (found version "2.25.2") + -- Running check for auto-generated files from make-based build system + -- Found MPI_CXX: /usr/lib64/mpich/lib/libmpicxx.so (found version "3.1") + -- Found MPI: TRUE (found version "3.1") + -- Looking for C++ include omp.h + -- Looking for C++ include omp.h - found + -- Found OpenMP_CXX: -fopenmp (found version "4.5") + -- Found OpenMP: TRUE (found version "4.5") + -- Found JPEG: /usr/lib64/libjpeg.so (found version "62") + -- Found PNG: /usr/lib64/libpng.so (found version "1.6.37") + -- Found ZLIB: /usr/lib64/libz.so (found version "1.2.11") + -- Found GZIP: /usr/bin/gzip + -- Found FFMPEG: /usr/bin/ffmpeg + -- Performing Test COMPILER_SUPPORTS-ffast-math + -- Performing Test COMPILER_SUPPORTS-ffast-math - Success + -- Performing Test COMPILER_SUPPORTS-march=native + -- Performing Test COMPILER_SUPPORTS-march=native - Success + -- Looking for C++ include cmath + -- Looking for C++ include cmath - found + -- Generating style_angle.h... + [...] + -- Generating lmpinstalledpkgs.h... + -- The following tools and libraries have been found and configured: + * Git + * MPI + * OpenMP + * JPEG + * PNG + * ZLIB + + -- <<< Build configuration >>> + Build type: RelWithDebInfo + Install path: /home/akohlmey/.local + Generator: Unix Makefiles using /usr/bin/gmake + -- <<< Compilers and Flags: >>> + -- C++ Compiler: /opt/tools/gcc-8.2.0/bin/c++ + Type: GNU + Version: 8.2.0 + C++ Flags: -O2 -g -DNDEBUG + Defines: LAMMPS_SMALLBIG;LAMMPS_MEMALIGN=64;LAMMPS_JPEG;LAMMPS_PNG;LAMMPS_GZIP;LAMMPS_FFMPEG + Options: -ffast-math;-march=native + -- <<< Linker flags: >>> + -- Executable name: lmp + -- Static library flags: + -- <<< MPI flags >>> + -- MPI includes: /usr/include/mpich-x86_64 + -- MPI libraries: /usr/lib64/mpich/lib/libmpicxx.so;/usr/lib64/mpich/lib/libmpi.so; + -- Configuring done + -- Generating done + -- Build files have been written to: /home/akohlmey/compile/lammps/build + +The ``cmake`` command has one mandatory argument, and that is a folder +with either the file ``CMakeLists.txt`` or ``CMakeCache.txt``. The +``CMakeCache.txt`` file is created during the CMake configuration run +and contains all active settings, thus after a first run of CMake +all future runs in the build folder can use the folder ``.`` and CMake +will know where to find the CMake scripts and reload the settings +from the previous step. This means, that one can modify an existing +configuration by re-running CMake, but only needs to provide flags +indicating the desired change, everything else will be retained. One +can also mix compilation and configuration, i.e. start with a minimal +configuration and then, if needed, enable additional features and +recompile. + +The steps above **will NOT compile the code**\ . The compilation can be +started in a portable fashion with ``cmake --build .``, or you use the +selected built tool, e.g. ``make``. + +TUI version +^^^^^^^^^^^ + +For the text mode UI CMake program the basic principle is the same. +You start the command ``ccmake ../cmake`` in the ``build`` folder. + +.. list-table:: + + * - .. figure:: JPG/ccmake-initial.png + :target: JPG/ccmake-initial.png + :align: center + + Initial ``ccmake`` screen + + - .. figure:: JPG/ccmake-config.png + :target: JPG/ccmake-config.png + :align: center + + Configure output of ``ccmake`` + + - .. figure:: JPG/ccmake-options.png + :target: JPG/ccmake-options.png + :align: center + + Options screen of ``ccmake`` + +This will show you the initial screen (left image) with the empty +configuration cache. Now you type the 'c' key to run the configuration +step. That will do a first configuration run and show the summary +(center image). You exit the summary screen with 'e' and see now the +main screen with detected options and settings. You can now make changes +by moving and down with the arrow keys of the keyboard and modify +entries. For on/off settings, the enter key will toggle the state. +For others, hitting enter will allow you to modify the value and +you commit the change by hitting the enter key again or cancel using +the escape key. All "new" settings will be marked with a star '\*' +and for as long as one setting is marked like this, you have to +re-run the configuration by hitting the 'c' key again, sometimes +multiple times unless the TUI shows the word "generate" next to the +letter 'g' and by hitting the 'g' key the build files will be written +to the folder and the TUI exits. You can quit without generating +build files by hitting 'q'. + +GUI version +^^^^^^^^^^^ + +For the graphical CMake program the steps are similar to the TUI +version. You can type the command ``cmake-gui ../cmake`` in the +``build`` folder. In this case the path to the CMake script folder is +not required, it can also be entered from the GUI. + +.. list-table:: + + * - .. figure:: JPG/cmake-gui-initial.png + :target: JPG/cmake-gui-initial.png + :align: center + + Initial ``cmake-gui`` screen + + - .. figure:: JPG/cmake-gui-popup.png + :target: JPG/cmake-gui-popup.png + :align: center + + Generator selection in ``cmake-gui`` + + - .. figure:: JPG/cmake-gui-options.png + :target: JPG/cmake-gui-options.png + :align: center + + Options screen of ``cmake-gui`` + +Again, you start with an empty configuration cache (left image) and need +to start the configuration step. For the very first configuration in a +folder, you will have a pop-up dialog (center image) asking to select +the desired build tool and some configuration settings (stick with the +default) and then you get the option screen with all new settings +highlighted in red. You can modify them (or not) and click on the +"configure" button again until satisfied and click on the "generate" +button to write out the build files. You can exit the GUI from the +"File" menu or hit "ctrl-q". + + +Setting options +--------------- + +Options that enable, disable or modify settings are modified by setting +the value of CMake variables. This is done on the command line with the +*-D* flag in the format ``-D VARIABLE=value``, e.g. ``-D +CMAKE_BUILD_TYPE=Release`` or ``-D BUILD_MPI=on``. There is one quirk: +when used before the CMake directory, there may be a space between the +*-D* flag and the variable, after it must not be. Such CMake variables +can have boolean values (on/off, yes/no, or 1/0 are all valid) or are +strings representing a choice, or a path, or are free format. If the +string would contain whitespace, it must be put in quotes, for example +``-D CMAKE_TUNE_FLAGS="-ftree-vectorize -ffast-math"``. + +CMake variables fall into two categories: 1) common CMake variables that +are used by default for any CMake configuration setup and 2) project +specific variables, i.e. settings that are specific for LAMMPS. +Also CMake variables can be flagged as *advanced*, which means they are +not shown in the text mode or graphical CMake program in the overview +of all settings by default, but only when explicitly requested (by hitting +the 't' key or clicking on the 'Advanced' check-box). + +Some common CMake variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + + * - Variable + - Description + * - ``CMAKE_INSTALL_PREFIX`` + - root directory of install location for ``make install`` (default: ``$HOME/.local``) + * - ``CMAKE_BUILD_TYPE`` + - controls compilation options: + one of ``RelWithDebInfo`` (default), ``Release``, ``Debug``, ``MinSizeRel`` + * - ``BUILD_SHARED_LIBS`` + - if set to ``on`` build the LAMMPS library as shared library (default: ``off``) + * - ``CMAKE_MAKE_PROGRAM`` + - name/path of the compilation command (default depends on *-G* option, usually ``make``) + * - ``CMAKE_VERBOSE_MAKEFILE`` + - if set to ``on`` echo commands while executing during build (default: ``off``) + * - ``CMAKE_C_COMPILER`` + - C compiler to be used for compilation (default: system specific, ``gcc`` on Linux) + * - ``CMAKE_CXX_COMPILER`` + - C++ compiler to be used for compilation (default: system specific, ``g++`` on Linux) + * - ``CMAKE_Fortran_COMPILER`` + - Fortran compiler to be used for compilation (default: system specific, ``gfortran`` on Linux) + * - ``CXX_COMPILER_LAUNCHER`` + - tool to launch the C++ compiler, e.g. ``ccache`` or ``distcc`` for faster compilation (default: empty) + +Some common LAMMPS specific variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + + * - Variable + - Description + * - ``BUILD_MPI`` + - build LAMMPS with MPI support (default: ``on`` if a working MPI available, else ``off``) + * - ``BUILD_OMP`` + - build LAMMPS with OpenMP support (default: ``on`` if compiler supports OpenMP fully, else ``off``) + * - ``BUILD_TOOLS`` + - compile some additional executables from the ``tools`` folder (default: ``off``) + * - ``BUILD_DOC`` + - include building the HTML format documentation for packaging/installing (default: ``off``) + * - ``CMAKE_TUNE_FLAGS`` + - common compiler flags, for optimization or instrumentation (default: compiler specific) + * - ``LAMMPS_MACHINE`` + - when set to ``name`` the LAMMPS executable and library will be called ``lmp_name`` and ``liblammps_name.a`` + * - ``LAMMPS_EXCEPTIONS`` + - when set to ``on`` errors will throw a C++ exception instead of aborting (default: ``off``) + * - ``FFT`` + - select which FFT library to use: ``FFTW3``, ``MKL``, ``KISS`` (default, unless FFTW3 is found) + * - ``FFT_SINGLE`` + - select whether to use single precision FFTs (default: ``off``) + * - ``WITH_JPEG`` + - whether to support JPEG format in :doc:`dump image ` (default: ``on`` if found) + * - ``WITH_PNG`` + - whether to support PNG format in :doc:`dump image ` (default: ``on`` if found) + * - ``WITH_GZIP`` + - whether to support reading and writing compressed files (default: ``on`` if found) + * - ``WITH_FFMPEG`` + - whether to support generating movies with :doc:`dump movie ` (default: ``on`` if found) + +Enabling or disabling LAMMPS packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The LAMMPS software is organized into a common core that is always +included and a large number of :doc:`add-on packages ` that +have to be enabled to be included into a LAMMPS executable. Packages +are enabled through setting variables of the kind ``PKG_`` to +``on`` and disabled by setting them to ``off`` (or using ``yes``, +``no``, ``1``, ``0`` correspondingly). ```` has to be replaced by +the name of the package, e.g. ``MOLECULE`` or ``USER-MISC``. + + +Using presets +------------- + +Since LAMMPS has a lot of optional features and packages, specifying +them all on the command line can be tedious. Or when selecting a +different compiler toolchain, multiple options have to be changed +consistently and that is rather error prone. Or when enabling certain +packages, they require consistent settings to be operated in a +particular mode. For this purpose, we are providing a selection of +"preset files" for CMake in the folder ``cmake/presets``. They +represent a way to pre-load or override the CMake configuration cache by +setting or changing CMake variables. Preset files are loaded using the +*-C* command line flag. You can combine loading multiple preset files or +change some variables later with additional *-D* flags. A few examples: + +.. code-block:: bash + + cmake -C ../cmake/presets/minimal.cmake -D PKG_MISC=on ../cmake + cmake -C ../cmake/presets/clang.cmake -C ../cmake/presets/most.cmake ../cmake + cmake -C ../cmake/presets/minimal.cmake -D BUILD_MPI=off ../cmake + +The first command will install the packages ``KSPACE``, ``MANYBODY``, +``MOLECULE``, ``RIGID`` and ``MISC``; the first four from the preset +file and the fifth from the explicit variable definition. The second +command will first switch the compiler toolchain to use the Clang +compilers and install a large number of packages that are not depending +on any special external libraries or tools and are not very unusual. +The third command will enable the first four packages like above and +then enforce compiling LAMMPS as a serial program (using the MPI STUBS +library). + +It is also possible to do this incrementally. + +.. code-block:: bash + + cmake -C ../cmake/presets/minimal.cmake ../cmake + cmake -D PKG_MISC=on . + +will achieve the same configuration like in the first example above. In +this scenario it is particularly convenient to do the second +configuration step using either the text mode or graphical user +interface (``ccmake`` or ``cmake-gui``). + +Compilation and build targets +----------------------------- + +The actual compilation will be started by running the selected build +command (on Linux this is by default ``make``, see below how to select +alternatives). You can also use the portable command ``cmake --build .`` +which will adapt to whatever the selected build command is. +This is particularly convenient, if you have set a custom build command +via the ``CMAKE_MAKE_PROGRAM`` variable. + +When calling the build program, you can also select which "target" is to +be build through appending the name of the target to the build command. +Example: ``cmake --build . all``. The following abstract targets are available: + +.. list-table:: + :header-rows: 1 + + * - Target + - Description + * - ``all`` + - build "everything" (default) + * - ``lammps`` + - build the LAMMPS library and executable + * - ``doc`` + - build the HTML documentation (if configured) + * - ``install`` + - install all target files into folders in ``CMAKE_INSTALL_PREFIX`` + * - ``test`` + - run some simple tests (if configured with ``-D ENABLE_TESTING=on``) + * - ``clean`` + - remove all generated files + + +Choosing generators +------------------- + +While CMake usually defaults to creating makefiles to compile software +with the ``make`` program, it supports multiple alternate build tools +(e.g. ``ninja-build`` which tends to be faster and more efficient in +parallelizing builds than ``make``) and can generate project files for +integrated development environments (IDEs) like VisualStudio, Eclipse or +CodeBlocks. This is specific to how the local CMake version was +configured and compiled. The list of available options can be seen at +the end of the output of ``cmake --help``. Example on Fedora 31 this is: + +.. code-block:: + + Generators + + The following generators are available on this platform (* marks default): + * Unix Makefiles = Generates standard UNIX makefiles. + Green Hills MULTI = Generates Green Hills MULTI files + (experimental, work-in-progress). + Ninja = Generates build.ninja files. + Ninja Multi-Config = Generates build-.ninja files. + Watcom WMake = Generates Watcom WMake makefiles. + CodeBlocks - Ninja = Generates CodeBlocks project files. + CodeBlocks - Unix Makefiles = Generates CodeBlocks project files. + CodeLite - Ninja = Generates CodeLite project files. + CodeLite - Unix Makefiles = Generates CodeLite project files. + Sublime Text 2 - Ninja = Generates Sublime Text 2 project files. + Sublime Text 2 - Unix Makefiles + = Generates Sublime Text 2 project files. + Kate - Ninja = Generates Kate project files. + Kate - Unix Makefiles = Generates Kate project files. + Eclipse CDT4 - Ninja = Generates Eclipse CDT 4.0 project files. + Eclipse CDT4 - Unix Makefiles= Generates Eclipse CDT 4.0 project files. + +Below is a screenshot of using the CodeBlocks IDE with the ninja build tool +after running CMake as follows: + +.. code-block:: bash + + cmake -G 'CodeBlocks - Ninja' ../cmake/presets/most.cmake ../cmake/ + +.. image:: JPG/cmake-codeblocks.png + :align: center diff --git a/doc/src/Howto_viscosity.rst b/doc/src/Howto_viscosity.rst index dff414ce9d..8c7eb91b12 100644 --- a/doc/src/Howto_viscosity.rst +++ b/doc/src/Howto_viscosity.rst @@ -1,10 +1,11 @@ Calculate viscosity =================== -The shear viscosity eta of a fluid can be measured in at least 5 ways +The shear viscosity eta of a fluid can be measured in at least 6 ways using various options in LAMMPS. See the examples/VISCOSITY directory for scripts that implement the 5 methods discussed here for a simple -Lennard-Jones fluid model. Also, see the :doc:`Howto kappa ` doc page for an analogous discussion for +Lennard-Jones fluid model and 1 method for SPC/E water model. +Also, see the :doc:`Howto kappa ` doc page for an analogous discussion for thermal conductivity. Eta is a measure of the propensity of a fluid to transmit momentum in @@ -130,9 +131,25 @@ time-integrated momentum fluxes play the role of Cartesian coordinates, whose mean-square displacement increases linearly with time at sufficiently long times. +The sixth is periodic perturbation method. It is also a non-equilibrium MD method. +However, instead of measure the momentum flux in response of applied velocity gradient, +it measures the velocity profile in response of applied stress. +A cosine-shaped periodic acceleration is added to the system via the +:doc:`fix accelerate/cos ` command, +and the :doc:`compute viscosity/cos` command is used to monitor the +generated velocity profile and remove the velocity bias before thermostatting. + +.. note:: + + An article by :ref:`(Hess) ` discussed the accuracy and efficiency of these methods. + ---------- .. _Daivis-viscosity: **(Daivis and Todd)** Daivis and Todd, Nonequilibrium Molecular Dynamics (book), Cambridge University Press, https://doi.org/10.1017/9781139017848, (2017). + +.. _Hess3: + +**(Hess)** Hess, B. The Journal of Chemical Physics 2002, 116 (1), 209-217. diff --git a/doc/src/Install_linux.rst b/doc/src/Install_linux.rst index ec4f3fcc8e..ca3f794fc2 100644 --- a/doc/src/Install_linux.rst +++ b/doc/src/Install_linux.rst @@ -24,12 +24,13 @@ allows you to install LAMMPS with a single command, and stay up-to-date with the current version of LAMMPS by simply updating your operating system. -To install the appropriate personal-package archive (PPA), do the +To install the appropriate personal-package archives (PPAs), do the following once: .. code-block:: bash $ sudo add-apt-repository ppa:gladky-anton/lammps + $ sudo add-apt-repository ppa:openkim/latest $ sudo apt-get update To install LAMMPS do the following once: @@ -38,7 +39,7 @@ To install LAMMPS do the following once: $ sudo apt-get install lammps-daily -This downloads an executable named "lmp_daily" to your box, which +This downloads an executable named ``lmp_daily`` to your box, which can then be used in the usual way to run input scripts: .. code-block:: bash @@ -60,11 +61,29 @@ To get a copy of the current documentation and examples: $ sudo apt-get install lammps-daily-doc which will download the doc files in -/usr/share/doc/lammps-daily-doc/doc and example problems in -/usr/share/doc/lammps-doc/examples. +``/usr/share/doc/lammps-daily-doc/doc`` and example problems in +``/usr/share/doc/lammps-doc/examples``. -Note that you may still wish to download the tarball to get potential -files and auxiliary tools. +To get a copy of the current potentials files: + +.. code-block:: bash + + $ sudo apt-get install lammps-daily-data + +which will download the potentials files to +``/usr/share/lammps-daily/potentials``. The ``lmp_daily`` binary is +hard-coded to look for potential files in this directory (it does not +use the `LAMMPS_POTENTIALS` environment variable, as described +in :doc:`pair_coeff ` command). + +The ``lmp_daily`` binary is built with the :ref:`KIM package ` which +results in the above command also installing the `kim-api` binaries when LAMMPS +is installed. In order to use potentials from `openkim.org `_, you +can install the `openkim-models` package + +.. code-block:: bash + + $ sudo apt-get install openkim-models To un-install LAMMPS, do the following: @@ -72,17 +91,8 @@ To un-install LAMMPS, do the following: $ sudo apt-get remove lammps-daily -Note that the lammps-daily executable is built with the following -sequence of make commands, as if you had done the same with the -unpacked tarball files in the src directory: - -.. code-block:: bash - - $ make yes-all - $ make no-lib - $ make mpi - -Thus it builds with FFTW3 and OpenMPI. +Please use ``lmp_daily -help`` to see which compilation options, packages, +and styles are included in the binary. Thanks to Anton Gladky (gladky.anton at gmail.com) for setting up this Ubuntu package capability. @@ -103,14 +113,14 @@ linking to the C library interface (lammps-devel, lammps-mpich-devel, lammps-openmpi-devel), the header for compiling programs using the C library interface (lammps-headers), and the LAMMPS python module for Python 3. All packages can be installed at the same -time and the name of the LAMMPS executable is *lmp* and *lmp_openmpi* -or *lmp_mpich* respectively. By default, *lmp* will refer to the +time and the name of the LAMMPS executable is ``lmp`` and ``lmp_openmpi`` +or ``lmp_mpich`` respectively. By default, ``lmp`` will refer to the serial executable, unless one of the MPI environment modules is loaded -("module load mpi/mpich-x86_64" or "module load mpi/openmpi-x86_64"). +(``module load mpi/mpich-x86_64`` or ``module load mpi/openmpi-x86_64``). Then the corresponding parallel LAMMPS executable can be used. The same mechanism applies when loading the LAMMPS python module. -To install LAMMPS with OpenMPI and run an input in.lj with 2 CPUs do: +To install LAMMPS with OpenMPI and run an input ``in.lj`` with 2 CPUs do: .. code-block:: bash @@ -118,10 +128,10 @@ To install LAMMPS with OpenMPI and run an input in.lj with 2 CPUs do: $ module load mpi/openmpi-x86_64 $ mpirun -np 2 lmp -in in.lj -The "dnf install" command is needed only once. In case of a new LAMMPS -stable release, "dnf update" will automatically update to the newer +The ``dnf install`` command is needed only once. In case of a new LAMMPS +stable release, ``dnf update`` will automatically update to the newer version as soon at the RPM files are built and uploaded to the download -mirrors. The "module load" command is needed once per (shell) session +mirrors. The ``module load`` command is needed once per (shell) session or shell terminal instance, unless it is automatically loaded from the shell profile. @@ -134,7 +144,7 @@ can install the `openkim-models` package $ dnf install openkim-models -Please use "lmp -help" to see which compilation options, packages, +Please use ``lmp -help`` to see which compilation options, packages, and styles are included in the binary. Thanks to Christoph Junghans (LANL) for making LAMMPS available in Fedora. @@ -153,10 +163,10 @@ in the `Extra Packages for Enterprise Linux (EPEL) repository ` which diff --git a/doc/src/Install_mac.rst b/doc/src/Install_mac.rst index 41b53c0595..642169ffe5 100644 --- a/doc/src/Install_mac.rst +++ b/doc/src/Install_mac.rst @@ -34,9 +34,9 @@ install the `openkim-models` package % brew install openkim-models If you have problems with the installation you can post issues to -`this link `_. +`this link `_. -.. _homebrew: https://github.com/Homebrew/homebrew-core/issues +.. _homebrew: https://brew.sh Thanks to Derek Thomas (derekt at cello.t.u-tokyo.ac.jp) for setting up the Homebrew capability. diff --git a/doc/src/JPG/bow_tutorial_01.png b/doc/src/JPG/bow_tutorial_01.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_01_small.png b/doc/src/JPG/bow_tutorial_01_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_02.png b/doc/src/JPG/bow_tutorial_02.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_02_small.png b/doc/src/JPG/bow_tutorial_02_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_03.png b/doc/src/JPG/bow_tutorial_03.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_03_small.png b/doc/src/JPG/bow_tutorial_03_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_04.png b/doc/src/JPG/bow_tutorial_04.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_04_small.png b/doc/src/JPG/bow_tutorial_04_small.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_05.png b/doc/src/JPG/bow_tutorial_05.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_06.png b/doc/src/JPG/bow_tutorial_06.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_07.png b/doc/src/JPG/bow_tutorial_07.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_08.png b/doc/src/JPG/bow_tutorial_08.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_09.png b/doc/src/JPG/bow_tutorial_09.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/bow_tutorial_10.png b/doc/src/JPG/bow_tutorial_10.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/ccmake-config.png b/doc/src/JPG/ccmake-config.png new file mode 100644 index 0000000000..ef75089646 Binary files /dev/null and b/doc/src/JPG/ccmake-config.png differ diff --git a/doc/src/JPG/ccmake-initial.png b/doc/src/JPG/ccmake-initial.png new file mode 100644 index 0000000000..d6291afde5 Binary files /dev/null and b/doc/src/JPG/ccmake-initial.png differ diff --git a/doc/src/JPG/ccmake-options.png b/doc/src/JPG/ccmake-options.png new file mode 100644 index 0000000000..b05d58c1d2 Binary files /dev/null and b/doc/src/JPG/ccmake-options.png differ diff --git a/doc/src/JPG/cmake-codeblocks.png b/doc/src/JPG/cmake-codeblocks.png new file mode 100644 index 0000000000..d0d96fec8c Binary files /dev/null and b/doc/src/JPG/cmake-codeblocks.png differ diff --git a/doc/src/JPG/cmake-gui-initial.png b/doc/src/JPG/cmake-gui-initial.png new file mode 100644 index 0000000000..062bc6ba83 Binary files /dev/null and b/doc/src/JPG/cmake-gui-initial.png differ diff --git a/doc/src/JPG/cmake-gui-options.png b/doc/src/JPG/cmake-gui-options.png new file mode 100644 index 0000000000..abbf634946 Binary files /dev/null and b/doc/src/JPG/cmake-gui-options.png differ diff --git a/doc/src/JPG/cmake-gui-popup.png b/doc/src/JPG/cmake-gui-popup.png new file mode 100644 index 0000000000..d623aad30b Binary files /dev/null and b/doc/src/JPG/cmake-gui-popup.png differ diff --git a/doc/src/JPG/offload_knc.png b/doc/src/JPG/offload_knc.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/rhodo_staggered.jpg b/doc/src/JPG/rhodo_staggered.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_ewald_intersect.jpg b/doc/src/JPG/saed_ewald_intersect.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_ewald_intersect_small.jpg b/doc/src/JPG/saed_ewald_intersect_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_mesh.jpg b/doc/src/JPG/saed_mesh.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/saed_mesh_small.jpg b/doc/src/JPG/saed_mesh_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/user_intel.png b/doc/src/JPG/user_intel.png old mode 100755 new mode 100644 diff --git a/doc/src/JPG/xrd_mesh.jpg b/doc/src/JPG/xrd_mesh.jpg old mode 100755 new mode 100644 diff --git a/doc/src/JPG/xrd_mesh_small.jpg b/doc/src/JPG/xrd_mesh_small.jpg old mode 100755 new mode 100644 diff --git a/doc/src/Modify_atom.rst b/doc/src/Modify_atom.rst index f863da3157..34a529020a 100644 --- a/doc/src/Modify_atom.rst +++ b/doc/src/Modify_atom.rst @@ -3,115 +3,169 @@ Atom styles Classes that define an :doc:`atom style ` are derived from the AtomVec class and managed by the Atom class. The atom style -determines what attributes are associated with an atom. A new atom -style can be created if one of the existing atom styles does not -define all the attributes you need to store and communicate with -atoms. +determines what attributes are associated with an atom and +communicated when it is a ghost atom or migrates to a new processor. +A new atom style can be created if one of the existing atom styles +does not define all the attributes you need to store and communicate +with atoms. -Atom_vec_atomic.cpp is a simple example of an atom style. +Atom_vec_atomic.cpp is the simplest example of an atom style. +Examining the code for others will make these instructions more clear. -Here is a brief description of methods you define in your new derived -class. See atom_vec.h for details. +Note that the :doc:`atom style hybrid ` command can be +used to define atoms or particles which have the union of properties +of individual styles. Also the :doc:`fix property/atom ` +command can be used to add a single property (e.g. charge +or a molecule ID) to a style that does not have it. It can also be +used to add custom properties to an atom, with options to communicate +them with ghost atoms or read them from a data file. Other LAMMPS +commands can access these custom properties, as can new pair, fix, +compute styles that are written to work with these properties. For +example, the :doc:`set ` command can be used to set the values of +custom per-atom properties from an input script. All of these methods +are less work than writing code for a new atom style. -+-------------------------+--------------------------------------------------------------------------------+ -| init | one time setup (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| grow | re-allocate atom arrays to longer lengths (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| grow_reset | make array pointers in Atom and AtomVec classes consistent (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| copy | copy info for one atom to another atom's array locations (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_comm | store an atom's info in a buffer communicated every timestep (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_comm_vel | add velocity info to communication buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_comm_hybrid | store extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_comm | retrieve an atom's info from the buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_comm_vel | also retrieve velocity info (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_comm_hybrid | retrieve extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_reverse | store an atom's info in a buffer communicating partial forces (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_reverse_hybrid | store extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_reverse | retrieve an atom's info from the buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_reverse_hybrid | retrieve extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_border | store an atom's info in a buffer communicated on neighbor re-builds (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_border_vel | add velocity info to buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_border_hybrid | store extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_border | retrieve an atom's info from the buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_border_vel | also retrieve velocity info (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_border_hybrid | retrieve extra info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_exchange | store all an atom's info to migrate to another processor (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_exchange | retrieve an atom's info from the buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| size_restart | number of restart quantities associated with proc's atoms (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| pack_restart | pack atom quantities into a buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| unpack_restart | unpack atom quantities from a buffer (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| create_atom | create an individual atom of this style (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| data_atom | parse an atom line from the data file (required) | -+-------------------------+--------------------------------------------------------------------------------+ -| data_atom_hybrid | parse additional atom info unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| data_vel | parse one line of velocity information from data file (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| data_vel_hybrid | parse additional velocity data unique to this atom style (optional) | -+-------------------------+--------------------------------------------------------------------------------+ -| memory_usage | tally memory allocated by atom arrays (required) | -+-------------------------+--------------------------------------------------------------------------------+ +If you follow these directions your new style will automatically work +in tandem with others via the :doc:`atom_style hybrid ` +command. -The constructor of the derived class sets values for several variables -that you must set when defining a new atom style, which are documented -in atom_vec.h. New atom arrays are defined in atom.cpp. Search for -the word "customize" and you will find locations you will need to -modify. +The first step is to define a set of strings in the constructor of the +new derived class. Each string will have zero or more space-separated +variable names which are identical to those used in the atom.h header +file for per-atom properties. Note that some represent per-atom +vectors (q, molecule) while other are per-atom arrays (x,v). For all +but the last 2 strings you do not need to specify any of +(id,type,x,v,f). Those are included automatically as needed in the +other strings. -.. note:: +.. list-table:: - It is possible to add some attributes, such as a molecule ID, to - atom styles that do not have them via the :doc:`fix property/atom ` command. This command also - allows new custom attributes consisting of extra integer or - floating-point values to be added to atoms. See the :doc:`fix property/atom ` doc page for examples of cases - where this is useful and details on how to initialize, access, and - output the custom values. + * - fields_grow + - full list of properties which is allocated and stored + * - fields_copy + - list of properties to copy atoms are rearranged on-processor + * - fields_comm + - list of properties communicated to ghost atoms every step + * - fields_comm_vel + - additional properties communicated if :doc:`comm_modify vel ` is used + * - fields_reverse + - list of properties summed from ghost atoms every step + * - fields_border + - list of properties communicated with ghost atoms every reneighboring step + * - fields_border_vel + - additional properties communicated if :doc:`comm_modify vel ` is used + * - fields_exchange + - list of properties communicated when an atom migrates to another processor + * - fields_restart + - list of properties written/read to/from a restart file + * - fields_create + - list of properties defined when an atom is created by :doc:`create_atoms ` + * - fields_data_atom + - list of properties (in order) in the Atoms section of a data file, as read by :doc:`read_data ` + * - fields_data_vel + - list of properties (in order) in the Velocities section of a data file, as read by :doc:`read_data ` -New :doc:`pair styles `, :doc:`fixes `, or -:doc:`computes ` can be added to LAMMPS, as discussed below. -The code for these classes can use the per-atom properties defined by -fix property/atom. The Atom class has a find_custom() method that is -useful in this context: +In these strings you can list variable names which LAMMPS already +defines (in some other atom style), or you can create new variable +names. You should not re-use a LAMMPS variable for something with +different meaning in your atom style. If the meaning is related, but +interpreted differently by your atom style, then using the same +variable name means a user should not use your style and the other +style together in a :doc:`atom_style hybrid ` command. +Because there will only be one value of the variable and different +parts of LAMMPS will then likely use it differently. LAMMPS has +no way of checking for this. -.. code-block:: c++ +If you are defining new variable names then make them descriptive and +unique to your new atom style. For example choosing "e" for energy is +a bad choice; it is too generic. A better choice would be "e_foo", +where "foo" is specific to your style. - int index = atom->find_custom(char *name, int &flag); +If any of the variable names in your new atom style do not exist in +LAMMPS, you need to add them to the src/atom.h and atom.cpp files. -The "name" of a custom attribute, as specified in the :doc:`fix property/atom ` command, is checked to verify -that it exists and its index is returned. The method also sets flag = -0/1 depending on whether it is an integer or floating-point attribute. -The vector of values associated with the attribute can then be -accessed using the returned index as +Search for the word "customize" or "customization" in these 2 files to +see where to add your variable. Adding a flag to the 2nd +customization section in atom.h is only necessary if your code (e.g. a +pair style) needs to check that a per-atom property is defined. These +flags should also be set in the constructor of the atom style child +class. -.. code-block:: c++ +In atom.cpp, aside from the constructor and destructor, there are 3 +methods that a new variable name or flag needs to be added to. - int *ivector = atom->ivector[index]; - double *dvector = atom->dvector[index]; +In Atom::peratom_create() when using the add_peratom() method, a +final length argument of 0 is for per-atom vectors, a length > 1 is +for per-atom arrays. Note the use of an extra per-thread flag and the +add_peratom_vary() method when last dimension of the array is +variable-length. -Ivector or dvector are vectors of length Nlocal = # of owned atoms, -which store the attributes of individual atoms. +Adding the variable name to Atom::extract() enable the per-atom data +to be accessed through the :doc:`LAMMPS library interface +` by a calling code, including from :doc:`Python +`. + +The constructor of the new atom style will also typically set a few +flags which are defined at the top of atom_vec.h. If these are +unclear, see how other atom styles use them. + +The grow_pointers() method is also required to make +a copy of peratom data pointers, as explained in the code. + +There are a number of other optional methods which your atom style can +implement. These are only needed if you need to do something +out-of-the-ordinary which the default operation of the AtomVec parent +class does not take care of. The best way to figure out why they are +sometimes useful is to look at how other atom styles use them. + +* process_args = use if the atom style has arguments +* init = called before each run +* force_clear = called before force computations each timestep + +A few atom styles define "bonus" data associated with some or all of +their particles, such as :doc:`atom_style ellipsoid or tri +`. These methods work with that data: + +* copy_bonus +* clear_bonus +* pack_comm_bonus +* unpack_comm_bonus +* pack_border_bonus +* unpack_border_bonus +* pack_exchange_bonus +* unpack_exchange_bonus +* size_restart_bonus +* pack_restart_bonus +* unpack_restart_bonus +* data_atom_bonus +* memory_usage_bonus + +The :doc:`atom_style body ` command can define a particle +geometry with an arbitrary number of values. This method reads it +from a data file: + +* data_body + +These methods are called before or after operations handled by the +parent AtomVec class. They allow an atom style to do customized +operations on the per-atom values. For example :doc:`atom_style +sphere ` reads a diameter and density of each particle +from a data file. But these need to be converted internally to a +radius and mass. That operation is done in the data_atom_post() +method. + +* pack_restart_pre +* pack_restart_post +* unpack_restart_init +* create_atom_post +* data_atom_post +* pack_data_pre +* pack_data_post + +These methods enable the :doc:`compute property/atom ` +command to access per-atom variables it does not +already define as arguments, so that they can be written to a dump +file or used by other LAMMPS commands. + +* property_atom +* pack_property_atom diff --git a/doc/src/Packages_details.rst b/doc/src/Packages_details.rst index e83da43116..2f99fc2bbc 100644 --- a/doc/src/Packages_details.rst +++ b/doc/src/Packages_details.rst @@ -366,12 +366,17 @@ KIM package This package contains a set of commands that serve as a wrapper on the `Open Knowledgebase of Interatomic Models (OpenKIM) `_ -repository of interatomic models (IMs) -enabling compatible ones to be used in LAMMPS simulations. -This includes :doc:`kim_init and kim_interactions ` -commands to select, initialize and instantiate the IM, and a -:doc:`kim_query ` command to perform web queries -for material property predictions of OpenKIM IMs. +repository of interatomic models (IMs) enabling compatible ones to be used in +LAMMPS simulations. + +This includes :doc:`kim_init `, and +:doc:`kim_interactions ` commands to select, initialize and +instantiate the IM, a :doc:`kim_query ` command to perform web +queries for material property predictions of OpenKIM IMs, a +:doc:`kim_param ` command to access KIM Model Parameters from +LAMMPS, and a :doc:`kim_property ` command to write material +properties computed in LAMMPS to standard KIM property instance format. + Support for KIM IMs that conform to the `KIM Application Programming Interface (API) `_ is provided by the :doc:`pair_style kim ` command. @@ -393,13 +398,16 @@ The KIM project is led by Ellad Tadmor and Ryan Elliott (U Minnesota) and is funded by the `National Science Foundation `_. **Authors:** Ryan Elliott (U Minnesota) is the main developer for the KIM -API and the *pair_style kim* command. Axel Kohlmeyer (Temple U) and -Ellad Tadmor (U Minnesota) contributed to the :doc:`kim_commands ` -interface in close collaboration with Ryan Elliott. +API and the *pair_style kim* command. Yaser Afshar (U Minnesota), +Axel Kohlmeyer (Temple U), Ellad Tadmor (U Minnesota), and +Daniel Karls (U Minnesota) contributed to the +:doc:`kim_commands ` interface in close collaboration with +Ryan Elliott. **Install:** -This package has :ref:`specific installation instructions ` on the :doc:`Build extras ` doc page. +This package has :ref:`specific installation instructions ` on the +:doc:`Build extras ` doc page. **Supporting info:** diff --git a/doc/src/Python_shlib.rst b/doc/src/Python_shlib.rst index 0c86362e12..b02fcd7bb8 100644 --- a/doc/src/Python_shlib.rst +++ b/doc/src/Python_shlib.rst @@ -1,6 +1,8 @@ Build LAMMPS as a shared library ================================ +.. TODO this is mostly redundant and should be addressed in the 'progguide' branch if it has not already + Build LAMMPS as a shared library using make ------------------------------------------- @@ -14,7 +16,7 @@ From the src directory, type .. code-block:: bash - make foo mode=shlib + make foo mode=shared where foo is the machine target name, such as mpi or serial. This should create the file liblammps_foo.so in the src directory, as @@ -39,7 +41,6 @@ shared library: .. code-block:: bash - -D BUILD_LIB=on # enable building LAMMPS as a library -D BUILD_SHARED_LIBS=on # enable building of LAMMPS shared library (both options are needed!) What this does is create a liblammps.so which contains the majority of LAMMPS @@ -59,7 +60,7 @@ CMAKE_INSTALL_PREFIX. # build library mkdir build cd build - cmake -D PKG_PYTHON=on -D BUILD_LIB=on -D BUILD_SHARED_LIBS=on -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ../cmake + cmake -D PKG_PYTHON=on -D BUILD_SHARED_LIBS=on -D CMAKE_INSTALL_PREFIX=$VIRTUAL_ENV ../cmake make -j 4 # install into prefix diff --git a/doc/src/Speed_compare.rst b/doc/src/Speed_compare.rst index d3947ec3ef..4ab616a5da 100644 --- a/doc/src/Speed_compare.rst +++ b/doc/src/Speed_compare.rst @@ -87,7 +87,7 @@ section below for examples where this has been done. * The GPU package requires neighbor lists to be built on the CPU when using exclusion lists, or a triclinic simulation box. * The GPU package can be compiled for CUDA or OpenCL and thus supports - both, Nvidia and AMD GPUs well. On Nvidia hardware, using CUDA is typically + both, NVIDIA and AMD GPUs well. On NVIDIA hardware, using CUDA is typically resulting in equal or better performance over OpenCL. * OpenCL in the GPU package does theoretically also support Intel CPUs or Intel Xeon Phi, but the native support for those in KOKKOS (or USER-INTEL) diff --git a/doc/src/Speed_kokkos.rst b/doc/src/Speed_kokkos.rst index 6658957006..3269564d21 100644 --- a/doc/src/Speed_kokkos.rst +++ b/doc/src/Speed_kokkos.rst @@ -9,10 +9,7 @@ different back end languages such as CUDA, OpenMP, or Pthreads. The Kokkos library also provides data abstractions to adjust (at compile time) the memory layout of data structures like 2d and 3d arrays to optimize performance on different hardware. For more information on -Kokkos, see `GitHub `_. Kokkos is -part of `Trilinos `_. The Kokkos -library was written primarily by Carter Edwards, Christian Trott, and -Dan Sunderland (all Sandia). +Kokkos, see `GitHub `_. The LAMMPS KOKKOS package contains versions of pair, fix, and atom styles that use data structures and macros provided by the Kokkos @@ -21,7 +18,7 @@ package was developed primarily by Christian Trott (Sandia) and Stan Moore (Sandia) with contributions of various styles by others, including Sikandar Mashayak (UIUC), Ray Shan (Sandia), and Dan Ibanez (Sandia). For more information on developing using Kokkos abstractions -see the Kokkos programmers' guide at /lib/kokkos/doc/Kokkos_PG.pdf. +see the Kokkos `Wiki `_. Kokkos currently provides support for 3 modes of execution (per MPI task). These are Serial (MPI-only for CPUs and Intel Phi), OpenMP @@ -31,33 +28,30 @@ compatible with specific hardware. .. note:: - Kokkos support within LAMMPS must be built with a C++11 compatible - compiler. This means GCC version 4.7.2 or later, Intel 14.0.4 or later, or - Clang 3.5.2 or later is required. + To build with Kokkos support for NVIDIA GPUs, the NVIDIA CUDA toolkit + software version 9.0 or later must be installed on your system. See + the discussion for the :doc:`GPU package ` for details of + how to check and do this. .. note:: - To build with Kokkos support for NVIDIA GPUs, NVIDIA CUDA - software version 7.5 or later must be installed on your system. See - the discussion for the :doc:`GPU package ` for details of how - to check and do this. + Kokkos with CUDA currently implicitly assumes that the MPI library is + CUDA-aware. This is not always the case, especially when using + pre-compiled MPI libraries provided by a Linux distribution. This is + not a problem when using only a single GPU with a single MPI + rank. When running with multiple MPI ranks, you may see segmentation + faults without CUDA-aware MPI support. These can be avoided by adding + the flags :doc:`-pk kokkos cuda/aware off ` to the + LAMMPS command line or by using the command :doc:`package kokkos + cuda/aware off ` in the input file. -.. note:: - - Kokkos with CUDA currently implicitly assumes that the MPI library - is CUDA-aware. This is not always the case, especially when using - pre-compiled MPI libraries provided by a Linux distribution. This is not - a problem when using only a single GPU with a single MPI rank. When - running with multiple MPI ranks, you may see segmentation faults without - CUDA-aware MPI support. These can be avoided by adding the flags :doc:`-pk kokkos cuda/aware off ` to the LAMMPS command line or by - using the command :doc:`package kokkos cuda/aware off ` in the - input file. - -**Building LAMMPS with the KOKKOS package:** +Building LAMMPS with the KOKKOS package +""""""""""""""""""""""""""""""""""""""" See the :ref:`Build extras ` doc page for instructions. -**Running LAMMPS with the KOKKOS package:** +Running LAMMPS with the KOKKOS package +"""""""""""""""""""""""""""""""""""""" All Kokkos operations occur within the context of an individual MPI task running on a single node of the machine. The total number of MPI @@ -66,7 +60,8 @@ usual manner via the mpirun or mpiexec commands, and is independent of Kokkos. E.g. the mpirun command in OpenMPI does this via its -np and -npernode switches. Ditto for MPICH via -np and -ppn. -**Running on a multi-core CPU:** +Running on a multi-core CPU +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Here is a quick overview of how to use the KOKKOS package for CPU acceleration, assuming one or more 16-core nodes. @@ -142,7 +137,8 @@ atom. When using the Kokkos Serial back end or the OpenMP back end with a single thread, no duplication or atomic operations are used. For CUDA and half neighbor lists, the KOKKOS package always uses atomic operations. -**Core and Thread Affinity:** +CPU Cores, Sockets and Thread Affinity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When using multi-threading, it is important for performance to bind both MPI tasks to physical cores, and threads to physical cores, so @@ -156,15 +152,16 @@ for your MPI installation), binding can be forced with these flags: OpenMPI 1.8: mpirun -np 2 --bind-to socket --map-by socket ./lmp_openmpi ... Mvapich2 2.0: mpiexec -np 2 --bind-to socket --map-by socket ./lmp_mvapich ... -For binding threads with KOKKOS OpenMP, use thread affinity -environment variables to force binding. With OpenMP 3.1 (gcc 4.7 or -later, intel 12 or later) setting the environment variable -OMP_PROC_BIND=true should be sufficient. In general, for best -performance with OpenMP 4.0 or better set OMP_PROC_BIND=spread and -OMP_PLACES=threads. For binding threads with the KOKKOS pthreads -option, compile LAMMPS the KOKKOS HWLOC=yes option as described below. +For binding threads with KOKKOS OpenMP, use thread affinity environment +variables to force binding. With OpenMP 3.1 (gcc 4.7 or later, intel 12 +or later) setting the environment variable ``OMP_PROC_BIND=true`` should +be sufficient. In general, for best performance with OpenMP 4.0 or later +set ``OMP_PROC_BIND=spread`` and ``OMP_PLACES=threads``. For binding +threads with the KOKKOS pthreads option, compile LAMMPS with the hwloc +or libnuma support enabled as described in the :ref:`extra build options page `. -**Running on Knight's Landing (KNL) Intel Xeon Phi:** +Running on Knight's Landing (KNL) Intel Xeon Phi +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Here is a quick overview of how to use the KOKKOS package for the Intel Knight's Landing (KNL) Xeon Phi: @@ -222,7 +219,8 @@ threads/task as Nt. The product of these two values should be N, i.e. them in "native" mode, not "offload" mode like the USER-INTEL package supports. -**Running on GPUs:** +Running on GPUs +^^^^^^^^^^^^^^^ Use the "-k" :doc:`command-line switch ` to specify the number of GPUs per node. Typically the -np setting of the mpirun command @@ -257,7 +255,7 @@ one or more nodes, each with two GPUs: running on GPUs is to use "full" neighbor lists and set the Newton flag to "off" for both pairwise and bonded interactions, along with threaded communication. When running on Maxwell or Kepler GPUs, this will - typically be best. For Pascal GPUs, using "half" neighbor lists and + typically be best. For Pascal GPUs and beyond, using "half" neighbor lists and setting the Newton flag to "on" may be faster. For many pair styles, setting the neighbor binsize equal to twice the CPU default value will give speedup, which is the default when running on GPUs. Use the "-pk @@ -270,13 +268,6 @@ one or more nodes, each with two GPUs: mpirun -np 2 lmp_kokkos_cuda_openmpi -k on g 2 -sf kk -pk kokkos newton on neigh half binsize 2.8 -in in.lj # Newton on, half neighbor list, set binsize = neighbor ghost cutoff -.. note:: - - For good performance of the KOKKOS package on GPUs, you must - have Kepler generation GPUs (or later). The Kokkos library exploits - texture cache options not supported by Telsa generation GPUs (or - older). - .. note:: When using a GPU, you will achieve the best performance if your @@ -293,7 +284,8 @@ one or more nodes, each with two GPUs: kspace, etc., you must set the environment variable CUDA_LAUNCH_BLOCKING=1. However, this will reduce performance and is not recommended for production runs. -**Run with the KOKKOS package by editing an input script:** +Run with the KOKKOS package by editing an input script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Alternatively the effect of the "-sf" or "-pk" switches can be duplicated by adding the :doc:`package kokkos ` or :doc:`suffix kk ` commands to your input script. @@ -316,17 +308,24 @@ You only need to use the :doc:`package kokkos ` command if you wish to change any of its option defaults, as set by the "-k on" :doc:`command-line switch `. -**Using OpenMP threading and CUDA together (experimental):** +**Using OpenMP threading and CUDA together:** With the KOKKOS package, both OpenMP multi-threading and GPUs can be -used together in a few special cases. In the Makefile, the -KOKKOS_DEVICES variable must include both "Cuda" and "OpenMP", as is -the case for /src/MAKE/OPTIONS/Makefile.kokkos_cuda_mpi +compiled and used together in a few special cases. In the makefile for +the conventional build, the KOKKOS_DEVICES variable must include both, +"Cuda" and "OpenMP", as is the case for ``/src/MAKE/OPTIONS/Makefile.kokkos_cuda_mpi``. .. code-block:: bash KOKKOS_DEVICES=Cuda,OpenMP +When building with CMake you need to enable both features as it is done +in the ``kokkos-cuda.cmake`` CMake preset file. + +.. code-block:: bash + + cmake ../cmake -DKokkos_ENABLE_CUDA=yes -DKokkos_ENABLE_OPENMP=yes + The suffix "/kk" is equivalent to "/kk/device", and for Kokkos CUDA, using the "-sf kk" in the command line gives the default CUDA version everywhere. However, if the "/kk/host" suffix is added to a specific @@ -360,7 +359,8 @@ suffix for kspace and bonds, angles, etc. in the input file and the sure the environment variable CUDA_LAUNCH_BLOCKING is not set to "1" so CPU/GPU overlap can occur. -**Speed-ups to expect:** +Performance to expect +""""""""""""""""""""" The performance of KOKKOS running in different modes is a function of your hardware, which KOKKOS-enable styles are used, and the problem @@ -377,52 +377,26 @@ Generally speaking, the following rules of thumb apply: performance of a KOKKOS style is a bit slower than the USER-OMP package. * When running large number of atoms per GPU, KOKKOS is typically faster - than the GPU package. + than the GPU package when compiled for double precision. The benefit + of using single or mixed precision with the GPU package depends + significantly on the hardware in use and the simulated system and pair + style. * When running on Intel hardware, KOKKOS is not as fast as - the USER-INTEL package, which is optimized for that hardware. + the USER-INTEL package, which is optimized for x86 hardware (not just + from Intel) and compilation with the Intel compilers. The USER-INTEL + package also can increase the vector length of vector instructions + by switching to single or mixed precision mode. See the `Benchmark page `_ of the LAMMPS web site for performance of the KOKKOS package on different hardware. -**Advanced Kokkos options:** +Advanced Kokkos options +""""""""""""""""""""""" -There are other allowed options when building with the KOKKOS package. -As explained on the :ref:`Build extras ` doc page, -they can be set either as variables on the make command line or in -Makefile.machine, or they can be specified as CMake variables. Each -takes a value shown below. The default value is listed, which is set -in the lib/kokkos/Makefile.kokkos file. - -* KOKKOS_DEBUG, values = *yes*\ , *no*\ , default = *no* -* KOKKOS_USE_TPLS, values = *hwloc*\ , *librt*\ , *experimental_memkind*, default = *none* -* KOKKOS_CXX_STANDARD, values = *c++11*\ , *c++1z*\ , default = *c++11* -* KOKKOS_OPTIONS, values = *aggressive_vectorization*, *disable_profiling*, default = *none* -* KOKKOS_CUDA_OPTIONS, values = *force_uvm*, *use_ldg*, *rdc*\ , *enable_lambda*, default = *enable_lambda* - -KOKKOS_USE_TPLS=hwloc binds threads to hardware cores, so they do not -migrate during a simulation. KOKKOS_USE_TPLS=hwloc should always be -used if running with KOKKOS_DEVICES=Pthreads for pthreads. It is not -necessary for KOKKOS_DEVICES=OpenMP for OpenMP, because OpenMP -provides alternative methods via environment variables for binding -threads to hardware cores. More info on binding threads to cores is -given on the :doc:`Speed omp ` doc page. - -KOKKOS_USE_TPLS=librt enables use of a more accurate timer mechanism -on most Unix platforms. This library is not available on all -platforms. - -KOKKOS_DEBUG is only useful when developing a Kokkos-enabled style -within LAMMPS. KOKKOS_DEBUG=yes enables printing of run-time -debugging information that can be useful. It also enables runtime -bounds checking on Kokkos data structures. - -KOKKOS_CXX_STANDARD and KOKKOS_OPTIONS are typically not changed when -building LAMMPS. - -KOKKOS_CUDA_OPTIONS are additional options for CUDA. The LAMMPS KOKKOS -package must be compiled with the *enable_lambda* option when using -GPUs. +There are other allowed options when building with the KOKKOS package +that can improve performance or assist in debugging or profiling. +They are explained on the :ref:`KOKKOS section of the build extras ` doc page, Restrictions """""""""""" diff --git a/doc/src/Speed_packages.rst b/doc/src/Speed_packages.rst index ab02ba7f48..685bd3c601 100644 --- a/doc/src/Speed_packages.rst +++ b/doc/src/Speed_packages.rst @@ -20,7 +20,7 @@ standard or user packages: +-----------------------------------------+-------------------------------------------------------+ | :doc:`USER-INTEL Package ` | for Intel CPUs and Intel Xeon Phi | +-----------------------------------------+-------------------------------------------------------+ -| :doc:`KOKKOS Package ` | for Nvidia GPUs, Intel Xeon Phi, and OpenMP threading | +| :doc:`KOKKOS Package ` | for NVIDIA GPUs, Intel Xeon Phi, and OpenMP threading | +-----------------------------------------+-------------------------------------------------------+ | :doc:`USER-OMP Package ` | for OpenMP threading and generic CPU optimizations | +-----------------------------------------+-------------------------------------------------------+ diff --git a/doc/src/atom_style.rst b/doc/src/atom_style.rst index fe368326ce..4957901676 100644 --- a/doc/src/atom_style.rst +++ b/doc/src/atom_style.rst @@ -10,7 +10,7 @@ Syntax atom_style style args -* style = *angle* or *atomic* or *body* or *bond* or *charge* or *dipole* or *dpd* or *edpd* or *mdpd* or *tdpd* or *electron* or *ellipsoid* or *full* or *line* or *meso* or *molecular* or *peri* or *smd* or *sphere* or *spin* or *tri* or *template* or *hybrid* +* style = *angle* or *atomic* or *body* or *bond* or *charge* or *dipole* or *dpd* or *edpd* or *electron* or *ellipsoid* or *full* or *line* or *mdpd* or *molecular* or *peri* or *smd* or *sph* or *sphere* or *spin* or *tdpd* or *tri* or *template* or *hybrid* .. parsed-literal:: @@ -18,7 +18,9 @@ Syntax *body* args = bstyle bstyle-args bstyle = style of body particles bstyle-args = additional arguments specific to the bstyle - see the :doc:`Howto body ` doc page for details + see the :doc:`Howto body ` doc + page for details + *sphere* arg = 0/1 (optional) for static/dynamic particle radii *tdpd* arg = Nspecies Nspecies = # of chemical species *template* arg = template-ID @@ -91,10 +93,6 @@ quantities. +--------------+-----------------------------------------------------+--------------------------------------+ | *edpd* | temperature and heat capacity | eDPD particles | +--------------+-----------------------------------------------------+--------------------------------------+ -| *mdpd* | density | mDPD particles | -+--------------+-----------------------------------------------------+--------------------------------------+ -| *tdpd* | chemical concentration | tDPD particles | -+--------------+-----------------------------------------------------+--------------------------------------+ | *electron* | charge and spin and eradius | electronic force field | +--------------+-----------------------------------------------------+--------------------------------------+ | *ellipsoid* | shape, quaternion, angular momentum | aspherical particles | @@ -103,7 +101,7 @@ quantities. +--------------+-----------------------------------------------------+--------------------------------------+ | *line* | end points, angular velocity | rigid bodies | +--------------+-----------------------------------------------------+--------------------------------------+ -| *meso* | rho, e, cv | SPH particles | +| *mdpd* | density | mDPD particles | +--------------+-----------------------------------------------------+--------------------------------------+ | *mesont* | mass, radius, length, buckling, connections, tube id| mesoscopic nanotubes | +--------------+-----------------------------------------------------+--------------------------------------+ @@ -113,10 +111,14 @@ quantities. +--------------+-----------------------------------------------------+--------------------------------------+ | *smd* | volume, kernel diameter, contact radius, mass | solid and fluid SPH particles | +--------------+-----------------------------------------------------+--------------------------------------+ +| *sph* | rho, esph, cv | SPH particles | ++--------------+-----------------------------------------------------+--------------------------------------+ | *sphere* | diameter, mass, angular velocity | granular models | +--------------+-----------------------------------------------------+--------------------------------------+ | *spin* | magnetic moment | system with magnetic particles | +--------------+-----------------------------------------------------+--------------------------------------+ +| *tdpd* | chemical concentration | tDPD particles | ++--------------+-----------------------------------------------------+--------------------------------------+ | *template* | template index, template atom | small molecules with fixed topology | +--------------+-----------------------------------------------------+--------------------------------------+ | *tri* | corner points, angular momentum | rigid bodies | @@ -146,9 +148,16 @@ basis. For the *sphere* style, the particles are spheres and each stores a per-particle diameter and mass. If the diameter > 0.0, the particle is a finite-size sphere. If the diameter = 0.0, it is a point -particle. Note that by use of the *disc* keyword with the :doc:`fix nve/sphere `, :doc:`fix nvt/sphere `, -:doc:`fix nph/sphere `, :doc:`fix npt/sphere ` commands, spheres can be effectively -treated as 2d discs for a 2d simulation if desired. See also the :doc:`set density/disc ` command. +particle. Note that by use of the *disc* keyword with the :doc:`fix +nve/sphere `, :doc:`fix nvt/sphere `, +:doc:`fix nph/sphere `, :doc:`fix npt/sphere +` commands, spheres can be effectively treated as 2d +discs for a 2d simulation if desired. See also the :doc:`set +density/disc ` command. The *sphere* style takes an optional 0 +or 1 argument. A value of 0 means the radius of each sphere is +constant for the duration of the simulation. A value of 1 means the +radii may vary dynamically during the simulation, e.g. due to use of +the :doc:`fix adapt ` command. For the *ellipsoid* style, the particles are ellipsoids and each stores a flag which indicates whether it is a finite-size ellipsoid or @@ -191,8 +200,8 @@ particles which store a set of chemical concentration. An integer "cc_species" is required to specify the number of chemical species involved in a tDPD system. -The *meso* style is for smoothed particle hydrodynamics (SPH) -particles which store a density (rho), energy (e), and heat capacity +The *sph* style is for smoothed particle hydrodynamics (SPH) +particles which store a density (rho), energy (esph), and heat capacity (cv). The *smd* style is for a general formulation of Smooth Particle @@ -337,7 +346,7 @@ for energy-conserving dissipative particle dynamics (eDPD), many-body dissipative particle dynamics (mDPD), and transport dissipative particle dynamics (tDPD), respectively. -The *meso* style is part of the USER-SPH package for smoothed particle +The *sph* style is part of the USER-SPH package for smoothed particle hydrodynamics (SPH). See `this PDF guide `_ to using SPH in LAMMPS. The *mesont* style is part of the USER-MESONT package. @@ -355,7 +364,8 @@ Related commands Default """"""" -atom_style atomic +The default atom style is atomic. If atom_style sphere is used its +default argument is 0. ---------- diff --git a/doc/src/balance.rst b/doc/src/balance.rst index 5e631fdd27..5873724924 100644 --- a/doc/src/balance.rst +++ b/doc/src/balance.rst @@ -175,23 +175,27 @@ a logical 3d grid of processors. They operate by changing the cutting planes (or lines) between processors in 3d (or 2d), to adjust the volume (area in 2d) assigned to each processor, as in the following 2d diagram where processor sub-domains are shown and particles are -colored by the processor that owns them. The leftmost diagram is the -default partitioning of the simulation box across processors (one -sub-box for each of 16 processors); the middle diagram is after a -"grid" method has been applied. +colored by the processor that owns them. -.. image:: JPG/balance_uniform_small.jpg - :target: JPG/balance_uniform.jpg -.. image:: JPG/balance_nonuniform_small.jpg - :target: JPG/balance_nonuniform.jpg -.. image:: JPG/balance_rcb_small.jpg - :target: JPG/balance_rcb.jpg +.. list-table:: -The *rcb* style is a "tiling" method which does not produce a logical -3d grid of processors. Rather it tiles the simulation domain with -rectangular sub-boxes of varying size and shape in an irregular -fashion so as to have equal numbers of particles (or weight) in each -sub-box, as in the rightmost diagram above. + * - .. figure:: JPG/balance_uniform_small.jpg + :target: JPG/balance_uniform.jpg + + - .. figure:: JPG/balance_nonuniform_small.jpg + :target: JPG/balance_nonuniform.jpg + + - .. figure:: JPG/balance_rcb_small.jpg + :target: JPG/balance_rcb.jpg + +The leftmost diagram is the default partitioning of the simulation box +across processors (one sub-box for each of 16 processors); the middle +diagram is after a "grid" method has been applied. The *rcb* style is a +"tiling" method which does not produce a logical 3d grid of processors. +Rather it tiles the simulation domain with rectangular sub-boxes of +varying size and shape in an irregular fashion so as to have equal +numbers of particles (or weight) in each sub-box, as in the rightmost +diagram above. The "grid" methods can be used with either of the :doc:`comm_style ` command options, *brick* or *tiled*\ . The diff --git a/doc/src/change_box.rst b/doc/src/change_box.rst index 2ab70ddec8..a21f74e93b 100644 --- a/doc/src/change_box.rst +++ b/doc/src/change_box.rst @@ -142,24 +142,40 @@ new owning processors. .. note:: - The simulation box size/shape can be changed by arbitrarily - large amounts by this command. This is not a problem, except that the + The simulation box size/shape can be changed by arbitrarily large + amounts by this command. This is not a problem, except that the mapping of processors to the simulation box is not changed from its initial 3d configuration; see the :doc:`processors ` command. Thus, if the box size/shape changes dramatically, the - mapping of processors to the simulation box may not end up as optimal - as the initial mapping attempted to be. + mapping of processors to the simulation box may not end up as + optimal as the initial mapping attempted to be. You may wish to + re-balance the atoms by using the :doc:`balance ` command + if that is the case. .. note:: - Because the keywords used in this command are applied one at a - time to the simulation box and the atoms in it, care must be taken - with triclinic cells to avoid exceeding the limits on skew after each - transformation in the sequence. If skew is exceeded before the final - transformation this can be avoided by changing the order of the - sequence, or breaking the transformation into two or more smaller - transformations. For more information on the allowed limits for box - skew see the discussion on triclinic boxes on :doc:`Howto triclinic ` doc page. + You cannot use this command after reading a restart file (and + before a run is performed) if the restart file stored per-atom + information from a fix and any of the specified keywords change the + box size or shape or boundary conditions. This is because atoms + may be moved to new processors and the restart info will not + migrate with them. LAMMPS will generate an error if this could + happen. Only the *ortho* and *triclinic* keywords do not trigger + this error. One solution is to perform a "run 0" command before + using the change_box command. This clears the per-atom restart + data, whether it has been re-assigned to a new fix or not. + +.. note:: + + Because the keywords used in this command are applied one at a time + to the simulation box and the atoms in it, care must be taken with + triclinic cells to avoid exceeding the limits on skew after each + transformation in the sequence. If skew is exceeded before the + final transformation this can be avoided by changing the order of + the sequence, or breaking the transformation into two or more + smaller transformations. For more information on the allowed + limits for box skew see the discussion on triclinic boxes on + :doc:`Howto triclinic ` doc page. ---------- @@ -283,6 +299,12 @@ match what is stored in the restart file. So if you wish to change them, you should use the change_box command after the read_restart command. +.. note:: + + Changing a periodic boundary to a non-periodic one will also + cause the image flag for that dimension to be reset to 0 for + all atoms. LAMMPS will print a warning message, if that happens. + ---------- The *ortho* and *triclinic* keywords convert the simulation box to be diff --git a/doc/src/compute.rst b/doc/src/compute.rst index bcf53aecdf..754cc3e1aa 100644 --- a/doc/src/compute.rst +++ b/doc/src/compute.rst @@ -225,9 +225,6 @@ The individual style names on the :doc:`Commands compute ` doc * :doc:`ke/atom/eff ` - per-atom translational and radial kinetic energy in the electron force field model * :doc:`ke/eff ` - kinetic energy of a group of nuclei and electrons in the electron force field model * :doc:`ke/rigid ` - translational kinetic energy of rigid bodies -* :doc:`meso/e/atom ` - per-atom internal energy of Smooth-Particle Hydrodynamics atoms -* :doc:`meso/rho/atom ` - per-atom mesoscopic density of Smooth-Particle Hydrodynamics atoms -* :doc:`meso/t/atom ` - per-atom internal temperature of Smooth-Particle Hydrodynamics atoms * :doc:`momentum ` - translational momentum * :doc:`msd ` - mean-squared displacement of group of atoms * :doc:`msd/chunk ` - mean-squared displacement for each chunk @@ -280,6 +277,9 @@ The individual style names on the :doc:`Commands compute ` doc * :doc:`sna/atom ` - bispectrum components for each atom * :doc:`snad/atom ` - derivative of bispectrum components for each atom * :doc:`snav/atom ` - virial contribution from bispectrum components for each atom +* :doc:`sph/e/atom ` - per-atom internal energy of Smooth-Particle Hydrodynamics atoms +* :doc:`sph/rho/atom ` - per-atom density of Smooth-Particle Hydrodynamics atoms +* :doc:`sph/t/atom ` - per-atom internal temperature of Smooth-Particle Hydrodynamics atoms * :doc:`spin ` - magnetic quantities for a system of atoms having spins * :doc:`stress/atom ` - stress tensor for each atom * :doc:`stress/mop ` - normal components of the local stress tensor using the method of planes @@ -308,6 +308,7 @@ The individual style names on the :doc:`Commands compute ` doc * :doc:`torque/chunk ` - torque applied on each chunk * :doc:`vacf ` - velocity auto-correlation function of group of atoms * :doc:`vcm/chunk ` - velocity of center-of-mass for each chunk +* :doc:`viscosity/cos ` - velocity profile under cosine-shaped acceleration * :doc:`voronoi/atom ` - Voronoi volume and neighbors for each atom * :doc:`xrd ` - x-ray diffraction intensity on a mesh of reciprocal lattice nodes diff --git a/doc/src/compute_coord_atom.rst b/doc/src/compute_coord_atom.rst index c51ea212f0..b8f2b9e8ee 100644 --- a/doc/src/compute_coord_atom.rst +++ b/doc/src/compute_coord_atom.rst @@ -3,6 +3,9 @@ compute coord/atom command ========================== +compute coord/atom/kk command +=================================== + Syntax """""" @@ -109,6 +112,30 @@ too frequently. :doc:`special_bonds ` command that includes all pairs in the neighbor list. +---------- + + +Styles with a *gpu*\ , *intel*\ , *kk*\ , *omp*\ , or *opt* suffix are +functionally the same as the corresponding style without the suffix. +They have been optimized to run faster, depending on your available +hardware, as discussed on the :doc:`Speed packages ` doc +page. The accelerated styles take the same arguments and should +produce the same results, except for round-off and precision issues. + +These accelerated styles are part of the GPU, USER-INTEL, KOKKOS, +USER-OMP and OPT packages, respectively. They are only enabled if +LAMMPS was built with those packages. See the :doc:`Build package ` doc page for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the :doc:`-suffix command-line switch ` when you invoke LAMMPS, or you can use the +:doc:`suffix ` command in your input script. + +See the :doc:`Speed packages ` doc page for more +instructions on how to use the accelerated styles effectively. + + +---------- + **Output info:** For *cstyle* cutoff, this compute can calculate a per-atom vector or diff --git a/doc/src/compute_orientorder_atom.rst b/doc/src/compute_orientorder_atom.rst index 7b894b886e..3b432e7df9 100644 --- a/doc/src/compute_orientorder_atom.rst +++ b/doc/src/compute_orientorder_atom.rst @@ -3,6 +3,9 @@ compute orientorder/atom command ================================ +compute orientorder/atom/kk command +=================================== + Syntax """""" @@ -16,13 +19,14 @@ Syntax .. parsed-literal:: - keyword = *cutoff* or *nnn* or *degrees* or *components* + keyword = *cutoff* or *nnn* or *degrees* or *components* or *chunksize* *cutoff* value = distance cutoff *nnn* value = number of nearest neighbors *degrees* values = nlvalues, l1, l2,... *wl* value = yes or no *wl/hat* value = yes or no *components* value = ldegree + *chunksize* value = number of atoms in each pass Examples """""""" @@ -104,6 +108,14 @@ in conjunction with :doc:`compute coord_atom ` to calculate the ten Wolde's criterion to identify crystal-like particles, as discussed in :ref:`ten Wolde `. +The optional keyword *chunksize* is only applicable when using the +the KOKKOS package and is ignored otherwise. This keyword controls +the number of atoms in each pass used to compute the bond-orientational +order parameters and is used to avoid running out of memory. For example +if there are 4000 atoms in the simulation and the *chunksize* +is set to 2000, the parameter calculation will be broken up +into two passes. + The value of :math:`Q_l` is set to zero for atoms not in the specified compute group, as well as for atoms that have less than *nnn* neighbors within the distance cutoff, unless *nnn* is NULL. @@ -128,6 +140,30 @@ too frequently. :doc:`special_bonds ` command that includes all pairs in the neighbor list. +---------- + + +Styles with a *gpu*\ , *intel*\ , *kk*\ , *omp*\ , or *opt* suffix are +functionally the same as the corresponding style without the suffix. +They have been optimized to run faster, depending on your available +hardware, as discussed on the :doc:`Speed packages ` doc +page. The accelerated styles take the same arguments and should +produce the same results, except for round-off and precision issues. + +These accelerated styles are part of the GPU, USER-INTEL, KOKKOS, +USER-OMP and OPT packages, respectively. They are only enabled if +LAMMPS was built with those packages. See the :doc:`Build package ` doc page for more info. + +You can specify the accelerated styles explicitly in your input script +by including their suffix, or you can use the :doc:`-suffix command-line switch ` when you invoke LAMMPS, or you can use the +:doc:`suffix ` command in your input script. + +See the :doc:`Speed packages ` doc page for more +instructions on how to use the accelerated styles effectively. + + +---------- + **Output info:** This compute calculates a per-atom array with *nlvalues* columns, @@ -165,7 +201,7 @@ Default The option defaults are *cutoff* = pair style cutoff, *nnn* = 12, *degrees* = 5 4 6 8 10 12 i.e. :math:`Q_4`, :math:`Q_6`, :math:`Q_8`, :math:`Q_{10}`, and :math:`Q_{12}`, -*wl* = no, *wl/hat* = no, and *components* off +*wl* = no, *wl/hat* = no, *components* off, and *chunksize* = 2000 ---------- diff --git a/doc/src/compute_property_atom.rst b/doc/src/compute_property_atom.rst index 6b93bc0118..33bc573c41 100644 --- a/doc/src/compute_property_atom.rst +++ b/doc/src/compute_property_atom.rst @@ -69,8 +69,8 @@ Syntax .. parsed-literal:: PERI package per-atom properties: - vfrac = ??? - s0 = ??? + vfrac = volume fraction + s0 = max stretch of any bond a particle is part of .. parsed-literal:: @@ -83,11 +83,11 @@ Syntax .. parsed-literal:: USER-SPH package per-atom properties: - rho = ??? - drho = ??? - e = ??? - de = ??? - cv = ??? + rho = density of SPH particles + drho = change in density + e = energy + de = change in thermal energy + cv = heat capacity .. parsed-literal:: @@ -110,13 +110,17 @@ Description Define a computation that simply stores atom attributes for each atom in the group. This is useful so that the values can be used by other -:doc:`output commands ` that take computes as inputs. See -for example, the :doc:`compute reduce `, :doc:`fix ave/atom `, :doc:`fix ave/histo `, :doc:`fix ave/chunk `, and :doc:`atom-style variable ` -commands. +:doc:`output commands ` that take computes as inputs. +See for example, the :doc:`compute reduce `, :doc:`fix +ave/atom `, :doc:`fix ave/histo `, +:doc:`fix ave/chunk `, and :doc:`atom-style variable +` commands. -The list of possible attributes is the same as that used by the :doc:`dump custom ` command, which describes their meaning, with some -additional quantities that are only defined for certain :doc:`atom styles `. Basically, this augmented list gives an -input script access to any per-atom quantity stored by LAMMPS. +The list of possible attributes is the same as that used by the +:doc:`dump custom ` command, which describes their meaning, with +some additional quantities that are only defined for certain +:doc:`atom styles `. Basically, this augmented list gives +an input script access to any per-atom quantity stored by LAMMPS. The values are stored in a per-atom vector or array as discussed below. Zeroes are stored for atoms not in the specified group or for @@ -134,8 +138,9 @@ particles and body particles and store the 4-vector quaternion representing the orientation of each particle. See the :doc:`set ` command for an explanation of the quaternion vector. -*End1x*\ , *end1y*\ , *end1z*\ , *end2x*\ , *end2y*\ , *end2z*\ , are defined for -line segment particles and define the end points of each line segment. +*End1x*\ , *end1y*\ , *end1z*\ , *end2x*\ , *end2y*\ , *end2z*\ , are +defined for line segment particles and define the end points of each +line segment. *Corner1x*\ , *corner1y*\ , *corner1z*\ , *corner2x*\ , *corner2y*\ , *corner2z*\ , *corner3x*\ , *corner3y*\ , *corner3z*\ , are defined for @@ -146,14 +151,14 @@ number of explicit bonds assigned to an atom. Note that if the :doc:`newton bond ` command is set to *on*\ , which is the default, then every bond in the system is assigned to only one of the two atoms in the bond. Thus a bond between atoms I,J may be tallied -for either atom I or atom J. If :doc:`newton bond off ` is set, -it will be tallied with both atom I and atom J. +for either atom I or atom J. If :doc:`newton bond off ` is +set, it will be tallied with both atom I and atom J. The *i_name* and *d_name* attributes refer to custom integer and floating-point properties that have been added to each atom via the -:doc:`fix property/atom ` command. When that command -is used specific names are given to each attribute which are what is -specified as the "name" portion of *i_name* or *d_name*. +:doc:`fix property/atom ` command. When that +command is used specific names are given to each attribute which are +what is specified as the "name" portion of *i_name* or *d_name*. **Output info:** @@ -162,8 +167,8 @@ on the number of input values. If a single input is specified, a per-atom vector is produced. If two or more inputs are specified, a per-atom array is produced where the number of columns = the number of inputs. The vector or array can be accessed by any command that uses -per-atom values from a compute as input. See the :doc:`Howto output ` doc page for an overview of LAMMPS output -options. +per-atom values from a compute as input. See the :doc:`Howto output +` doc page for an overview of LAMMPS output options. The vector or array values will be in whatever :doc:`units ` the corresponding attribute is in, e.g. velocity units for vx, charge @@ -180,7 +185,8 @@ Restrictions Related commands """""""""""""""" -:doc:`dump custom `, :doc:`compute reduce `, :doc:`fix ave/atom `, :doc:`fix ave/chunk `, -:doc:`fix property/atom ` +:doc:`dump custom `, :doc:`compute reduce `, +:doc::doc:`fix ave/atom `, :doc:`fix ave/chunk +:doc:`, `fix property/atom ` **Default:** none diff --git a/doc/src/compute_meso_e_atom.rst b/doc/src/compute_sph_e_atom.rst similarity index 81% rename from doc/src/compute_meso_e_atom.rst rename to doc/src/compute_sph_e_atom.rst index ad8e46cafb..f35976ed10 100644 --- a/doc/src/compute_meso_e_atom.rst +++ b/doc/src/compute_sph_e_atom.rst @@ -1,6 +1,6 @@ -.. index:: compute meso/e/atom +.. index:: compute sph/e/atom -compute meso/e/atom command +compute sph/e/atom command =========================== Syntax @@ -8,17 +8,17 @@ Syntax .. parsed-literal:: - compute ID group-ID meso/e/atom + compute ID group-ID sph/e/atom * ID, group-ID are documented in :doc:`compute ` command -* meso/e/atom = style name of this compute command +* sph/e/atom = style name of this compute command Examples """""""" .. code-block:: LAMMPS - compute 1 all meso/e/atom + compute 1 all sph/e/atom Description """"""""""" @@ -27,8 +27,8 @@ Define a computation that calculates the per-atom internal energy for each atom in a group. The internal energy is the energy associated with the internal degrees -of freedom of a mesoscopic particles, e.g. a Smooth-Particle -Hydrodynamics particle. +of freedom of an SPH particle, i.e. a Smooth-Particle Hydrodynamics +particle. See `this PDF guide `_ to using SPH in LAMMPS. diff --git a/doc/src/compute_meso_rho_atom.rst b/doc/src/compute_sph_rho_atom.rst similarity index 64% rename from doc/src/compute_meso_rho_atom.rst rename to doc/src/compute_sph_rho_atom.rst index d4611c5cb7..4237bf0bf5 100644 --- a/doc/src/compute_meso_rho_atom.rst +++ b/doc/src/compute_sph_rho_atom.rst @@ -1,6 +1,6 @@ -.. index:: compute meso/rho/atom +.. index:: compute sph/rho/atom -compute meso/rho/atom command +compute sph/rho/atom command ============================= Syntax @@ -8,32 +8,31 @@ Syntax .. parsed-literal:: - compute ID group-ID meso/rho/atom + compute ID group-ID sph/rho/atom * ID, group-ID are documented in :doc:`compute ` command -* meso/rho/atom = style name of this compute command +* sph/rho/atom = style name of this compute command Examples """""""" .. code-block:: LAMMPS - compute 1 all meso/rho/atom + compute 1 all sph/rho/atom Description """"""""""" -Define a computation that calculates the per-atom mesoscopic density -for each atom in a group. +Define a computation that calculates the per-atom SPH density for each +atom in a group, i.e. a Smooth-Particle Hydrodynamics density. -The mesoscopic density is the mass density of a mesoscopic particle, -calculated by kernel function interpolation using "pair style -sph/rhosum". +The SPH density is the mass density of an SPH particle, calculated by +kernel function interpolation using "pair style sph/rhosum". See `this PDF guide `_ to using SPH in LAMMPS. -The value of the mesoscopic density will be 0.0 for atoms not in the +The value of the SPH density will be 0.0 for atoms not in the specified compute group. **Output info:** diff --git a/doc/src/compute_meso_t_atom.rst b/doc/src/compute_sph_t_atom.rst similarity index 79% rename from doc/src/compute_meso_t_atom.rst rename to doc/src/compute_sph_t_atom.rst index e44425a65c..1766e1d712 100644 --- a/doc/src/compute_meso_t_atom.rst +++ b/doc/src/compute_sph_t_atom.rst @@ -1,6 +1,6 @@ -.. index:: compute meso/t/atom +.. index:: compute sph/t/atom -compute meso/t/atom command +compute sph/t/atom command =========================== Syntax @@ -8,17 +8,17 @@ Syntax .. parsed-literal:: - compute ID group-ID meso/t/atom + compute ID group-ID sph/t/atom * ID, group-ID are documented in :doc:`compute ` command -* meso/t/atom = style name of this compute command +* sph/t/atom = style name of this compute command Examples """""""" .. code-block:: LAMMPS - compute 1 all meso/t/atom + compute 1 all sph/t/atom Description """"""""""" @@ -27,8 +27,8 @@ Define a computation that calculates the per-atom internal temperature for each atom in a group. The internal temperature is the ratio of internal energy over the heat -capacity associated with the internal degrees of freedom of a mesoscopic -particles, e.g. a Smooth-Particle Hydrodynamics particle. +capacity associated with the internal degrees of freedom of an SPH +particles, i.e. a Smooth-Particle Hydrodynamics particle. .. math:: diff --git a/doc/src/compute_stress_atom.rst b/doc/src/compute_stress_atom.rst index 48e8b9c69c..b8b3cb0ea1 100644 --- a/doc/src/compute_stress_atom.rst +++ b/doc/src/compute_stress_atom.rst @@ -2,6 +2,7 @@ compute stress/atom command =========================== + compute centroid/stress/atom command ==================================== @@ -223,15 +224,14 @@ The per-atom array values will be in pressure\*volume Restrictions """""""""""" -Currently, compute *centroid/stress/atom* does not support -pair styles with many-body interactions, -such as :doc:`Tersoff `, -and LAMMPS will generate an error in such cases. -In principal, equivalent formulation -to that of angle, dihedral and improper contributions -in the virial :math:`W_{ab}` formula -can also be applied to the many-body pair styles, -and is planned in the future. + +Currently (Spring 2020), compute *centroid/stress/atom* does not support +pair styles with many-body interactions, such as :doc:`Tersoff +`, or pair styles with long-range Coulomb interactions. +LAMMPS will generate an error in such cases. In principal, equivalent +formulation to that of angle, dihedral and improper contributions in the +virial :math:`W_{ab}` formula can also be applied to the many-body pair +styles, and is planned in the future. Related commands """""""""""""""" diff --git a/doc/src/compute_viscosity_cos.rst b/doc/src/compute_viscosity_cos.rst new file mode 100644 index 0000000000..6f783b481e --- /dev/null +++ b/doc/src/compute_viscosity_cos.rst @@ -0,0 +1,156 @@ +.. index:: compute viscosity/cos + +compute viscosity/cos command +============================= + +Syntax +"""""" + + +.. parsed-literal:: + + compute ID group-ID viscosity/cos + +* ID, group-ID are documented in :doc:`compute ` command +* viscosity/cos = style name of this compute command + + +Examples +"""""""" + + +.. code-block:: LAMMPS + + compute cos all viscosity/cos + variable V equal c_cos[7] + variable A equal 0.02E-5 + variable density equal density + variable lz equal lz + variable reciprocalViscosity equal v_V/${A}/v_density*39.4784/v_lz/v_lz*100 + +Description +""""""""""" + +Define a computation that calculates the velocity amplitude of a group of atoms +with an cosine-shaped velocity profile and the temperature of them +after subtracting out the velocity profile before computing the kinetic energy. +A compute of this style can be used by any command that computes a temperature, +e.g. :doc:`thermo_modify `, :doc:`fix npt `, etc. + +This command together with :doc:`fix_accelerate/cos` +enables viscosity calculation with periodic perturbation method, +as described by :ref:`Hess`. +An acceleration along the x-direction is applied to the simulation system +by using :doc:`fix_accelerate/cos` command. +The acceleration is a periodic function along the z-direction: + +.. math:: + + a_{x}(z) = A \cos \left(\frac{2 \pi z}{l_{z}}\right) + +where :math:`A` is the acceleration amplitude, :math:`l_z` is the z-length +of the simulation box. At steady state, the acceleration generates +a velocity profile: + +.. math:: + + v_{x}(z) = V \cos \left(\frac{2 \pi z}{l_{z}}\right) + +The generated velocity amplitude :math:`V` is related to the +shear viscosity :math:`\eta` by: + +.. math:: + + V = \frac{A \rho}{\eta}\left(\frac{l_{z}}{2 \pi}\right)^{2} + + +and it can be obtained from ensemble average of the velocity profile: + +.. math:: + + V = \frac{\sum_i 2 m_{i} v_{i, x} \cos \left(\frac{2 \pi z_i}{l_{z}}\right)}{\sum_i m_{i}} + + +where :math:`m_i`, :math:`v_{i,x}` and :math:`z_i` are the mass, +x-component velocity and z coordinate of a particle. + +After the cosine-shaped collective velocity in :math:`x` direction +has been subtracted for each atom, the temperature is calculated by the formula +KE = dim/2 N k T, where KE = total kinetic energy of the group of +atoms (sum of 1/2 m v\^2), dim = 2 or 3 = dimensionality of the +simulation, N = number of atoms in the group, k = Boltzmann constant, +and T = temperature. + +A kinetic energy tensor, stored as a 6-element vector, is also +calculated by this compute for use in the computation of a pressure +tensor. The formula for the components of the tensor is the same as +the above formula, except that v\^2 is replaced by vx\*vy for the xy +component, etc. The 6 components of the vector are ordered xx, yy, +zz, xy, xz, yz. + +The number of atoms contributing to the temperature is assumed to be +constant for the duration of the run; use the *dynamic* option of the +:doc:`compute_modify ` command if this is not the case. +However, in order to get meaningful result, the group ID of this compute should be all. + +The removal of the cosine-shaped velocity component by this command is +essentially computing the temperature after a "bias" has been removed +from the velocity of the atoms. If this compute is used with a fix +command that performs thermostatting then this bias will be subtracted +from each atom, thermostatting of the remaining thermal velocity will +be performed, and the bias will be added back in. Thermostatting +fixes that work in this way include :doc:`fix nvt `, :doc:`fix temp/rescale `, :doc:`fix temp/berendsen `, and :doc:`fix langevin `. + +This compute subtracts out degrees-of-freedom due to fixes that +constrain molecular motion, such as :doc:`fix shake ` and +:doc:`fix rigid `. This means the temperature of groups of +atoms that include these constraints will be computed correctly. If +needed, the subtracted degrees-of-freedom can be altered using the +*extra* option of the :doc:`compute_modify ` command. + +See the :doc:`Howto thermostat ` doc page for a +discussion of different ways to compute temperature and perform +thermostatting. + +---------- + +**Output info:** + +This compute calculates a global scalar (the temperature) and a global +vector of length 7, which can be accessed by indices 1-7. +The first 6 elements of the vector are the KE tensor, +and the 7-th is the cosine-shaped velocity amplitude :math:`V`, +which can be used to calculate the reciprocal viscosity, as shown in the example. +These values can be used by any command that uses global scalar or +vector values from a compute as input. +See the :doc:`Howto output ` doc page for an overview of LAMMPS output options. + +The scalar value calculated by this compute is "intensive". The +first 6 elements of vector values are "extensive", +and the 7-th element of vector values is "intensive". + +The scalar value will be in temperature :doc:`units `. The +first 6 elements of vector values will be in energy :doc:`units `. +The 7-th element of vector value will be in velocity :doc:`units `. + +Restrictions +"""""""""""" + +This command is only available when LAMMPS was built with the USER-MISC package. +Since this compute depends on :doc:`fix accelerate/cos ` which can +only work for 3d systems, it cannot be used for 2d systems. + +Related commands +"""""""""""""""" + +:doc:`fix accelerate/cos ` + +Default +""""""" + none + +---------- + +.. _Hess1: + +**(Hess)** Hess, B. The Journal of Chemical Physics 2002, 116 (1), 209-217. diff --git a/doc/src/fix.rst b/doc/src/fix.rst index 2b5ed48ac9..89d0f31a75 100644 --- a/doc/src/fix.rst +++ b/doc/src/fix.rst @@ -165,6 +165,7 @@ The individual style names on the :doc:`Commands fix ` doc page are followed by one or more of (g,i,k,o,t) to indicate which accelerated styles exist. +* :doc:`accelerate/cos ` - apply cosine-shaped acceleration to atoms * :doc:`adapt ` - change a simulation parameter over time * :doc:`adapt/fep ` - enhanced version of fix adapt * :doc:`addforce ` - add a force to each atom @@ -237,9 +238,7 @@ accelerated styles exist. * :doc:`lb/viscous ` - * :doc:`lineforce ` - constrain atoms to move in a line * :doc:`manifoldforce ` - restrain atoms to a manifold during minimization -* :doc:`meso ` - time integration for SPH/DPDE particles * :doc:`meso/move ` - move mesoscopic SPH/SDPD particles in a prescribed fashion -* :doc:`meso/stationary ` - * :doc:`momentum ` - zero the linear and/or angular momentum of a group of atoms * :doc:`move ` - move atoms in a prescribed fashion * :doc:`mscg ` - apply MSCG method for force-matching to generate coarse grain models @@ -344,6 +343,8 @@ accelerated styles exist. * :doc:`smd/move_tri_surf ` - * :doc:`smd/setvel ` - * :doc:`smd/wall_surface ` - +* :doc:`sph ` - time integration for SPH/DPDE particles +* :doc:`sph/stationary ` - * :doc:`spring ` - apply harmonic spring force to group of atoms * :doc:`spring/chunk ` - apply harmonic spring force to each chunk of atoms * :doc:`spring/rg ` - spring on radius of gyration of group of atoms diff --git a/doc/src/fix_accelerate_cos.rst b/doc/src/fix_accelerate_cos.rst new file mode 100644 index 0000000000..b8d1cca59c --- /dev/null +++ b/doc/src/fix_accelerate_cos.rst @@ -0,0 +1,104 @@ +.. index:: fix accelerate/cos + +fix accelerate/cos command +========================== + +Syntax +"""""" + + +.. parsed-literal:: + + fix ID group-ID accelerate value + +* ID, group-ID are documented in :doc:`fix ` command +* accelerate/cos = style name of this fix command +* value = amplitude of acceleration (in unit of force/mass) + + +Examples +"""""""" + + +.. code-block:: LAMMPS + + fix 1 all accelerate/cos 0.02e-5 + +Description +""""""""""" + +Give each atom a acceleration in x-direction based on its z coordinate. +The acceleration is a periodic function along the z-direction: + +.. math:: + + a_{x}(z) = A \cos \left(\frac{2 \pi z}{l_{z}}\right) + +where :math:`A` is the acceleration amplitude, :math:`l_z` is the z-length +of the simulation box. +At steady state, the acceleration generates a velocity profile: + +.. math:: + + v_{x}(z) = V \cos \left(\frac{2 \pi z}{l_{z}}\right) + +The generated velocity amplitude :math:`V` is related to the +shear viscosity :math:`\eta` by: + +.. math:: + + V = \frac{A \rho}{\eta}\left(\frac{l_{z}}{2 \pi}\right)^{2} + + +and it can be obtained from ensemble average of the velocity profile: + +.. math:: + + V = \frac{\sum_i 2 m_{i} v_{i, x} \cos \left(\frac{2 \pi z_i}{l_{z}}\right)}{\sum_i m_{i}} + +where :math:`m_i`, :math:`v_{i,x}` and :math:`z_i` are the mass, +x-component velocity and z coordinate of a particle. + +The velocity amplitude :math:`V` can be calculated with :doc:`compute viscosity/cos `, +which enables viscosity calculation with periodic perturbation method, +as described by :ref:`Hess`. +Because the applied acceleration drives the system away from equilibration, +the calculated shear viscosity is lower than the intrinsic viscosity +due to the shear-thinning effect. +Extrapolation to zero acceleration should generally be performed to +predict the zero-shear viscosity. +As the shear stress decreases, the signal-noise ratio decreases rapidly, +the simulation time must be extended accordingly to get converged result. + +In order to get meaningful result, the group ID of this fix should be all. + +---------- + +**Restart, fix_modify, output, run start/stop, minimize info:** + +No information about this fix is written to binary restart files. +None of the fix_modify options are relevant to this fix. +No global or per-atom quantities are stored by this fix for access by various output commands. +No parameter of this fix can be used with the start/stop keywords of the run command. +This fix is not invoked during energy minimization. + +Restrictions +"""""""""""" + +This command is only available when LAMMPS was built with the USER-MISC package. +Since this fix depends on the z-coordinate of atoms, it cannot be used in 2d simulations. + +Related commands +"""""""""""""""" + +:doc:`compute viscosity/cos ` + +Default +""""""" + none + +---------- + +.. _Hess2: + +**(Hess)** Hess, B. The Journal of Chemical Physics 2002, 116 (1), 209-217. diff --git a/doc/src/fix_adapt.rst b/doc/src/fix_adapt.rst index eab8f92639..fbc4aa4661 100644 --- a/doc/src/fix_adapt.rst +++ b/doc/src/fix_adapt.rst @@ -14,7 +14,7 @@ Syntax * adapt = style name of this fix command * N = adapt simulation settings every this many timesteps * one or more attribute/arg pairs may be appended -* attribute = *pair* or *kspace* or *atom* +* attribute = *pair* or *bond* or *kspace* or *atom* .. parsed-literal:: @@ -86,8 +86,8 @@ the end of a simulation. Even if *reset* is specified as *yes*\ , a restart file written during a simulation will contain the modified settings. -If the *scale* keyword is set to *no*\ , then the value the parameter is -set to will be whatever the variable generates. If the *scale* +If the *scale* keyword is set to *no*\ , then the value of the altered +parameter will be whatever the variable generates. If the *scale* keyword is set to *yes*\ , then the value of the altered parameter will be the initial value of that parameter multiplied by whatever the variable generates. I.e. the variable is now a "scale factor" applied @@ -288,11 +288,19 @@ Currently *bond* does not support bond_style hybrid nor bond_style hybrid/overlay as bond styles. The only bonds that currently are working with fix_adapt are -+---------------------------------+-------+------------+ -| :doc:`gromos ` | k, r0 | type bonds | -+---------------------------------+-------+------------+ -| :doc:`harmonic ` | k,r0 | type bonds | -+---------------------------------+-------+------------+ ++------------------------------------+-------+------------+ +| :doc:`class2 ` | r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`fene ` | k, r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`gromos ` | k, r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`harmonic ` | k,r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`morse ` | r0 | type bonds | ++------------------------------------+-------+------------+ +| :doc:`nonlinear ` | r0 | type bonds | ++------------------------------------+-------+------------+ ---------- @@ -311,26 +319,23 @@ The *atom* keyword enables various atom properties to be changed. The current list of atom parameters that can be varied by this fix: * charge = charge on particle -* diameter = diameter of particle +* diameter, or, diameter/disc = diameter of particle The *v_name* argument of the *atom* keyword is the name of an :doc:`equal-style variable ` which will be evaluated each time -this fix is invoked to set the parameter to a new value. It should be -specified as v_name, where name is the variable name. See the +this fix is invoked to set, or scale the parameter to a new value. +It should be specified as v_name, where name is the variable name. See the discussion above describing the formulas associated with equal-style variables. The new value is assigned to the corresponding attribute for all atoms in the fix group. -.. note:: - - The *atom* keyword works this way whether the *scale* keyword is - set to *no* or *yes*\ . I.e. the use of scale yes is not yet supported - by the *atom* keyword. - If the atom parameter is *diameter* and per-atom density and per-atom mass are defined for particles (e.g. :doc:`atom_style granular `), then the mass of each particle is also -changed when the diameter changes (density is assumed to stay -constant). +changed when the diameter changes. The mass is set from the particle volume +for 3d systems (density is assumed to stay constant). For 2d, the default is +for LAMMPS to model particles with a radius attribute as spheres. +However, if the atom parameter is *diameter/disc*, then the mass is +set from the particle area (the density is assumed to be in mass/distance^2 units). For example, these commands would shrink the diameter of all granular particles in the "center" group from 1.0 to 0.1 in a linear fashion diff --git a/doc/src/fix_ave_correlate_long.rst b/doc/src/fix_ave_correlate_long.rst index affa7df127..24ba504f1b 100644 --- a/doc/src/fix_ave_correlate_long.rst +++ b/doc/src/fix_ave_correlate_long.rst @@ -26,7 +26,7 @@ Syntax v_name = global value calculated by an equal-style variable with name * zero or more keyword/arg pairs may be appended -* keyword = *type* or *start* or *file* or *overwrite* or *title1* or *title2* or *ncorr* or *p* or *m* +* keyword = *type* or *start* or *file* or *overwrite* or *title1* or *title2* or *ncorr* or *nlen* or *ncount* .. parsed-literal:: diff --git a/doc/src/fix_balance.rst b/doc/src/fix_balance.rst index 13fdf6b1c7..b92644fcd9 100644 --- a/doc/src/fix_balance.rst +++ b/doc/src/fix_balance.rst @@ -154,28 +154,27 @@ of processors. It operates by changing the cutting planes (or lines) between processors in 3d (or 2d), to adjust the volume (area in 2d) assigned to each processor, as in the following 2d diagram where processor sub-domains are shown and atoms are colored by the processor -that owns them. The leftmost diagram is the default partitioning of -the simulation box across processors (one sub-box for each of 16 -processors); the middle diagram is after a "grid" method has been -applied. +that owns them. -.. |bal_uni| image:: JPG/balance_uniform_small.jpg - :target: JPG/balance_uniform.jpg - :width: 31% -.. |bal_non| image:: JPG/balance_nonuniform_small.jpg - :target: JPG/balance_nonuniform.jpg - :width: 31% -.. |bal_rcb| image:: JPG/balance_rcb_small.jpg - :target: JPG/balance_rcb.jpg - :width: 31% +.. list-table:: -|bal_uni| |bal_non| |bal_rcb| + * - .. figure:: JPG/balance_uniform_small.jpg + :target: JPG/balance_uniform.jpg -The *rcb* style is a "tiling" method which does not produce a logical -3d grid of processors. Rather it tiles the simulation domain with -rectangular sub-boxes of varying size and shape in an irregular -fashion so as to have equal numbers of particles (or weight) in each -sub-box, as in the rightmost diagram above. + - .. figure:: JPG/balance_nonuniform_small.jpg + :target: JPG/balance_nonuniform.jpg + + - .. figure:: JPG/balance_rcb_small.jpg + :target: JPG/balance_rcb.jpg + +The leftmost diagram is the default partitioning of the simulation box +across processors (one sub-box for each of 16 processors); the middle +diagram is after a "grid" method has been applied. The *rcb* style is a +"tiling" method which does not produce a logical 3d grid of processors. +Rather it tiles the simulation domain with rectangular sub-boxes of +varying size and shape in an irregular fashion so as to have equal +numbers of particles (or weight) in each sub-box, as in the rightmost +diagram above. The "grid" methods can be used with either of the :doc:`comm_style ` command options, *brick* or *tiled*\ . The diff --git a/doc/src/fix_bond_react.rst b/doc/src/fix_bond_react.rst old mode 100644 new mode 100755 index 8daa52a73d..fc260de324 --- a/doc/src/fix_bond_react.rst +++ b/doc/src/fix_bond_react.rst @@ -158,7 +158,9 @@ The following comments pertain to each *react* argument (in other words, can be customized for each reaction, or reaction step): A check for possible new reaction sites is performed every *Nevery* -timesteps. +timesteps. *Nevery* can be specified with an equal-style +:doc:`variable `, whose value is rounded up to the nearest +integer. Three physical conditions must be met for a reaction to occur. First, a bonding atom pair must be identified within the reaction distance @@ -171,19 +173,29 @@ modified to match the post-reaction template. A bonding atom pair will be identified if several conditions are met. First, a pair of atoms I,J within the specified react-group-ID of type itype and jtype must be separated by a distance between *Rmin* and -*Rmax*\ . It is possible that multiple bonding atom pairs are -identified: if the bonding atoms in the pre-reacted template are 1-2 -neighbors, i.e. directly bonded, the farthest bonding atom partner is -set as its bonding partner; otherwise, the closest potential partner -is chosen. Then, if both an atom I and atom J have each other as their -bonding partners, these two atoms are identified as the bonding atom -pair of the reaction site. Once this unique bonding atom pair is -identified for each reaction, there could two or more reactions that -involve a given atom on the same timestep. If this is the case, only -one such reaction is permitted to occur. This reaction is chosen -randomly from all potential reactions. This capability allows e.g. for -different reaction pathways to proceed from identical reaction sites -with user-specified probabilities. +*Rmax*\ . *Rmin* and *Rmax* can be specified with equal-style +:doc:`variables `. For example, these reaction cutoffs can +be a function of the reaction conversion using the following commands: + +.. code-block:: LAMMPS + + variable rmax equal 0 # initialize variable before bond/react + fix myrxn all bond/react react myrxn1 all 1 0 v_rmax mol1 mol2 map_file.txt + variable rmax equal 3+f_myrxn[1]/100 # arbitrary function of reaction count + +It is possible that multiple bonding atom pairs are identified: if the +bonding atoms in the pre-reacted template are 1-2 neighbors, i.e. +directly bonded, the farthest bonding atom partner is set as its +bonding partner; otherwise, the closest potential partner is chosen. +Then, if both an atom I and atom J have each other as their bonding +partners, these two atoms are identified as the bonding atom pair of +the reaction site. Once this unique bonding atom pair is identified +for each reaction, there could two or more reactions that involve a +given atom on the same timestep. If this is the case, only one such +reaction is permitted to occur. This reaction is chosen randomly from +all potential reactions. This capability allows e.g. for different +reaction pathways to proceed from identical reaction sites with +user-specified probabilities. The pre-reacted molecule template is specified by a molecule command. This molecule template file contains a sample reaction site and its @@ -419,7 +431,8 @@ it occurs: The *prob* keyword can affect whether or not an eligible reaction actually occurs. The fraction setting must be a value between 0.0 and -1.0. A uniform random number between 0.0 and 1.0 is generated and the +1.0, and can be specified with an equal-style :doc:`variable `. +A uniform random number between 0.0 and 1.0 is generated and the eligible reaction only occurs if the random number is less than the fraction. Up to N reactions are permitted to occur, as optionally specified by the *max_rxn* keyword. @@ -489,10 +502,11 @@ local command. **Restart, fix_modify, output, run start/stop, minimize info:** -Cumulative reaction counts for each reaction are written to :doc:`binary restart files `. These values are associated with the -reaction name (react-ID). Additionally, internally-created per-atom -properties are stored to allow for smooth restarts. None of the -:doc:`fix_modify ` options are relevant to this fix. +Cumulative reaction counts for each reaction are written to :doc:`binary restart files `. +These values are associated with the reaction name (react-ID). +Additionally, internally-created per-atom properties are stored to +allow for smooth restarts. None of the :doc:`fix_modify ` +options are relevant to this fix. This fix computes one statistic for each *react* argument that it stores in a global vector, of length 'number of react arguments', that diff --git a/doc/src/fix_halt.rst b/doc/src/fix_halt.rst index 9836679de3..0bc3154cdb 100644 --- a/doc/src/fix_halt.rst +++ b/doc/src/fix_halt.rst @@ -160,7 +160,7 @@ the :doc:`run ` command. Restrictions """""""""""" -The *diskfree* attribute is currently only supported on Linux and MacOS. +The *diskfree* attribute is currently only supported on Linux, MacOSX, and BSD. Related commands """""""""""""""" diff --git a/doc/src/fix_heat.rst b/doc/src/fix_heat.rst index 82a2b6e7f8..5bb6402d46 100644 --- a/doc/src/fix_heat.rst +++ b/doc/src/fix_heat.rst @@ -47,16 +47,22 @@ and the specified geometric :doc:`region ` in order to have energy added or subtracted to it. If not specified, then the atoms in the group are affected wherever they may move to. -Heat addition/subtraction is performed every N timesteps. The *eflux* -parameter can be specified as a numeric constant or as a variable (see -below). If it is a numeric constant or equal-style variable which -evaluates to a scalar value, then the *eflux* determines the change in -aggregate energy of the entire group of atoms per unit time, e.g. in -eV/psec for :doc:`metal units `. In this case it is an -"extensive" quantity, meaning its magnitude should be scaled with the -number of atoms in the group. Note that since *eflux* has per-time -units (i.e. it is a flux), this means that a larger value of N will -add/subtract a larger amount of energy each time the fix is invoked. +Heat addition/subtraction is performed every N timesteps. + +The *eflux* parameter can be specified as a numeric constant or as an +equal- or atom-style :doc:`variable `. If the value is a +variable, it should be specified as v_name, where *name* is the variable +name. In this case, the variable will be evaluated each timestep, and +its current value(s) used to determine the flux. + +If *eflux* is a numeric constant or equal-style variable which evaluates +to a scalar value, then *eflux* determines the change in aggregate energy +of the entire group of atoms per unit time, e.g. in eV/psec for +:doc:`metal units `. In this case it is an "extensive" quantity, +meaning its magnitude should be scaled with the number of atoms in the +group. Note that since *eflux* also has per-time units (i.e. it is a +flux), this means that a larger value of N will add/subtract a larger +amount of energy each time the fix is invoked. .. note:: @@ -71,12 +77,6 @@ the energy flux for a single atom, again in units of energy per unit time. In this case, each value is an "intensive" quantity, which need not be scaled with the number of atoms in the group. -As mentioned above, the *eflux* parameter can be specified as an -equal-style or atom_style :doc:`variable `. If the value is a -variable, it should be specified as v_name, where name is the variable -name. In this case, the variable will be evaluated each timestep, and -its value(s) used to determine the flux. - Equal-style variables can specify formulas with various mathematical functions, and include :doc:`thermo_style ` command keywords for the simulation box parameters and timestep and elapsed diff --git a/doc/src/fix_meso_move.rst b/doc/src/fix_meso_move.rst index a976c2de34..b97b029b9d 100644 --- a/doc/src/fix_meso_move.rst +++ b/doc/src/fix_meso_move.rst @@ -60,17 +60,19 @@ internal energy and extrapolated velocity are also updated. .. note:: - The particles affected by this fix should not be time integrated - by other fixes (e.g. :doc:`fix meso `, :doc:`fix meso/stationary `), since that will change their + The particles affected by this fix should not be time integrated by + other fixes (e.g. :doc:`fix sph `, :doc:`fix + sph/stationary `), since that will change their positions and velocities twice. .. note:: As particles move due to this fix, they will pass through periodic boundaries and be remapped to the other side of the simulation box, - just as they would during normal time integration (e.g. via the :doc:`fix meso ` command). It is up to you to decide whether periodic - boundaries are appropriate with the kind of particle motion you are - prescribing with this fix. + just as they would during normal time integration (e.g. via the + :doc:`fix sph ` command). It is up to you to decide + whether periodic boundaries are appropriate with the kind of + particle motion you are prescribing with this fix. .. note:: @@ -100,7 +102,7 @@ specified, *V* is the specified velocity vector with components specified. This style also sets the velocity of each particle to V = (Vx,Vy,Vz). If any of the velocity components is specified as NULL, then the position and velocity of that component is time integrated -the same as the :doc:`fix meso ` command would perform, using +the same as the :doc:`fix sph ` command would perform, using the corresponding force component on the particle. Note that the *linear* style is identical to using the *variable* @@ -128,7 +130,7 @@ elapsed since the fix was specified. This style also sets the velocity of each particle to the time derivative of this expression. If any of the amplitude components is specified as NULL, then the position and velocity of that component is time integrated the same as -the :doc:`fix meso ` command would perform, using the +the :doc:`fix sph ` command would perform, using the corresponding force component on the particle. Note that the *wiggle* style is identical to using the *variable* @@ -180,15 +182,15 @@ particle. Any of the 6 variables can be specified as NULL. If both the displacement and velocity variables for a particular x,y,z component are specified as NULL, then the position and velocity of that -component is time integrated the same as the :doc:`fix meso ` +component is time integrated the same as the :doc:`fix sph ` command would perform, using the corresponding force component on the -particle. If only the velocity variable for a component is specified as -NULL, then the displacement variable will be used to set the position -of the particle, and its velocity component will not be changed. If only -the displacement variable for a component is specified as NULL, then -the velocity variable will be used to set the velocity of the particle, -and the position of the particle will be time integrated using that -velocity. +particle. If only the velocity variable for a component is specified +as NULL, then the displacement variable will be used to set the +position of the particle, and its velocity component will not be +changed. If only the displacement variable for a component is +specified as NULL, then the velocity variable will be used to set the +velocity of the particle, and the position of the particle will be +time integrated using that velocity. The *units* keyword determines the meaning of the distance units used to define the *linear* velocity and *wiggle* amplitude and *rotate* @@ -236,17 +238,18 @@ Restrictions """""""""""" This fix is part of the USER-SDPD package. It is only enabled if -LAMMPS was built with that package. See the :doc:`Build package ` doc page for more info. +LAMMPS was built with that package. See the :doc:`Build package +` doc page for more info. This fix requires that atoms store density and internal energy as -defined by the :doc:`atom_style meso ` command. +defined by the :doc:`atom_style sph ` command. All particles in the group must be mesoscopic SPH/SDPD particles. Related commands """""""""""""""" -:doc:`fix move `, :doc:`fix meso `, +:doc:`fix move `, :doc:`fix sph `, :doc:`displace_atoms ` Default diff --git a/doc/src/fix_precession_spin.rst b/doc/src/fix_precession_spin.rst index 065f894926..783963af72 100644 --- a/doc/src/fix_precession_spin.rst +++ b/doc/src/fix_precession_spin.rst @@ -76,6 +76,7 @@ function for the same parameters. .. image:: JPG/zeeman_langevin.jpg :align: center + :width: 600 The temperature effects are accounted for by connecting the spin :math:`i` to a thermal bath using a Langevin thermostat (see @@ -154,6 +155,11 @@ The *precession/spin* style is part of the SPIN package. This style is only enabled if LAMMPS was built with this package, and if the atom_style "spin" was declared. See the :doc:`Build package ` doc page for more info. +The *precession/spin* style can only be declared once. If more +than one precession type (for example combining an anisotropy and a Zeeman interactions) +has to be declared, they have to be chained in the same command +line (as shown in the examples above). + Related commands """""""""""""""" diff --git a/doc/src/fix_restrain.rst b/doc/src/fix_restrain.rst index 3965cee04d..692336bec6 100644 --- a/doc/src/fix_restrain.rst +++ b/doc/src/fix_restrain.rst @@ -17,10 +17,18 @@ Syntax .. parsed-literal:: - *bond* args = atom1 atom2 Kstart Kstop r0 + *bond* args = atom1 atom2 Kstart Kstop r0start (r0stop) atom1,atom2 = IDs of 2 atoms in bond Kstart,Kstop = restraint coefficients at start/end of run (energy units) - r0 = equilibrium bond distance (distance units) + r0start = equilibrium bond distance at start of run (distance units) + r0stop = equilibrium bond distance at end of run (optional) (distance units). If not + specified it is assumed to be equal to r0start + *lbond* args = atom1 atom2 Kstart Kstop r0start (r0stop) + atom1,atom2 = IDs of 2 atoms in bond + Kstart,Kstop = restraint coefficients at start/end of run (energy units) + r0start = equilibrium bond distance at start of run (distance units) + r0stop = equilibrium bond distance at end of run (optional) (distance units). If not + specified it is assumed to be equal to r0start *angle* args = atom1 atom2 atom3 Kstart Kstop theta0 atom1,atom2,atom3 = IDs of 3 atoms in angle, atom2 = middle atom Kstart,Kstop = restraint coefficients at start/end of run (energy units) @@ -38,6 +46,7 @@ Examples .. code-block:: LAMMPS fix holdem all restrain bond 45 48 2000.0 2000.0 2.75 + fix holdem all restrain lbond 45 48 2000.0 2000.0 2.75 fix holdem all restrain dihedral 1 2 3 4 2000.0 2000.0 120.0 fix holdem all restrain bond 45 48 2000.0 2000.0 2.75 dihedral 1 2 3 4 2000.0 2000.0 120.0 fix texas_holdem all restrain dihedral 1 2 3 4 0.0 2000.0 120.0 dihedral 1 2 3 5 0.0 2000.0 -120.0 dihedral 1 2 3 6 0.0 2000.0 0.0 @@ -141,6 +150,29 @@ is included in :math:`K`. ---------- +The *lbond* keyword applies a lower bound bond restraint to the specified atoms +using the same functional form used by the :doc:`bond_style harmonic ` command if the distance between +the atoms is smaller than the equilibrium bond distance and 0 otherwise. The potential associated with +the restraint is + +.. math:: + + E = K (r - r_0)^2 ,if\;r < r_0 + +.. math:: + + E = 0 \qquad\quad\quad ,if\;r \ge r_0 + +with the following coefficients: + +* :math:`K` (energy/distance\^2) +* :math:`r_0` (distance) + +:math:`K` and :math:`r_0` are specified with the fix. Note that the usual 1/2 factor +is included in :math:`K`. + +---------- + The *angle* keyword applies an angle restraint to the specified atoms using the same functional form used by the :doc:`angle_style harmonic ` command. The potential associated with the restraint is diff --git a/doc/src/fix_rigid_meso.rst b/doc/src/fix_rigid_meso.rst index c9a709175f..9961a871dc 100644 --- a/doc/src/fix_rigid_meso.rst +++ b/doc/src/fix_rigid_meso.rst @@ -75,19 +75,21 @@ internal energy and extrapolated velocity are also updated. .. note:: You should not update the particles in rigid bodies via other - time-integration fixes (e.g. :doc:`fix meso `, - :doc:`fix meso/stationary `), or you will have conflicting - updates to positions and velocities resulting in unphysical behavior in most - cases. When performing a hybrid simulation with some atoms in rigid bodies, - and some not, a separate time integration fix like :doc:`fix meso ` - should be used for the non-rigid particles. + time-integration fixes (e.g. :doc:`fix sph `, :doc:`fix + sph/stationary `), or you will have conflicting + updates to positions and velocities resulting in unphysical + behavior in most cases. When performing a hybrid simulation with + some atoms in rigid bodies, and some not, a separate time + integration fix like :doc:`fix sph ` should be used for + the non-rigid particles. .. note:: - These fixes are overkill if you simply want to hold a collection - of particles stationary or have them move with a constant velocity. To - hold particles stationary use :doc:`fix meso/stationary ` instead. If you would like to - move particles with a constant velocity use :doc:`fix meso/move `. + These fixes are overkill if you simply want to hold a collection of + particles stationary or have them move with a constant velocity. To + hold particles stationary use :doc:`fix sph/stationary + ` instead. If you would like to move particles + with a constant velocity use :doc:`fix meso/move `. .. warning:: @@ -346,7 +348,7 @@ package. It is only enabled if LAMMPS was built with both packages. See the :doc:`Build package ` doc page for more info. This fix requires that atoms store density and internal energy as -defined by the :doc:`atom_style meso ` command. +defined by the :doc:`atom_style sph ` command. All particles in the group must be mesoscopic SPH/SDPD particles. diff --git a/doc/src/fix_meso.rst b/doc/src/fix_sph.rst similarity index 78% rename from doc/src/fix_meso.rst rename to doc/src/fix_sph.rst index dc9f1cb991..7e9159cac4 100644 --- a/doc/src/fix_meso.rst +++ b/doc/src/fix_sph.rst @@ -1,6 +1,6 @@ -.. index:: fix meso +.. index:: fix sph -fix meso command +fix sph command ================ Syntax @@ -8,25 +8,26 @@ Syntax .. parsed-literal:: - fix ID group-ID meso + fix ID group-ID sph * ID, group-ID are documented in :doc:`fix ` command -* meso = style name of this fix command +* sph = style name of this fix command Examples """""""" .. code-block:: LAMMPS - fix 1 all meso + fix 1 all sph Description """"""""""" Perform time integration to update position, velocity, internal energy and local density for atoms in the group each timestep. This fix is -needed to time-integrate mesoscopic systems where particles carry -internal variables such as SPH or DPDE. +needed to time-integrate SPH systems where particles carry internal +variables such as internal energy. SPH stands for Smoothed Particle +Hydrodynamics. See `this PDF guide `_ to using SPH in LAMMPS. @@ -48,6 +49,6 @@ LAMMPS was built with that package. See the :doc:`Build package Related commands """""""""""""""" -"fix meso/stationary" +:doc:`fix sph/stationary ` **Default:** none diff --git a/doc/src/fix_meso_stationary.rst b/doc/src/fix_sph_stationary.rst similarity index 84% rename from doc/src/fix_meso_stationary.rst rename to doc/src/fix_sph_stationary.rst index 89c30ece14..cd0d3e4d73 100644 --- a/doc/src/fix_meso_stationary.rst +++ b/doc/src/fix_sph_stationary.rst @@ -1,6 +1,6 @@ -.. index:: fix meso/stationary +.. index:: fix sph/stationary -fix meso/stationary command +fix sph/stationary command =========================== Syntax @@ -8,17 +8,17 @@ Syntax .. parsed-literal:: - fix ID group-ID meso/stationary + fix ID group-ID sph/stationary * ID, group-ID are documented in :doc:`fix ` command -* meso = style name of this fix command +* sph = style name of this fix command Examples """""""" .. code-block:: LAMMPS - fix 1 boundary meso/stationary + fix 1 boundary sph/stationary Description """"""""""" @@ -27,7 +27,7 @@ Perform time integration to update internal energy and local density, but not position or velocity for atoms in the group each timestep. This fix is needed for SPH simulations to correctly time-integrate fixed boundary particles which constrain a fluid to a given region in -space. +space. SPH stands for Smoothed Particle Hydrodynamics. See `this PDF guide `_ to using SPH in LAMMPS. @@ -49,6 +49,6 @@ LAMMPS was built with that package. See the :doc:`Build package Related commands """""""""""""""" -"fix meso" +:doc:`fix sph ` **Default:** none diff --git a/doc/src/fix_wall_body_polyhedron.rst b/doc/src/fix_wall_body_polyhedron.rst index 4b01234f0f..35be8e9259 100644 --- a/doc/src/fix_wall_body_polyhedron.rst +++ b/doc/src/fix_wall_body_polyhedron.rst @@ -15,15 +15,14 @@ Syntax * k_n = normal repulsion strength (force/distance units or pressure units - see discussion below) * c_n = normal damping coefficient (force/distance units or pressure units - see discussion below) * c_t = tangential damping coefficient (force/distance units or pressure units - see discussion below) -* wallstyle = *xplane* or *yplane* or *zplane* or *zcylinder* +* wallstyle = *xplane* or *yplane* or *zplane* * args = list of arguments for a particular style .. parsed-literal:: - *xplane* or *yplane* args = lo hi + *xplane* or *yplane* or *zplane* args = lo hi lo,hi = position of lower and upper plane (distance units), either can be NULL) - *zcylinder* args = radius - radius = cylinder radius (distance units) + * zero or more keyword/value pairs may be appended to args * keyword = *wiggle* @@ -60,8 +59,7 @@ those specified with the :doc:`pair_style body/rounded/polyhedron ` by -Art Voter and collaborators. Similar to parallel replica dynamics -(PRD), global and local HD are methods for performing accelerated -dynamics that are suitable for infrequent-event systems that obey -first-order kinetics. A good overview of accelerated dynamics methods -for such systems in given in :ref:`(Voter2002) ` from the same -group. To quote from the review paper: "The dynamical evolution is -characterized by vibrational excursions within a potential basin, -punctuated by occasional transitions between basins." The transition -probability is characterized by p(t) = k\*exp(-kt) where k is the rate -constant. Running multiple replicas gives an effective enhancement in -the timescale spanned by the multiple simulations, while waiting for -an event to occur. +Both global and local HD are described in :ref:`(Voter2013) +` by Art Voter and collaborators. Similar to parallel +replica dynamics (PRD), global and local HD are methods for performing +accelerated dynamics that are suitable for infrequent-event systems +that obey first-order kinetics. A good overview of accelerated +dynamics methods (AMD) for such systems in given in :ref:`(Voter2002) +` from the same group. To quote from the review paper: +"The dynamical evolution is characterized by vibrational excursions +within a potential basin, punctuated by occasional transitions between +basins. The transition probability is characterized by p(t) = +k\*exp(-kt) where k is the rate constant." Both HD and PRD produce a time-accurate trajectory that effectively extends the timescale over which a system can be simulated, but they diff --git a/doc/src/kim_commands.rst b/doc/src/kim_commands.rst index c710ddfe5d..99f7efffd5 100644 --- a/doc/src/kim_commands.rst +++ b/doc/src/kim_commands.rst @@ -1,16 +1,19 @@ -.. index:: kim_init, kim_interactions, kim_query, kim_param +.. index:: kim_init, kim_interactions, kim_query, kim_param, kim_property -kim_init command -================= +:ref:`kim_init` command +========================================= -kim_interactions command -========================= +:ref:`kim_interactions` command +========================================================= -kim_query command -================== +:ref:`kim_query` command +=========================================== -kim_param command -================== +:ref:`kim_param` command +=========================================== + +:ref:`kim_property` command +================================================= Syntax """""" @@ -22,6 +25,11 @@ Syntax kim_query variable formatarg query_function queryargs kim_param get param_name index_range variables formatarg kim_param set param_name index_range values + kim_property create instance_id property_id + kim_property modify instance_id key key_name key_name_key key_name_value + kim_property remove instance_id key key_name + kim_property destroy instance_id + kim_property dump file .. _formatarg_options: @@ -48,6 +56,12 @@ Syntax * param_name = name of a KIM portable model parameter * index_range = KIM portable model parameter index range (an integer for a single element, or pair of integers separated by a colon for a range of elements) * values = new value(s) to replace the current value(s) of a KIM portable model parameter +* instance_id = a positive integer identifying the KIM property instance +* property_id = identifier of a `KIM Property Definition `_, which can be (1) a property short name, (2) the full unique ID of the property (including the contributor and date), (3) a file name corresponding to a local property definition file +* key_name = one of the keys belonging to the specified KIM property definition +* key_name_key = a key belonging to a key-value pair (standardized in the `KIM Properties Framework `__) +* key_name_value = value to be associated with a key_name_key in a key-value pair +* file = name of a file to write the currently defined set of KIM property instances to Examples """""""" @@ -64,6 +78,15 @@ Examples kim_query a0 get_lattice_constant_cubic crystal=["fcc"] species=["Al"] units=["angstrom"] kim_param get gamma 1 varGamma kim_param set gamma 1 3.0 + kim_property create 1 atomic-mass + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key species source-value Al + kim_property remove 1 key species + kim_property destroy 1 + kim_property dump results.edn + + +.. _kim_description: Description """"""""""" @@ -157,11 +180,10 @@ See the `current list of KIM PMs and SMs archived in OpenKIM `_ to -learn how to install a pre-build binary of the OpenKIM Repository of Models. +See `Obtaining KIM Models `_ to +learn how to install a pre-built binary of the OpenKIM Repository of Models. .. note:: - It is also possible to locally install IMs not archived in OpenKIM, in which case their names do not have to conform to the KIM ID format. @@ -169,15 +191,17 @@ Using OpenKIM IMs with LAMMPS ----------------------------- Two commands are employed when using OpenKIM IMs, one to select the -IM and perform necessary initialization (*kim_init*), and the second +IM and perform necessary initialization (\ *kim_init*\ ), and the second to set up the IM for use by executing any necessary LAMMPS commands -(*kim_interactions*). Both are required. +(\ *kim_interactions*\ ). Both are required. See the *examples/kim* directory for example input scripts that use KIM PMs and KIM SMs. +.. _kim_init command: + OpenKIM IM Initialization (*kim_init*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The *kim_init* mode command must be issued **before** the simulation box is created (normally at the top of the file). @@ -219,7 +243,7 @@ either match the required units of the IM or the IM must be able to adjust its units to match. (The latter is only possible with some KIM PMs; SMs can never adjust their units.) If a match is possible, the LAMMPS :doc:`units ` command is called to set the units to -*user_units*. If the match fails, the simulation is terminated with +*user_units*\ . If the match fails, the simulation is terminated with an error. Here is an example of a LAMMPS script to compute the cohesive energy @@ -324,8 +348,10 @@ be done to convert the box and all atomic positions to the correct units: all appropriate places in the input script. It is up to the user to do this correctly. +.. _kim_interactions command: + OpenKIM IM Execution (*kim_interactions*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The second and final step in using an OpenKIM IM is to execute the *kim_interactions* command. This command must be preceded by a *kim_init* @@ -399,12 +425,17 @@ the *kim_interactions* command executes the following LAMMPS input commands: pair_coeff * * ffield.reax.rdx C H N O fix reaxqeq all qeq/reax 1 0.0 10.0 1.0e-6 param.qeq -Note that the files *lmp_control*, *ffield.reax.rdx* and *param.qeq* -are specific to the Strachan et al. (2003) ReaxFF parameterization -and are archived as part of the SM package in OpenKIM. -Note also that parameters like cutoff radii and charge tolerances, -which have an effect on IM predictions, are also included in the -SM definition ensuring reproducibility. +.. note:: + + The files *lmp_control*, *ffield.reax.rdx* and *param.qeq* + are specific to the Strachan et al. (2003) ReaxFF parameterization + and are archived as part of the SM package in OpenKIM. + +.. note:: + + Parameters like cutoff radii and charge tolerances, + which have an effect on IM predictions, are also included in the + SM definition ensuring reproducibility. .. note:: @@ -414,8 +445,10 @@ SM definition ensuring reproducibility. bond_coeff, fixes related to charge equilibration, etc.) should normally not appear in the input script. +.. _kim_query command: + Using OpenKIM Web Queries in LAMMPS (*kim_query*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The *kim_query* command performs a web query to retrieve the predictions of an IM set by *kim_init* for material properties archived in @@ -427,6 +460,7 @@ of an IM set by *kim_init* for material properties archived in The syntax for the *kim_query* command is as follows: + .. code-block:: LAMMPS kim_query variable formatarg query_function queryargs @@ -442,7 +476,7 @@ individual variables of the form *prefix_I*, where *prefix* is set to the *kim_query* *variable* argument and *I* ranges from 1 to the number of returned values. The number and order of the returned values is determined by the type of query performed. (Note that the "explicit" setting of -*formatarg* is not supported by *kim_query*.) +*formatarg* is not supported by *kim_query*\ .) .. note:: @@ -452,7 +486,7 @@ by the type of query performed. (Note that the "explicit" setting of cases will generate an error. The second required argument *query_function* is the name of the -query function to be called (e.g. *get_lattice_constant_cubic*). +query function to be called (e.g. *get_lattice_constant_cubic*\ ). All following :doc:`arguments ` are parameters handed over to the web query in the format *keyword=value*\ , where *value* is always an array of one or more comma-separated items in brackets. @@ -466,7 +500,7 @@ is available on the OpenKIM webpage at All query functions require the *model* keyword, which identifies the IM whose predictions are being queried. This keyword is automatically generated by *kim_query* based on the IM set in *kim_init* and must not - be specified as an argument to *kim_query*. + be specified as an argument to *kim_query*\ . .. note:: @@ -475,11 +509,11 @@ is available on the OpenKIM webpage at used to compute this property. In cases where there are multiple methods in OpenKIM for computing a property, a *method* keyword can be provided to select the method of choice. See the - `query documentation `_ - to see which methods are available for a given *query function*\ . + `query documentation `_ + to see which methods are available for a given *query_function*\ . *kim_query* Usage Examples and Further Clarifications -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The data obtained by *kim_query* commands can be used as part of the setup or analysis phases of LAMMPS simulations. Some examples are given below. @@ -502,10 +536,12 @@ crystal. By using *kim_query*, the user is saved the trouble and possible error of tracking this value down, or of having to perform an energy minimization to find the equilibrium lattice constant. -Note that in *unit_conversion_mode* the results obtained from a -*kim_query* would need to be converted to the appropriate units system. -For example, in the above script, the lattice command would need to be -changed to: "lattice fcc ${a0}\*${_u_distance}". +.. note:: + + In *unit_conversion_mode* the results obtained from a + *kim_query* would need to be converted to the appropriate units system. + For example, in the above script, the lattice command would need to be + changed to: "lattice fcc ${a0}*${_u_distance}". **Define an equilibrium hcp crystal** @@ -524,7 +560,7 @@ changed to: "lattice fcc ${a0}\*${_u_distance}". In this case the *kim_query* returns two arguments (since the hexagonal close packed (hcp) structure has two independent lattice constants). The *formatarg* keyword "split" places the two values into -the variables *latconst_1* and *latconst_2*. (These variables are +the variables *latconst_1* and *latconst_2*\ . (These variables are created if they do not already exist.) For convenience the variables *a0* and *c0* are created in order to make the remainder of the input script more readable. @@ -555,9 +591,9 @@ potential. If no tolerance is passed a default value is used. If multiple results are returned (indicating that the tolerance is too large), *kim_query* will return an error. See the - `query documentation `_ + `query documentation `_ to see which numerical arguments and tolerances are available for a - given *query function*\ . + given *query_function*\ . **Compute defect formation energy** @@ -586,8 +622,10 @@ ideal fcc cohesive energy of the atoms in the system obtained from from these programs are queried is tracked. No other information about the nature of the query or its source is recorded. +.. _kim_param command: + Accessing KIM Model Parameters from LAMMPS (*kim_param*) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ All IMs are functional forms containing a set of parameters. The values of these parameters are typically @@ -620,7 +658,7 @@ for details). .. note:: - The *kim_param get/set* commands must be preceded by *kim_init*. + The *kim_param get/set* commands must be preceded by *kim_init*\ . The *kim_param set* command must additionally be preceded by a *kim_interactions* command (or alternatively by a *pair_style kim* and *pair_coeff* commands). The *kim_param set* command may be used wherever a *pair_coeff* command may occur. @@ -674,13 +712,13 @@ Multiple parameters can be retrieved with a single call to *kim_param get* by repeating the argument list following *get*\ . For a *set* operation, the *values* argument contains the new value(s) -for the element(s) of the parameter specified by *index_range*. For the case +for the element(s) of the parameter specified by *index_range*\ . For the case where multiple values are being set, *values* contains a set of values separated by spaces. Multiple parameters can be set with a single call to *kim_param set* by repeating the argument list following *set*\ . *kim_param* Usage Examples and Further Clarifications -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of getting and setting KIM PM parameters with further clarifications are provided below. @@ -722,7 +760,7 @@ determined by the *formatarg* argument. In this case, *formatarg* is not specified and therefore the default "explicit" mode is used. (The behavior would be the same if the word -*explicit* were added after *LAM_TeSe*.) Elements 7, 8 and 9 of parameter +*explicit* were added after *LAM_TeSe*\ .) Elements 7, 8 and 9 of parameter lambda retrieved by the *get* operation are placed in the LAMMPS variables *LAM_TeTe*, *LAM_TeZn* and *LAM_TeSe*, respectively. @@ -765,7 +803,7 @@ contains the current value of lambda. In this case, the "split" mode of *formatarg* is used. The three values retrieved by the *get* operation are stored in -the three LAMMPS variables *LAM_15*, *LAM_16* and *LAM_17*. +the three LAMMPS variables *LAM_15*, *LAM_16* and *LAM_17*\ . The provided name "LAM" is used as prefix and the location in the lambda array is appended to create the variable names. @@ -797,7 +835,7 @@ potential, while *NEW_GAMMA* will contain the value 2.6. **Setting multiple scalar parameters with a single call** -.. parsed-literal:: +.. code-block:: LAMMPS kim_init SW_ZhouWardMartin_2013_CdTeZnSeHgS__MO_503261197030_002 metal ... @@ -824,6 +862,421 @@ In this case, elements 2 through 6 of the parameter *sigma* are set to the values 2.35214, 2.23869, 2.04516, 2.43269 and 1.80415 in order. +.. _kim_property command: + +Writing material properties computed in LAMMPS to standard KIM property instance format (*kim_property*) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As explained :ref:`above`, +The OpenKIM system includes a collection of Tests (material property calculation codes), +Models (interatomic potentials), Predictions, and Reference Data (DFT or experiments). +Specifically, a KIM Test is a computation that when coupled with a KIM Model generates +the prediction of that model for a specific material property rigorously defined +by a KIM Property Definition (see the +`KIM Properties Framework `__ +for further details). A prediction of a material property for a given model is a specific +numerical realization of a property definition, referred to as a "Property +Instance." The objective of the *kim_property* command is to make it easy to +output material properties in a standardized, machine readable, format that can be easily +ingested by other programs. +Additionally, it aims to make it as easy as possible to convert a LAMMPS script that computes a +material property into a KIM Test that can then be uploaded to `openkim.org `_ + +A developer interested in creating a KIM Test using a LAMMPS script should +first determine whether a property definition that applies to their calculation +already exists in OpenKIM by searching the `properties page +`_. If none exists, it is possible to use a +locally defined property definition contained in a file until it can be +uploaded to the official repository (see below). Once one or more applicable +property definitions have been identified, the *kim_property create*, +*kim_property modify*, *kim_property remove*, and *kim_property destroy*, +commands provide an interface to create, set, modify, remove, and destroy +instances of them within a LAMMPS script. Their general syntax is as follows: + +.. code-block:: LAMMPS + + kim_property create instance_id property_id + kim_property modify instance_id key key_name key_name_key key_name_value + kim_property remove instance_id key key_name + kim_property destroy instance_id + kim_property dump file + +Here, *instance_id* is a positive integer used to uniquely identify each +property instance; (note that the results file can contain multiple property +instances). A property_id is an identifier of a +`KIM Property Definition `_, +which can be (1) a property short name, (2) the full unique ID of the property +(including the contributor and date), (3) a file name corresponding to a local +property definition file. Examples of each of these cases are shown below: + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property create 2 cohesive-energy-relation-cubic-crystal + +.. code-block:: LAMMPS + + kim_property create 1 tag:brunnels@noreply.openkim.org,2016-05-11:property/atomic-mass + kim_property create 2 tag:staff@noreply.openkim.org,2014-04-15:property/cohesive-energy-relation-cubic-crystal + +.. code-block:: LAMMPS + + kim_property create 1 new-property.edn + kim_property create 2 /home/mary/marys-kim-properties/dissociation-energy.edn + +In the last example, "new-property.edn" and "/home/mary/marys-kim-properties/dissociation-energy.edn" are the +names of files that contain user-defined (local) property definitions. + +A KIM property instance takes the form of a "map," i.e. a set of key-value +pairs akin to Perl's hash, Python's dictionary, or Java's Hashtable. It +consists of a set of property key names, each of which is referred to here by +the *key_name* argument, that are defined as part of the relevant KIM Property +Definition and include only lowercase alphanumeric characters and dashes. The +value paired with each property key is itself a map whose possible keys are +defined as part of the `KIM Properties Framework +`__; these keys are +referred to by the *key_name_key* argument and their associated values by the +*key_name_value* argument. These values may either be scalars or arrays, +as stipulated in the property definition. + +.. note:: + + Each map assigned to a *key_name* must contain the *key_name_key* + "source-value" and an associated *key_name_value* of the appropriate + type (as defined in the relevant KIM Property Definition). For keys that are + defined as having physical units, the + "source-unit" *key_name_key* must also be given a string value recognized + by `GNU units `_. + +Once a *kim_property create* command has been given to instantiate a property +instance, maps associated with the property's keys can be edited using the +*kim_property modify* command. In using this command, the special keyword +"key" should be given, followed by the property key name and the key-value pair +in the map associated with the key that is to be set. For example, the +`atomic-mass `_ +property definition consists of two property keys named "mass" and "species." +An instance of this property could be created like so: + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property modify 1 key species source-value Al + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key mass source-unit amu + +or, equivalently, + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property modify 1 key species source-value Al & + key mass source-value 26.98154 & + source-unit amu + +*kim_property* Usage Examples and Further Clarifications +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Create** + +.. code-block:: LAMMPS + + kim_property create instance_id property_id + +The *kim_property create* command takes as input a property instance ID and the +property definition name, and creates an initial empty property instance data +structure. For example, + +.. code-block:: LAMMPS + + kim_property create 1 atomic-mass + kim_property create 2 cohesive-energy-relation-cubic-crystal + +creates an empty property instance of the "atomic-mass" property definition +with instance ID 1 and an empty instance of the +"cohesive-energy-relation-cubic-crystal" property with ID 2. A list of +published property definitions in OpenKIM can be found on the `properties page +`_. + +One can also provide the name of a file in the current working directory or the +path of a file containing a valid property definition. For example, + +.. code-block:: LAMMPS + + kim_property create 1 new-property.edn + +where "new-property.edn" refers to a file name containing a new property +definition that does not exist in OpenKIM. + +If the *property_id* given cannot be found in OpenKIM and no file of this name +containing a valid property definition can be found, this command will produce +an error with an appropriate message. Calling *kim_property create* with the +same instance ID multiple times will also produce an error. + +**Modify** + +.. code-block:: LAMMPS + + kim_property modify instance_id key key_name key_name_key key_name_value + +The *kim_property modify* command incrementally builds the property instance +by receiving property definition keys along with associated arguments. Each +*key_name* is associated with a map containing one or more key-value pairs (in +the form of *key_name_key*-*key_name_value* pairs). For example, + +.. code-block:: LAMMPS + + kim_property modify 1 key species source-value Al + kim_property modify 1 key mass source-value 26.98154 + kim_property modify 1 key mass source-unit amu + +where the special keyword "key" is followed by a *key_name* ("species" or +"mass" in the above) and one or more key-value pairs. These key-value pairs +may continue until either another "key" keyword is given or the end of the +command line is reached. Thus, the above could equivalently be written as + +.. code-block:: LAMMPS + + kim_property modify 1 key species source-value Al & + key mass source-value 26.98154 & + key mass source-unit amu + +As an example of modifying multiple key-value pairs belonging to the map of a +single property key, the following command modifies the map of the +"cohesive-potential-energy" property key to contain the key "source-unit" which +is assigned a value of "eV" and the key "digits" which is assigned a value of +5: + +.. code-block:: LAMMPS + + kim_property modify 2 key cohesive-potential-energy source-unit eV digits 5 + +.. note:: + + The relevant data types of the values in the map are handled + automatically based on the specification of the key in the + KIM Property Definition. In the example above, + this means that the value "eV" will automatically be interpreted as a string + while the value 5 will be interpreted as an integer. + +The values contained in maps can either be scalars, as in all of the examples +above, or arrays depending on which is stipulated in the corresponding Property +Definition. For one-dimensional arrays, a single one-based index must be +supplied that indicates which element of the array is to be modified. For +multidimensional arrays, multiple indices must be given depending on the +dimensionality of the array. + +.. note:: + + All array indexing used by *kim_property modify* is one-based, i.e. the + indices are enumerated 1, 2, 3, ... + +.. note:: + + The dimensionality of arrays are defined in the the corresponding Property + Definition. The extent of each dimension of an array can either be a + specific finite number or indefinite and determined at run time. If + an array has a fixed extent, attempting to modify an out-of-range index will + fail with an error message. + +For example, the "species" property key of the +`cohesive-energy-relation-cubic-crystal +`_ +property is a one-dimensional array that can contain any number of entries +based on the number of atoms in the unit cell of a given cubic crystal. To +assign an array containing the string "Al" four times to the "source-value" key +of the "species" property key, we can do so by issuing: + +.. code-block:: LAMMPS + + kim_property modify 2 key species source-value 1 Al + kim_property modify 2 key species source-value 2 Al + kim_property modify 2 key species source-value 3 Al + kim_property modify 2 key species source-value 4 Al + +.. note:: + + No declaration of the number of elements in this array was given; + *kim_property modify* will automatically handle memory management to allow + an arbitrary number of elements to be added to the array. + +.. note:: + + In the event that *kim_property modify* is used to set the value of an + array index without having set the values of all lesser indices, they will + be assigned default values based on the data type associated with the key in + the map: + + .. table_from_list:: + :columns: 2 + + * Data type + * Default value + * int + * 0 + * float + * 0.0 + * string + * \"\" + * file + * \"\" + + For example, doing the following: + + .. code-block:: LAMMPS + + kim_property create 2 cohesive-energy-relation-cubic-crystal + kim_property modify 2 key species source-value 4 Al + + will result in the "source-value" key in the map for the property key + "species" being assigned the array ["", "", "", "Al"]. + +For convenience, the index argument provided may refer to an inclusive range of +indices by specifying two integers separated by a colon (the first integer must +be less than or equal to the second integer, and no whitespace should be +included). Thus, the snippet above could equivalently be written: + +.. code-block:: LAMMPS + + kim_property modify 2 key species source-value 1:4 Al Al Al Al + +Calling this command with a non-positive index, e.g. +``kim_property modify 2 key species source-value 0 Al``, or an incorrect +number of input arguments, e.g. +``kim_property modify 2 key species source-value 1:4 Al Al``, will result in an +error. + +As an example of modifying multidimensional arrays, consider the "basis-atoms" +key in the `cohesive-energy-relation-cubic-crystal +`_ +property definition. This is a two-dimensional array containing the fractional +coordinates of atoms in the unit cell of the cubic crystal. In the case of, +e.g. a conventional fcc unit cell, the "source-value" key in the map associated +with this key should be assigned the following value: + +.. code-block:: LAMMPS + + [[0.0, 0.0, 0.0], + [0.5, 0.5, 0.0], + [0.5, 0.0, 0.5], + [0.0, 0.5, 0.5]] + +While each of the twelve components could be set individually, we can instead set +each row at a time using colon notation: + +.. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 + kim_property modify 2 key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 + kim_property modify 2 key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 + kim_property modify 2 key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + +Where the first index given refers to a row and the second index refers to a +column. We could, instead, choose to set each column at a time like so: + +.. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1:4 1 0.0 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 1:4 2 0.0 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 1:4 3 0.0 0.0 0.5 0.5 + +.. note:: + + Multiple calls of *kim_property modify* made for the same instance ID + can be combined into a single invocation, meaning the following are + both valid: + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + + .. code-block:: LAMMPS + + kim_property modify 2 key short-name source-value 1 fcc & + key species source-value 1:4 Al Al Al Al & + key a source-value 1:5 3.9149 4.0000 4.032 4.0817 4.1602 & + source-unit angstrom & + digits 5 & + key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + key basis-atom-coordinates source-value 2 1:3 0.5 0.5 0.0 & + key basis-atom-coordinates source-value 3 1:3 0.5 0.0 0.5 & + key basis-atom-coordinates source-value 4 1:3 0.0 0.5 0.5 + +.. note:: + + For multidimensional arrays, only one colon-separated range is allowed + in the index listing. Therefore, + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates 1 1:3 0.0 0.0 0.0 + + is valid but + + .. code-block:: LAMMPS + + kim_property modify 2 key basis-atom-coordinates 1:2 1:3 0.0 0.0 0.0 0.0 0.0 0.0 + + is not. + +.. note:: + + After one sets a value in a map with the *kim_property modify* command, + additional calls will overwrite the previous value. + +**Remove** + +.. code-block:: LAMMPS + + kim_property remove instance_id key key_name + +The *kim_property remove* command can be used to remove a property key from a +property instance. For example, + +.. code-block:: LAMMPS + + kim_property remove 2 key basis-atom-coordinates + +**Destroy** + +.. code-block:: LAMMPS + + kim_property destroy instance_id + +The *kim_property destroy* command deletes a previously created property +instance ID. For example, + +.. code-block:: LAMMPS + + kim_property destroy 2 + +.. note:: + + If this command is called with an instance ID that does not exist, no + error is raised. + +**Dump** + +The *kim_property dump* command can be used to write the content of all +currently defined property instances to a file: + +.. code-block:: LAMMPS + + kim_property dump file + +For example, + +.. code-block:: LAMMPS + + kim_property dump results.edn + +.. note:: + + Issuing the *kim_property dump* command clears all existing property + instances from memory. + Citation of OpenKIM IMs ----------------------- @@ -847,8 +1300,11 @@ LAMMPS is built with that package. A requirement for the KIM package, is the KIM API library that must be downloaded from the `OpenKIM website `_ and installed before LAMMPS is compiled. When installing LAMMPS from binary, the kim-api package -is a dependency that is automatically downloaded and installed. See the KIM -section of the :doc:`Packages details ` for details. +is a dependency that is automatically downloaded and installed. The *kim_query* +command requires the *libcurl* library to be installed. The *kim_property* +command requires *Python* 3.6 or later and the *kim-property* python package to +be installed. See the KIM section of the :doc:`Packages details ` +for details. Furthermore, when using *kim_commands* to run KIM SMs, any packages required by the native potential being used or other commands or fixes that it invokes diff --git a/doc/src/min_modify.rst b/doc/src/min_modify.rst index a01aab173d..e541571b7b 100644 --- a/doc/src/min_modify.rst +++ b/doc/src/min_modify.rst @@ -87,8 +87,7 @@ The choice of a norm can be modified for the min styles *cg*\ , *sd*\ the 2-norm (Euclidean length) of the global force vector: .. math:: - - || \vec{F} ||_{2} = \sqrt{\vec{F}_1+ \cdots + \vec{F}_N} + || \vec{F} ||_{2} = \sqrt{\vec{F}_1^2+ \cdots + \vec{F}_N^2} The *max* norm computes the length of the 3-vector force for each atom (2-norm), and takes the maximum value of those across diff --git a/doc/src/minimize.rst b/doc/src/minimize.rst index dde5284b7a..81234ae80e 100644 --- a/doc/src/minimize.rst +++ b/doc/src/minimize.rst @@ -64,7 +64,7 @@ performed using a line search algorithm. The line search typically evaluates forces and energies several times to set new coordinates. Currently, a backtracking algorithm is used which may not be optimal in terms of the number of force evaluations performed, but appears to -be more robust than previous line searches we've tried. The +be more robust than previous line searches we have tried. The backtracking method is described in Nocedal and Wright's Numerical Optimization (Procedure 3.1 on p 41). diff --git a/doc/src/pair_adp.rst b/doc/src/pair_adp.rst index 6c4796fb85..bbfd98f677 100644 --- a/doc/src/pair_adp.rst +++ b/doc/src/pair_adp.rst @@ -56,8 +56,10 @@ command to specify them. * The NIST WWW site at http://www.ctcms.nist.gov/potentials. Note that ADP potentials obtained from NIST must be converted into the extended DYNAMO *setfl* format discussed below. -* The OpenKIM Project at https://openkim.org/browse/models/by-type provides - ADP potentials that can be used directly in LAMMPS with the :doc:`kim_commands interface `. +* The OpenKIM Project at + `https://openkim.org/browse/models/by-type `_ + provides ADP potentials that can be used directly in LAMMPS with the + :doc:`kim_commands ` interface. ---------- diff --git a/doc/src/pair_cosine_squared.rst b/doc/src/pair_cosine_squared.rst index b7fa29bbd5..a8cf206135 100644 --- a/doc/src/pair_cosine_squared.rst +++ b/doc/src/pair_cosine_squared.rst @@ -46,7 +46,7 @@ Style *cosine/squared* computes a potential of the form E = \begin{cases} -\epsilon& \quad r < \sigma \\ - -\epsilon\cos\left(\frac{\pi\left(r - \sigma\right)}{2\left(r_c - \sigma\right)}\right)&\quad \sigma \leq r < r_c \\ + -\epsilon\cos\left(\frac{\pi\left(r - \sigma\right)}{2\left(r_c - \sigma\right)}\right)^2&\quad \sigma \leq r < r_c \\ 0& \quad r \geq r_c \end{cases} diff --git a/doc/src/pair_cs.rst b/doc/src/pair_cs.rst index 72332a87b2..9790c2defd 100644 --- a/doc/src/pair_cs.rst +++ b/doc/src/pair_cs.rst @@ -30,6 +30,9 @@ pair_style coul/wolf/cs command pair_style lj/cut/coul/long/cs command ======================================= +pair_style lj/class2/coul/long/cs command +========================================== + Syntax """""" @@ -37,7 +40,7 @@ Syntax pair_style style args -* style = *born/coul/dsf/cs* or *born/coul/long/cs* or *born/coul/wolf/cs* or *buck/coul/long/cs* or *coul/long/cs* or *coul/wolf/cs* or *lj/cut/coul/long/cs* +* style = *born/coul/dsf/cs* or *born/coul/long/cs* or *born/coul/wolf/cs* or *buck/coul/long/cs* or *coul/long/cs* or *coul/wolf/cs* or *lj/cut/coul/long/cs* or *lj/class2/coul/long/cs* * args = list of arguments for a particular style .. parsed-literal:: @@ -64,6 +67,9 @@ Syntax *lj/cut/coul/long/cs* args = cutoff (cutoff2) cutoff = global cutoff for LJ (and Coulombic if only 1 arg) (distance units) cutoff2 = global cutoff for Coulombic (optional) (distance units) + *lj/class2/coul/long/cs* args = cutoff (cutoff2) + cutoff = global cutoff for LJ (and Coulombic if only 1 arg) (distance units) + cutoff2 = global cutoff for Coulombic (optional) (distance units) Examples """""""" @@ -115,6 +121,7 @@ the "/cs" in the name: * :doc:`pair_style coul/long ` * :doc:`pair_style coul/wolf ` * :doc:`pair_style lj/cut/coul/long ` +* :doc:`pair_style lj/class2/coul/long ` except that they correctly treat the special case where the distance between two charged core and shell atoms in the same core/shell pair diff --git a/doc/src/pair_eam.rst b/doc/src/pair_eam.rst index 4710f8a9a1..7384a4d54c 100644 --- a/doc/src/pair_eam.rst +++ b/doc/src/pair_eam.rst @@ -149,6 +149,7 @@ potentials stored in DYNAMO or other formats: http://www.ctcms.nist.gov/potentials http://cst-www.nrl.navy.mil/ccm6/ap http://enpub.fulton.asu.edu/cms/potentials/main/main.htm + https://openkim.org These potentials should be usable with LAMMPS, though the alternate formats would need to be converted to the DYNAMO format used by LAMMPS @@ -156,6 +157,11 @@ and described on this page. The NIST site is maintained by Chandler Becker (cbecker at nist.gov) who is good resource for info on interatomic potentials and file formats. +The OpenKIM Project at +`https://openkim.org/browse/models/by-type `_ +provides EAM potentials that can be used directly in LAMMPS with the +:doc:`kim_commands ` interface. + ---------- For style *eam*\ , potential values are read from a file that is in the diff --git a/doc/src/pair_meamc.rst b/doc/src/pair_meamc.rst index 850be5b583..f227af8b45 100644 --- a/doc/src/pair_meamc.rst +++ b/doc/src/pair_meamc.rst @@ -93,12 +93,13 @@ and the 4th to be C, you would use the following pair_coeff command: pair_coeff * * library.meam Si C sic.meam Si Si Si C The 1st 2 arguments must be \* \* so as to span all LAMMPS atom types. -The two filenames are for the library and parameter file respectively. -The Si and C arguments (between the file names) are the two elements -for which info will be extracted from the library file. The first -three trailing Si arguments map LAMMPS atom types 1,2,3 to the MEAM Si -element. The final C argument maps LAMMPS atom type 4 to the MEAM C -element. +The first filename is the element library file. The list of elements following +it extracts lines from the library file and assigns numeric indices to these +elements. The second filename is the alloy parameter file, which refers to +elements using the numeric indices assigned before. +The arguments after the parameter file map LAMMPS atom types to elements, i.e. +LAMMPS atom types 1,2,3 to the MEAM Si element. The final C argument maps +LAMMPS atom type 4 to the MEAM C element. If the 2nd filename is specified as NULL, no parameter file is read, which simply means the generic parameters in the library file are @@ -140,7 +141,7 @@ not required. The other numeric parameters match values in the formulas above. The value of the "elt" string is what is used in the pair_coeff command to identify which settings from the library file you wish to read in. There can be multiple entries in the library -file with the same "elt" value; LAMMPS reads the 1st matching entry it +file with the same "elt" value; LAMMPS reads the first matching entry it finds and ignores the rest. Other parameters in the MEAM library file correspond to single-element @@ -192,7 +193,7 @@ trailing comment (starting with #) which is ignored. The indices I, J, K correspond to the elements selected from the MEAM library file numbered in the order of how those elements were -selected starting from 1. Thus for the example given below +selected starting from 1. Thus for the example given before .. code-block:: LAMMPS @@ -202,11 +203,6 @@ an index of 1 would refer to Si and an index of 2 to C. The recognized keywords for the parameter file are as follows: -Ec, alpha, rho0, delta, lattce, attrac, repuls, nn2, Cmin, Cmax, rc, delr, -augt1, gsmooth_factor, re - -where - .. parsed-literal:: rc = cutoff radius for cutoff function; default = 4.0 diff --git a/doc/src/pair_momb.rst b/doc/src/pair_momb.rst index ed9a0b98dd..99dce23bf8 100644 --- a/doc/src/pair_momb.rst +++ b/doc/src/pair_momb.rst @@ -31,7 +31,7 @@ Style *momb* computes pairwise van der Waals (vdW) and short-range interactions using the Morse potential and :ref:`(Grimme) ` method implemented in the Many-Body Metal-Organic (MOMB) force field described comprehensively in :ref:`(Fichthorn) ` and -:ref:`(Zhou) `. Grimme's method is widely used to correct for +:ref:`(Zhou) `. Grimme's method is widely used to correct for dispersion in density functional theory calculations. .. math:: @@ -76,6 +76,6 @@ Related commands **(Fichthorn)** Fichthorn, Balankura, Qi, CrystEngComm, 18(29), 5410-5417 (2016). -.. _Zhou4: +.. _Zhou5: **(Zhou)** Zhou, Saidi, Fichthorn, J Phys Chem C, 118(6), 3366-3374 (2014). diff --git a/doc/src/pair_polymorphic.rst b/doc/src/pair_polymorphic.rst index c0db3f10a4..a23a9f12f9 100644 --- a/doc/src/pair_polymorphic.rst +++ b/doc/src/pair_polymorphic.rst @@ -18,21 +18,22 @@ Examples .. code-block:: LAMMPS pair_style polymorphic - pair_coeff * * TlBr_msw.polymorphic Tl Br - pair_coeff * * AlCu_eam.polymorphic Al Cu - pair_coeff * * GaN_tersoff.polymorphic Ga N - pair_coeff * * GaN_sw.polymorphic GaN + pair_coeff * * FeCH_BOP_I.poly Fe C H + pair_coeff * * TlBr_msw.poly Tl Br + pair_coeff * * CuTa_eam.poly Cu Ta + pair_coeff * * GaN_tersoff.poly Ga N + pair_coeff * * GaN_sw.poly Ga N Description """"""""""" The *polymorphic* pair style computes a 3-body free-form potential -(:ref:`Zhou `) for the energy E of a system of atoms as +(:ref:`Zhou3 `) for the energy E of a system of atoms as .. math:: - E & = \frac{1}{2}\sum_{i=1}^{i=N}\sum_{j=1}^{j=N}\left[\left(1-\delta_{ij}\right)\cdot U_{IJ}\left(r_{ij}\right)-\left(1-\eta_{ij}\right)\cdot F_{IJ}\left(r_{ij}\right)\cdot V_{IJ}\left(r_{ij}\right)\right] \\ - X_{ij} & = \sum_{k=i_1,k\neq i,j}^{i_N}W_{IK}\left(r_{ik}\right)\cdot G_{JIK}\left(\theta_{jik}\right)\cdot P_{IK}\left(\Delta r_{jik}\right) \\ + E & = \frac{1}{2}\sum_{i=1}^{i=N}\sum_{j=1}^{j=N}\left[\left(1-\delta_{ij}\right)\cdot U_{IJ}\left(r_{ij}\right)-\left(1-\eta_{ij}\right)\cdot F_{IJ}\left(X_{ij}\right)\cdot V_{IJ}\left(r_{ij}\right)\right] \\ + X_{ij} & = \sum_{k=i_1,k\neq j}^{i_N}W_{IK}\left(r_{ik}\right)\cdot G_{JIK}\left(\cos\theta_{jik}\right)\cdot P_{JIK}\left(\Delta r_{jik}\right) \\ \Delta r_{jik} & = r_{ij}-\xi_{IJ}\cdot r_{ik} where I, J, K represent species of atoms i, j, and k, :math:`i_1, ..., @@ -42,230 +43,267 @@ Dirac constant (i.e., :math:`\delta_{ij} = 1` when :math:`i = j`, and constant that can be set either to :math:`\eta_{ij} = \delta_{ij}` or :math:`\eta_{ij} = 1 - \delta_{ij}` depending on the potential type, :math:`U_{IJ}(r_{ij})`, :math:`V_{IJ}(r_{ij})`, :math:`W_{IK}(r_{ik})` -are pair functions, :math:`G_{JIK}(\cos(\theta))` is an angular -function, :math:`P_{IK}(\Delta r_{jik})` is a function of atomic spacing -differential :math:`\Delta r_{jik} = r_{ij} - \xi_{IJ} \cdot r_{ik}` -with :math:`\xi_{IJ}` being a pair-dependent parameter, and +are pair functions, :math:`G_{JIK}(\cos\theta_{jik})` is an angular +function, :math:`P_{JIK}(\Delta r_{jik})` is a function of atomic +spacing differential :math:`\Delta r_{jik} = r_{ij} - \xi_{IJ} \cdot +r_{ik}` with :math:`\xi_{IJ}` being a pair-dependent parameter, and :math:`F_{IJ}(X_{ij})` is a function of the local environment variable :math:`X_{ij}`. This generic potential is fully defined once the constants :math:`\eta_{ij}` and :math:`\xi_{IJ}`, and the six functions :math:`U_{IJ}(r_{ij})`, :math:`V_{IJ}(r_{ij})`, :math:`W_{IK}(r_{ik})`, -:math:`G_{JIK}(\cos(\theta))`, :math:`P_{IK}(\Delta r_{jik})`, and -:math:`F_{IJ}(X_{ij})` are given. Note that these six functions are all -one dimensional, and hence can be provided in an analytic or tabular -form. This allows users to design different potentials solely based on a -manipulation of these functions. For instance, the potential reduces to -Stillinger-Weber potential (:ref:`SW `) if we set +:math:`G_{JIK}(\cos\theta_{jik})`, :math:`P_{JIK}(\Delta r_{jik})`, and +:math:`F_{IJ}(X_{ij})` are given. Here LAMMPS uses a global parameter +:math:`\eta` to represent :math:`\eta_{ij}`. When :math:`\eta = 1`, +:math:`\eta_{ij} = 1 - \delta_{ij}`, otherwise :math:`\eta_{ij} = +\delta_{ij}`. Additionally, :math:`\eta = 3` indicates that the function +:math:`P_{JIK}(\Delta r)` depends on species I, J and K, otherwise +:math:`P_{JIK}(\Delta r) = P_{IK}(\Delta r)` only depends on species I +and K. Note that these six functions are all one dimensional, and hence +can be provided in a tabular form. This allows users to design different +potentials solely based on a manipulation of these functions. For +instance, the potential reduces to a Stillinger-Weber potential +(:ref:`SW `) if we set .. math:: - \left\{\begin{array}{l} - \eta_{ij} = \delta_{ij},\xi_{IJ}=0 \\ - U_{IJ}\left(r\right)=A_{IJ}\cdot\epsilon_{IJ}\cdot \left(\frac{\sigma_{IJ}}{r}\right)^q\cdot \left[B_{IJ}\cdot \left(\frac{\sigma_{IJ}}{r}\right)^{p-q}-1\right]\cdot exp\left(\frac{\sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ - V_{IJ}\left(r\right)=\sqrt{\lambda_{IJ}\cdot \epsilon_{IJ}}\cdot exp\left(\frac{\gamma_{IJ}\cdot \sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ - F_{IJ}\left(X\right)=-X \\ - P_{IJ}\left(\Delta r\right)=1 \\ - W_{IJ}\left(r\right)=\sqrt{\lambda_{IJ}\cdot \epsilon_{IJ}}\cdot exp\left(\frac{\gamma_{IJ}\cdot \sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ - G_{JIK}\left(\theta\right)=\left(cos\theta+\frac{1}{3}\right)^2 - \end{array}\right. + \eta_{ij} & = \delta_{ij} (\eta = 2~or~\eta = 0),\xi_{IJ}=0 \\ + U_{IJ}\left(r\right) & = A_{IJ}\cdot\epsilon_{IJ}\cdot \left(\frac{\sigma_{IJ}}{r}\right)^q\cdot \left[B_{IJ}\cdot \left(\frac{\sigma_{IJ}}{r}\right)^{p-q}-1\right]\cdot exp\left(\frac{\sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ + V_{IJ}\left(r\right) & = \sqrt{\lambda_{IJ}\cdot \epsilon_{IJ}}\cdot exp\left(\frac{\gamma_{IJ}\cdot \sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ + F_{IJ}\left(X\right) & = -X \\ + P_{JIK}\left(\Delta r\right) & = P_{IK}\left(\Delta r\right) = 1 \\ + W_{IJ}\left(r\right) & = \sqrt{\lambda_{IJ}\cdot \epsilon_{IJ}}\cdot exp\left(\frac{\gamma_{IJ}\cdot \sigma_{IJ}}{r-a_{IJ}\cdot \sigma_{IJ}}\right) \\ + G_{JIK}\left(\cos\theta\right) & = \left(\cos\theta+\frac{1}{3}\right)^2 -The potential reduces to Tersoff types of potential -(:ref:`Tersoff ` or :ref:`Albe `) if we set +The potential reduces to a Tersoff potential (:ref:`Tersoff ` +or :ref:`Albe `) if we set .. math:: - \left\{\begin{array}{l} - \eta_{ij}=\delta_{ij},\xi_{IJ}=1 \\ - U_{IJ}\left(r\right)=\frac{D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{2S_{IJ}\left(r-r_{e,IJ}\right)}\right]\cdot f_{c,IJ}\left(r\right) \\ - V_{IJ}\left(r\right)=\frac{S_{IJ}\cdot D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{\frac{2}{S_{IJ}}\left(r-r_{e,IJ}\right)}\right]\cdot f_{c,IJ}\left(r\right) \\ - F_{IJ}\left(X\right)=\left(1+X\right)^{-\frac{1}{2}} \\ - P_{IJ}\left(\Delta r\right)=exp\left(2\mu_{IK}\cdot \Delta r\right) \\ - W_{IJ}\left(r\right)=f_{c,IK}\left(r\right) \\ - G_{JIK}\left(\theta\right)=\gamma_{IK}\left[1+\frac{c_{IK}^2}{d_{IK}^2}-\frac{c_{IK}^2}{d_{IK}^2+\left(h_{IK}+cos\theta\right)^2}\right] - \end{array}\right. + \eta_{ij} & = \delta_{ij} (\eta = 2~or~\eta = 0),\xi_{IJ}=1 \\ + U_{IJ}\left(r\right) & = \frac{D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{2S_{IJ}}\left(r-r_{e,IJ}\right)\right]\cdot f_{c,IJ}\left(r\right) \\ + V_{IJ}\left(r\right) & = \frac{S_{IJ}\cdot D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{\frac{2}{S_{IJ}}}\left(r-r_{e,IJ}\right)\right]\cdot f_{c,IJ}\left(r\right) \\ + F_{IJ}\left(X\right) & = \left(1+X\right)^{-\frac{1}{2}} \\ + P_{JIK}\left(\Delta r\right) & = P_{IK}\left(\Delta r\right) = exp\left(2\mu_{IK}\cdot \Delta r\right) \\ + W_{IJ}\left(r\right) & = f_{c,IJ}\left(r\right) \\ + G_{JIK}\left(\cos\theta\right) & = \gamma_{IK}\left[1+\frac{c_{IK}^2}{d_{IK}^2}-\frac{c_{IK}^2}{d_{IK}^2+\left(h_{IK}+\cos\theta\right)^2}\right] + +where .. math:: - f_{c,IJ}=\left\{\begin{array}{lr} - 1, & r\leq r_{s,IJ} \\ - \frac{1}{2}+\frac{1}{2} cos \left[\frac{\pi \left(r-r_{s,IJ}\right)}{r_{c,IJ}-r_{s,IJ}}\right], & r_{s,IJ}`) type if we set +The potential reduces to a modified Stillinger-Weber potential +(:ref:`Zhou3 `) if we set .. math:: - \left\{\begin{array}{l} - \eta_{ij}=\delta_{ij},\xi_{IJ}=1 \\ - U_{IJ}\left(r\right)=\left\{\begin{array}{lr} - A_{IJ}\cdot exp\left(-\lambda_{1,IJ}\cdot r\right)\cdot f_{c,IJ}\left(r\right), & r\leq r_{s,1,IJ} \\ - A_{IJ}\cdot exp\left(-\lambda_{1,IJ}\cdot r\right)\cdot f_{c,IJ}\left(r\right)\cdot f_{c,1,IJ}\left(r\right), & r_{s,1,IJ}`) if we set .. math:: - f_{c,IJ}=\left\{\begin{array}{lr} - 1, & r\leq r_{s,IJ} \\ - \frac{1}{2}+\frac{1}{2} cos \left[\frac{\pi \left(r-r_{s,IJ}\right)}{r_{c,IJ}-r_{s,IJ}}\right], & r_{s,IJ}`) if we set +The potential becomes the embedded atom method (:ref:`Daw `) +if we set .. math:: - \left\{\begin{array}{l} - \eta_{ij}=1-\delta_{ij},\xi_{IJ}=0 \\ - U_{IJ}\left(r\right)=\phi_{IJ}\left(r\right) \\ - V_{IJ}\left(r\right)=1 \\ - F_{II}\left(X\right)=-2F_I\left(X\right) \\ - P_{IJ}\left(\Delta r\right)=1 \\ - W_{IJ}\left(r\right)=f_{K}\left(r\right) \\ - G_{JIK}\left(\theta\right)=1 - \end{array}\right. + \eta_{ij} & = 1-\delta_{ij} (\eta = 1),\xi_{IJ}=0 \\ + U_{IJ}\left(r\right) & = \phi_{IJ}\left(r\right) \\ + V_{IJ}\left(r\right) & = 1 \\ + F_{II}\left(X\right) & = -2F_I\left(X\right) \\ + P_{JIK}\left(\Delta r\right) & = P_{IK}\left(\Delta r\right) = 1 \\ + W_{IJ}\left(r\right) & = f_{J}\left(r\right) \\ + G_{JIK}\left(\cos\theta\right) & = 1 -In the embedded atom method case, :math:`\phi_{IJ}(r_{ij})` is the pair +In the embedded atom method case, :math:`\phi_{IJ}(r)` is the pair energy, :math:`F_I(X)` is the embedding energy, *X* is the local -electron density, and :math:`f_K(r)` is the atomic electron density function. +electron density, and :math:`f_J(r)` is the atomic electron density +function. -If the tabulated functions are created using the parameters of sw, -tersoff, and eam potentials, the polymorphic pair style will produce -the same global properties (energies and stresses) and the same forces -as the sw, tersoff, and eam pair styles. The polymorphic pair style -also produces the same atom properties (energies and stresses) as the -corresponding tersoff and eam pair styles. However, due to a different -partition of global properties to atom properties, the polymorphic -pair style will produce different atom properties (energies and -stresses) as the sw pair style. This does not mean that polymorphic -pair style is different from the sw pair style in this case. It just -means that the definitions of the atom energies and atom stresses are -different. +The potential reduces to another type of Tersoff potential (:ref:`Zhou4 +`) if we set -Only a single pair_coeff command is used with the polymorphic style -which specifies an potential file for all needed elements. These are +.. math:: + + \eta_{ij} & = \delta_{ij} (\eta = 3),\xi_{IJ}=1 \\ + U_{IJ}\left(r\right) & = \frac{D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{2S_{IJ}}\left(r-r_{e,IJ}\right)\right]\cdot f_{c,IJ}\left(r\right) \cdot T_{IJ}\left(r\right)+V_{ZBL,IJ}\left(r\right)\left[1-T_{IJ}\left(r\right)\right] \\ + V_{IJ}\left(r\right) & = \frac{S_{IJ}\cdot D_{e,IJ}}{S_{IJ}-1}\cdot exp\left[-\beta_{IJ}\sqrt{\frac{2}{S_{IJ}}}\left(r-r_{e,IJ}\right)\right]\cdot f_{c,IJ}\left(r\right) \cdot T_{IJ}\left(r\right) \\ + F_{IJ}\left(X\right) & = \left(1+X\right)^{-\frac{1}{2}} \\ + P_{JIK}\left(\Delta r\right) & = \omega_{JIK} \cdot exp\left(\alpha_{JIK}\cdot \Delta r\right) \\ + W_{IJ}\left(r\right) & = f_{c,IJ}\left(r\right) \\ + G_{JIK}\left(\cos\theta\right) & = \gamma_{JIK}\left[1+\frac{c_{JIK}^2}{d_{JIK}^2}-\frac{c_{JIK}^2}{d_{JIK}^2+\left(h_{JIK}+\cos\theta\right)^2}\right] \\ + T_{IJ}\left(r\right) & = \frac{1}{1+exp\left[-b_{f,IJ}\left(r-r_{f,IJ}\right)\right]} \\ + V_{ZBL,IJ}\left(r\right) & = 14.4 \cdot \frac{Z_I \cdot Z_J}{r}\sum_{k=1}^{4}\mu_k \cdot exp\left[-\nu_k \left(Z_I^{0.23}+Z_J^{0.23}\right) r\right] + +where :math:`f_{c,IJ}(r)` is the same as defined above. This Tersoff +potential differs from the one above because the :math:`P_{JIK}(\Delta +r)` function is now dependent on all three species I, J, and K. + +If the tabulated functions are created using the parameters of +Stillinger-Weber, Tersoff, and EAM potentials, the polymorphic pair +style will produce the same global properties (energies and stresses) +and the same forces as the :doc:`sw `, :doc:`tersoff +`, and :doc:`eam ` pair styles. The polymorphic +pair style also produces the same per-atom properties (energies and +stresses) as the corresponding :doc:`tersoff ` and +:doc:`eam ` pair styles. However, due to a different +partitioning of global properties to per-atom properties, the +polymorphic pair style will produce different per-atom properties +(energies and stresses) as the :doc:`sw ` pair style. This does +not mean that polymorphic pair style is different from the sw pair +style. It just means that the definitions of the atom energies and atom +stresses are different. + +Only a single pair_coeff command is used with the polymorphic pair style +which specifies a potential file for all needed elements. These are mapped to LAMMPS atom types by specifying N additional arguments after -the filename in the pair_coeff command, where N is the number of -LAMMPS atom types: +the filename in the pair_coeff command, where N is the number of LAMMPS +atom types: * filename -* N element names = mapping of Tersoff elements to atom types +* N element names = mapping of polymorphic potential elements to atom types See the pair_coeff doc page for alternate ways to specify the path for -the potential file. Several files for polymorphic potentials are -included in the potentials directory of the LAMMPS distribution. They +the potential file. Several files for polymorphic potentials are +included in the potentials directory of the LAMMPS distribution. They have a "poly" suffix. -As an example, imagine the SiC_tersoff.poly file has tabulated -functions for Si-C tersoff potential. If your LAMMPS simulation has 4 -atoms types and you want the 1st 3 to be Si, and the 4th to be C, you -would use the following pair_coeff command: +As an example, imagine the GaN_tersoff.poly file has tabulated functions +for Ga-N tersoff potential. If your LAMMPS simulation has 4 atoms types +and you want the 1st 3 to be Ga, and the 4th to be N, you would use the +following pair_coeff command: .. code-block:: LAMMPS - pair_coeff * * SiC_tersoff.poly Si Si Si C + pair_coeff * * GaN_tersoff.poly Ga Ga Ga N -The 1st 2 arguments must be \* \* so as to span all LAMMPS atom -types. The first three Si arguments map LAMMPS atom types 1,2,3 to the -Si element in the polymorphic file. The final C argument maps LAMMPS -atom type 4 to the C element in the polymorphic file. If a mapping -value is specified as NULL, the mapping is not performed. This can be -used when an polymorphic potential is used as part of the hybrid pair -style. The NULL values are placeholders for atom types that will be -used with other potentials. +The first two arguments must be \* \* to span all pairs of LAMMPS atom +types. The first three Ga arguments map LAMMPS atom types 1,2,3 to the +Ga element in the polymorphic file. The final N argument maps LAMMPS +atom type 4 to the N element in the polymorphic file. If a mapping value +is specified as NULL, the mapping is not performed. This can be used +when an polymorphic potential is used as part of the hybrid pair +style. The NULL values are placeholders for atom types that will be used +with other potentials. Potential files in the potentials directory of the LAMMPS distribution have a ".poly" suffix. At the beginning of the files, an unlimited -number of lines starting with '#' are used to describe the potential -and are ignored by LAMMPS. The next line lists two numbers: +number of lines starting with '#' are used to describe the potential and +are ignored by LAMMPS. The next line lists two numbers: .. parsed-literal:: - ntypes :math:`\eta` + ntypes eta -Here ntypes represent total number of species defined in the potential -file, and :math:`\eta = 0` or 1. The number ntypes must equal the total -number of different species defined in the pair_coeff command. When -:math:`\eta = 1`, :math:\eta_{ij}` defined in the potential functions -above is set to :math:`1 - \delta_{ij}`, otherwise :math:`\eta_{ij}` is -set to :math:`\delta_{ij}`. The next ntypes lines each lists two numbers -and a character string representing atomic number, atomic mass, and name -of the species of the ntypes elements: +Here *ntypes* represent total number of species defined in the potential +file, :math:`\eta = 1` reduces to embedded atom method, :math:`\eta = 3` +assumes a three species dependent :math:`P_{JIK}(\Delta r)` function, +and all other values of :math:`\eta` assume a two species dependent +:math:`P_{JK}(\Delta r)` function. The value of *ntypes* must equal the +total number of different species defined in the pair_coeff command. The +next *ntypes* lines each lists two numbers and a character string +representing atomic number, atomic mass, and name of the species of the +ntypes elements: .. parsed-literal:: - atomic_number atomic-mass element (1) - atomic_number atomic-mass element (2) + atomic-number atomic-mass element-name(1) + atomic-number atomic-mass element-name(2) ... - atomic_number atomic-mass element (ntypes) + atomic-number atomic-mass element-name(ntypes) + +The next line contains four numbers: + +.. parsed-literal:: + + nr ntheta nx xmax + +Here nr is total number of tabular points for radial functions U, V, W, +P, ntheta is total number of tabular points for the angular function G, +nx is total number of tabular points for the function F, xmax is a +maximum value of the argument of function F. Note that the pair +functions :math:`U_{IJ}(r)`, :math:`V_{IJ}(r)`, :math:`W_{IJ}(r)` are +uniformly tabulated between 0 and cutoff distance of the IJ pair, +:math:`G_{JIK}(\cos\theta)` is uniformly tabulated between -1 and 1, +:math:`P_{JIK}(\Delta r)` is uniformly tabulated between -rcmax and +rcmax where rcmax is the maximum cutoff distance of all pairs, and +:math:`F_{IJ}(X)` is uniformly tabulated between 0 and xmax. Linear +extrapolation is assumed if actual simulations exceed these ranges. The next ntypes\*(ntypes+1)/2 lines contain two numbers: .. parsed-literal:: - cut :math:`xi` (1) - cut :math:`xi` (2) + cut xi(1) + cut xi(2) ... - cut :math:`xi` (ntypes\*(ntypes+1)/2) + cut xi(ntypes\*(ntypes+1)/2) -Here cut means the cutoff distance of the pair functions, :math:`\xi` is -the same as defined in the potential functions above. The +Here cut means the cutoff distance of the pair functions, "xi" is +:math:`\xi` as defined in the potential functions above. The ntypes\*(ntypes+1)/2 lines are related to the pairs according to the -sequence of first ii (self) pairs, i = 1, 2, ..., ntypes, and then then -ij (cross) pairs, i = 1, 2, ..., ntypes-1, and j = i+1, i+2, ..., ntypes +sequence of first ii (self) pairs, i = 1, 2, ..., ntypes, and then ij +(cross) pairs, i = 1, 2, ..., ntypes-1, and j = i+1, i+2, ..., ntypes (i.e., the sequence of the ij pairs follows 11, 22, ..., 12, 13, 14, ..., 23, 24, ...). -The final blocks of the potential file are the U, V, W, P, G, and F -functions are listed sequentially. First, U functions are given for -each of the ntypes\*(ntypes+1)/2 pairs according to the sequence -described above. For each of the pairs, nr values are listed. Next, -similar arrays are given for V, W, and P functions. Then G functions -are given for all the ntypes\*ntypes\*ntypes ijk triplets in a natural -sequence i from 1 to ntypes, j from 1 to ntypes, and k from 1 to -ntypes (i.e., ijk = 111, 112, 113, ..., 121, 122, 123 ..., 211, 212, -...). Each of the ijk functions contains ng values. Finally, the F -functions are listed for all ntypes\*(ntypes+1)/2 pairs, each -containing nx values. Either analytic or tabulated functions can be -specified. Currently, constant, exponential, sine and cosine analytic -functions are available which are specified with: constant c1 , where -f(x) = c1 exponential c1 c2 , where f(x) = c1 exp(c2\*x) sine c1 c2 , -where f(x) = c1 sin(c2\*x) cos c1 c2 , where f(x) = c1 cos(c2\*x) -Tabulated functions are specified by spline n x1 x2, where n=number of -point, (x1,x2)=range and then followed by n values evaluated uniformly -over these argument ranges. The valid argument ranges of the -functions are between 0 <= r <= cut for the U(r), V(r), W(r) -functions, -cutmax <= delta_r <= cutmax for the P(delta_r) functions, --1 <= :math:`\cos\theta` <= 1 for the G(:math:`\cos\theta`) functions, -and 0 <= X <= maxX for the F(X) functions. +In the final blocks of the potential file, U, V, W, P, G, and F +functions are listed sequentially. First, U functions are given for each +of the ntypes\*(ntypes+1)/2 pairs according to the sequence described +above. For each of the pairs, nr values are listed. Next, similar arrays +are given for V and W functions. If P functions depend only on pair +species, i.e., :math:`\eta \neq 3`, then P functions are also listed the +same way the next. If P functions depend on three species, i.e., +:math:`\eta = 3`, then P functions are listed for all the +ntypes*ntypes*ntypes IJK triplets in a natural sequence I from 1 to +ntypes, J from 1 to ntypes, and K from 1 to ntypes (i.e., IJK = 111, +112, 113, ..., 121, 122, 123 ..., 211, 212, ...). Next, G functions are +listed for all the ntypes*ntypes*ntypes IJK triplets similarly. For each +of the G functions, ntheta values are listed. Finally, F functions are +listed for all the ntypes*(ntypes+1)/2 pairs in the same sequence as +described above. For each of the F functions, nx values are listed. **Mixing, shift, table tail correction, restart**\ : This pair styles does not support the :doc:`pair_modify ` shift, table, and tail options. -This pair style does not write their information to :doc:`binary restart files `, since it is stored in potential files. Thus, you -need to re-specify the pair_style and pair_coeff commands in an input -script that reads a restart file. +This pair style does not write their information to :doc:`binary restart +files `, since it is stored in potential files. Thus, you need +to re-specify the pair_style and pair_coeff commands in an input script +that reads a restart file. ---------- @@ -277,15 +315,15 @@ input script. If using read_data, atomic masses must be defined in the atomic structure data file. This pair style is part of the MANYBODY package. It is only enabled if -LAMMPS was built with that package. See the :doc:`Build package ` doc page for more info. +LAMMPS was built with that package. See the :doc:`Build package +` doc page for more info. This pair potential requires the :doc:`newtion ` setting to be "on" for pair interactions. -The potential files provided with LAMMPS (see the potentials -directory) are parameterized for metal :doc:`units `. You can use -any LAMMPS units, but you would need to create your own potential -files. +The potential files provided with LAMMPS (see the potentials directory) +are parameterized for metal :doc:`units `. You can use any LAMMPS +units, but you would need to create your own potential files. Related commands """""""""""""""" @@ -296,12 +334,15 @@ Related commands .. _Zhou3: -**(Zhou)** X. W. Zhou, M. E. Foster, R. E. Jones, P. Yang, H. Fan, and -F. P. Doty, J. Mater. Sci. Res., 4, 15 (2015). +**(Zhou3)** X. W. Zhou, M. E. Foster, R. E. Jones, P. Yang, H. Fan, and F. P. Doty, J. Mater. Sci. Res., 4, 15 (2015). + +.. _Zhou4: + +**(Zhou4)** X. W. Zhou, M. E. Foster, J. A. Ronevich, and C. W. San Marchi, J. Comp. Chem., 41, 1299 (2020). .. _SW: -**(SW)** F. H. Stillinger-Weber, and T. A. Weber, Phys. Rev. B, 31, 5262 (1985). +**(SW)** F. H. Stillinger, and T. A. Weber, Phys. Rev. B, 31, 5262 (1985). .. _Tersoff: @@ -309,8 +350,7 @@ F. P. Doty, J. Mater. Sci. Res., 4, 15 (2015). .. _poly-Albe: -**(Albe)** K. Albe, K. Nordlund, J. Nord, and A. Kuronen, Phys. Rev. B, -66, 035205 (2002). +**(Albe)** K. Albe, K. Nordlund, J. Nord, and A. Kuronen, Phys. Rev. B, 66, 035205 (2002). .. _Wang3: diff --git a/doc/src/pair_reaxc.rst b/doc/src/pair_reaxc.rst index 1481b0bf4f..b01f046d29 100644 --- a/doc/src/pair_reaxc.rst +++ b/doc/src/pair_reaxc.rst @@ -21,12 +21,13 @@ Syntax .. parsed-literal:: - keyword = *checkqeq* or *lgvdw* or *safezone* or *mincap* + keyword = *checkqeq* or *lgvdw* or *safezone* or *mincap* or *minhbonds* *checkqeq* value = *yes* or *no* = whether or not to require qeq/reax fix *enobonds* value = *yes* or *no* = whether or not to tally energy of atoms with no bonds *lgvdw* value = *yes* or *no* = whether or not to use a low gradient vdW correction *safezone* = factor used for array allocation *mincap* = minimum size for array allocation + *minhbonds* = minimum size use for storing hydrogen bonds Examples """""""" @@ -146,11 +147,11 @@ zero. The latter behavior is usual not desired, as it causes discontinuities in the potential energy when the bonding of an atom drops to zero. -Optional keywords *safezone* and *mincap* are used for allocating -reax/c arrays. Increasing these values can avoid memory problems, -such as segmentation faults and bondchk failed errors, that could -occur under certain conditions. These keywords are not used by the -Kokkos version, which instead uses a more robust memory allocation +Optional keywords *safezone*\ , *mincap*\ , and *minhbonds* are used +for allocating reax/c arrays. Increasing these values can avoid memory +problems, such as segmentation faults and bondchk failed errors, that +could occur under certain conditions. These keywords are not used by +the Kokkos version, which instead uses a more robust memory allocation scheme that checks if the sizes of the arrays have been exceeded and automatically allocates more memory. @@ -352,7 +353,7 @@ Default """"""" The keyword defaults are checkqeq = yes, enobonds = yes, lgvdw = no, -safezone = 1.2, mincap = 50. +safezone = 1.2, mincap = 50, minhbonds = 25. ---------- diff --git a/doc/src/pair_style.rst b/doc/src/pair_style.rst index 294ea15b1f..5a5081c67e 100644 --- a/doc/src/pair_style.rst +++ b/doc/src/pair_style.rst @@ -111,41 +111,41 @@ accelerated styles exist. * :doc:`born ` - Born-Mayer-Huggins potential * :doc:`born/coul/dsf ` - Born with damped-shifted-force model * :doc:`born/coul/dsf/cs ` - Born with damped-shifted-force and core/shell model -* :doc:`born/coul/long ` - Born with long-range Coulombics -* :doc:`born/coul/long/cs ` - Born with long-range Coulombics and core/shell -* :doc:`born/coul/msm ` - Born with long-range MSM Coulombics -* :doc:`born/coul/wolf ` - Born with Wolf potential for Coulombics -* :doc:`born/coul/wolf/cs ` - Born with Wolf potential for Coulombics and core/shell model +* :doc:`born/coul/long ` - Born with long-range Coulomb +* :doc:`born/coul/long/cs ` - Born with long-range Coulomb and core/shell +* :doc:`born/coul/msm ` - Born with long-range MSM Coulomb +* :doc:`born/coul/wolf ` - Born with Wolf potential for Coulomb +* :doc:`born/coul/wolf/cs ` - Born with Wolf potential for Coulomb and core/shell model * :doc:`brownian ` - Brownian potential for Fast Lubrication Dynamics * :doc:`brownian/poly ` - Brownian potential for Fast Lubrication Dynamics with polydispersity * :doc:`buck ` - Buckingham potential * :doc:`buck/coul/cut ` - Buckingham with cutoff Coulomb -* :doc:`buck/coul/long ` - Buckingham with long-range Coulombics -* :doc:`buck/coul/long/cs ` - Buckingham with long-range Coulombics and core/shell -* :doc:`buck/coul/msm ` - Buckingham with long-range MSM Coulombics -* :doc:`buck/long/coul/long ` - long-range Buckingham with long-range Coulombics +* :doc:`buck/coul/long ` - Buckingham with long-range Coulomb +* :doc:`buck/coul/long/cs ` - Buckingham with long-range Coulomb and core/shell +* :doc:`buck/coul/msm ` - Buckingham with long-range MSM Coulomb +* :doc:`buck/long/coul/long ` - long-range Buckingham with long-range Coulomb * :doc:`buck/mdf ` - Buckingham with a taper function * :doc:`buck6d/coul/gauss/dsf ` - dispersion-damped Buckingham with damped-shift-force model -* :doc:`buck6d/coul/gauss/long ` - dispersion-damped Buckingham with long-range Coulombics +* :doc:`buck6d/coul/gauss/long ` - dispersion-damped Buckingham with long-range Coulomb * :doc:`colloid ` - integrated colloidal potential * :doc:`comb ` - charge-optimized many-body (COMB) potential * :doc:`comb3 ` - charge-optimized many-body (COMB3) potential * :doc:`cosine/squared ` - Cooke-Kremer-Deserno membrane model potential -* :doc:`coul/cut ` - cutoff Coulombic potential -* :doc:`coul/cut/soft ` - Coulombic potential with a soft core -* :doc:`coul/debye ` - cutoff Coulombic potential with Debye screening +* :doc:`coul/cut ` - cutoff Coulomb potential +* :doc:`coul/cut/soft ` - Coulomb potential with a soft core +* :doc:`coul/debye ` - cutoff Coulomb potential with Debye screening * :doc:`coul/diel ` - Coulomb potential with dielectric permittivity -* :doc:`coul/dsf ` - Coulombics with damped-shifted-force model -* :doc:`coul/long ` - long-range Coulombic potential -* :doc:`coul/long/cs ` - long-range Coulombic potential and core/shell -* :doc:`coul/long/soft ` - long-range Coulombic potential with a soft core -* :doc:`coul/msm ` - long-range MSM Coulombics -* :doc:`coul/slater/cut ` - smeared out Coulombics -* :doc:`coul/slater/long ` - long-range smeared out Coulombics -* :doc:`coul/shield ` - Coulombics for boron nitride for use with :doc:`ilp/graphene/hbn ` potential -* :doc:`coul/streitz ` - Coulombics via Streitz/Mintmire Slater orbitals -* :doc:`coul/wolf ` - Coulombics via Wolf potential -* :doc:`coul/wolf/cs ` - ditto with core/shell adjustments +* :doc:`coul/dsf ` - Coulomb with damped-shifted-force model +* :doc:`coul/long ` - long-range Coulomb potential +* :doc:`coul/long/cs ` - long-range Coulomb potential and core/shell +* :doc:`coul/long/soft ` - long-range Coulomb potential with a soft core +* :doc:`coul/msm ` - long-range MSM Coulomb +* :doc:`coul/slater/cut ` - smeared out Coulomb +* :doc:`coul/slater/long ` - long-range smeared out Coulomb +* :doc:`coul/shield ` - Coulomb for boron nitride for use with :doc:`ilp/graphene/hbn ` potential +* :doc:`coul/streitz ` - Coulomb via Streitz/Mintmire Slater orbitals +* :doc:`coul/wolf ` - Coulomb via Wolf potential +* :doc:`coul/wolf/cs ` - Coulomb via Wolf potential with core/shell adjustments * :doc:`dpd ` - dissipative particle dynamics (DPD) * :doc:`dpd/fdt ` - DPD for constant temperature and pressure * :doc:`dpd/fdt/energy ` - DPD for constant energy and enthalpy @@ -189,44 +189,45 @@ accelerated styles exist. * :doc:`lj/charmm/coul/charmm/implicit ` - CHARMM for implicit solvent * :doc:`lj/charmm/coul/long ` - CHARMM with long-range Coulomb * :doc:`lj/charmm/coul/long/soft ` - CHARMM with long-range Coulomb and a soft core -* :doc:`lj/charmm/coul/msm ` - CHARMM with long-range MSM Coulombics +* :doc:`lj/charmm/coul/msm ` - CHARMM with long-range MSM Coulomb * :doc:`lj/charmmfsw/coul/charmmfsh ` - CHARMM with force switching and shifting -* :doc:`lj/charmmfsw/coul/long ` - CHARMM with force switching and long-rnage Coulombics -* :doc:`lj/class2 ` - COMPASS (class 2) force field with no Coulomb +* :doc:`lj/charmmfsw/coul/long ` - CHARMM with force switching and long-rnage Coulomb +* :doc:`lj/class2 ` - COMPASS (class 2) force field without Coulomb * :doc:`lj/class2/coul/cut ` - COMPASS with cutoff Coulomb * :doc:`lj/class2/coul/cut/soft ` - COMPASS with cutoff Coulomb with a soft core * :doc:`lj/class2/coul/long ` - COMPASS with long-range Coulomb +* :doc:`lj/class2/coul/long/cs ` - COMPASS with long-range Coulomb with core/shell adjustments * :doc:`lj/class2/coul/long/soft ` - COMPASS with long-range Coulomb with a soft core * :doc:`lj/class2/soft ` - COMPASS (class 2) force field with no Coulomb with a soft core * :doc:`lj/cubic ` - LJ with cubic after inflection point -* :doc:`lj/cut ` - cutoff Lennard-Jones potential with no Coulomb +* :doc:`lj/cut ` - cutoff Lennard-Jones potential without Coulomb * :doc:`lj/cut/coul/cut ` - LJ with cutoff Coulomb * :doc:`lj/cut/coul/cut/soft ` - LJ with cutoff Coulomb with a soft core * :doc:`lj/cut/coul/debye ` - LJ with Debye screening added to Coulomb -* :doc:`lj/cut/coul/dsf ` - LJ with Coulombics via damped shifted forces -* :doc:`lj/cut/coul/long ` - LJ with long-range Coulombics -* :doc:`lj/cut/coul/long/cs ` - ditto with core/shell adjustments -* :doc:`lj/cut/coul/long/soft ` - LJ with long-range Coulombics with a soft core -* :doc:`lj/cut/coul/msm ` - LJ with long-range MSM Coulombics -* :doc:`lj/cut/coul/wolf ` - LJ with Coulombics via Wolf potential +* :doc:`lj/cut/coul/dsf ` - LJ with Coulomb via damped shifted forces +* :doc:`lj/cut/coul/long ` - LJ with long-range Coulomb +* :doc:`lj/cut/coul/long/cs ` - LJ with long-range Coulomb with core/shell adjustments +* :doc:`lj/cut/coul/long/soft ` - LJ with long-range Coulomb with a soft core +* :doc:`lj/cut/coul/msm ` - LJ with long-range MSM Coulomb +* :doc:`lj/cut/coul/wolf ` - LJ with Coulomb via Wolf potential * :doc:`lj/cut/dipole/cut ` - point dipoles with cutoff * :doc:`lj/cut/dipole/long ` - point dipoles with long-range Ewald * :doc:`lj/cut/soft ` - LJ with a soft core -* :doc:`lj/cut/thole/long ` - LJ with Coulombics with thole damping +* :doc:`lj/cut/thole/long ` - LJ with Coulomb with thole damping * :doc:`lj/cut/tip4p/cut ` - LJ with cutoff Coulomb for TIP4P water * :doc:`lj/cut/tip4p/long ` - LJ with long-range Coulomb for TIP4P water * :doc:`lj/cut/tip4p/long/soft ` - LJ with cutoff Coulomb for TIP4P water with a soft core * :doc:`lj/expand ` - Lennard-Jones for variable size particles -* :doc:`lj/expand/coul/long ` - Lennard-Jones for variable size particles with long-range Coulombics +* :doc:`lj/expand/coul/long ` - Lennard-Jones for variable size particles with long-range Coulomb * :doc:`lj/gromacs ` - GROMACS-style Lennard-Jones potential -* :doc:`lj/gromacs/coul/gromacs ` - GROMACS-style LJ and Coulombic potential -* :doc:`lj/long/coul/long ` - long-range LJ and long-range Coulombics +* :doc:`lj/gromacs/coul/gromacs ` - GROMACS-style LJ and Coulomb potential +* :doc:`lj/long/coul/long ` - long-range LJ and long-range Coulomb * :doc:`lj/long/dipole/long ` - long-range LJ and long-range point dipoles -* :doc:`lj/long/tip4p/long ` - long-range LJ and long-range Coulombics for TIP4P water +* :doc:`lj/long/tip4p/long ` - long-range LJ and long-range Coulomb for TIP4P water * :doc:`lj/mdf ` - LJ potential with a taper function * :doc:`lj/sdk ` - LJ for SDK coarse-graining -* :doc:`lj/sdk/coul/long ` - LJ for SDK coarse-graining with long-range Coulombics -* :doc:`lj/sdk/coul/msm ` - LJ for SDK coarse-graining with long-range Coulombics via MSM +* :doc:`lj/sdk/coul/long ` - LJ for SDK coarse-graining with long-range Coulomb +* :doc:`lj/sdk/coul/msm ` - LJ for SDK coarse-graining with long-range Coulomb via MSM * :doc:`lj/sf/dipole/sf ` - LJ with dipole interaction with shifted forces * :doc:`lj/smooth ` - smoothed Lennard-Jones potential * :doc:`lj/smooth/linear ` - linear smoothed LJ potential @@ -256,7 +257,7 @@ accelerated styles exist. * :doc:`nb3b/harmonic ` - non-bonded 3-body harmonic potential * :doc:`nm/cut ` - N-M potential * :doc:`nm/cut/coul/cut ` - N-M potential with cutoff Coulomb -* :doc:`nm/cut/coul/long ` - N-M potential with long-range Coulombics +* :doc:`nm/cut/coul/long ` - N-M potential with long-range Coulomb * :doc:`oxdna/coaxstk ` - * :doc:`oxdna/excv ` - * :doc:`oxdna/hbond ` - @@ -316,7 +317,7 @@ accelerated styles exist. * :doc:`tersoff/zbl ` - Tersoff/ZBL 3-body potential * :doc:`thole ` - Coulomb interactions with thole damping * :doc:`tip4p/cut ` - Coulomb for TIP4P water w/out LJ -* :doc:`tip4p/long ` - long-range Coulombics for TIP4P water w/out LJ +* :doc:`tip4p/long ` - long-range Coulomb for TIP4P water w/out LJ * :doc:`tip4p/long/soft ` - * :doc:`tri/lj ` - LJ potential between triangles * :doc:`ufm ` - diff --git a/doc/src/prd.rst b/doc/src/prd.rst index c3bd1ad1d3..d1acaf645e 100644 --- a/doc/src/prd.rst +++ b/doc/src/prd.rst @@ -52,19 +52,16 @@ replicas of a system. One or more replicas can be used. The total number of steps *N* to run can be interpreted in one of two ways; see discussion of the *time* keyword below. -PRD is described in :ref:`(Voter1998) ` by Art Voter. Similar to -global or local hyperdynamics (HD), PRD is a method for performing -accelerated dynamics that is suitable for infrequent-event systems -that obey first-order kinetics. A good overview of accelerated -dynamics methods for such systems in given in this review paper -:ref:`(Voter2002) ` from Art's group. To quote from the -paper: "The dynamical evolution is characterized by vibrational -excursions within a potential basin, punctuated by occasional -transitions between basins." The transition probability is -characterized by p(t) = k\*exp(-kt) where k is the rate constant. -Running multiple replicas gives an effective enhancement in the -timescale spanned by the multiple simulations, while waiting for an -event to occur. +PRD is described in :ref:`(Voter1998) ` by Art Voter. +Similar to global or local hyperdynamics (HD), PRD is a method for +performing accelerated dynamics that is suitable for infrequent-event +systems that obey first-order kinetics. A good overview of +accelerated dynamics methods (AMD) for such systems in given in this +review paper :ref:`(Voter2002) ` from Art's group. To +quote from the paper: "The dynamical evolution is characterized by +vibrational excursions within a potential basin, punctuated by +occasional transitions between basins. The transition probability is +characterized by p(t) = k\*exp(-kt) where k is the rate constant." Both PRD and HD produce a time-accurate trajectory that effectively extends the timescale over which a system can be simulated, but they diff --git a/doc/src/read_data.rst b/doc/src/read_data.rst index 642f930b9d..79f0e08fcb 100644 --- a/doc/src/read_data.rst +++ b/doc/src/read_data.rst @@ -107,11 +107,11 @@ command, after the third read_data command is used. The *add*\ , *offset*\ , *shift*\ , *extra*\ , and *group* keywords are useful in this context. -If a simulation box does not yet exist, the *add* keyword -cannot be used; the read_data command is being used for the first -time. If a simulation box does exist, due to using the -:doc:`create_box ` command, or a previous read_data command, -then the *add* keyword must be used. +If a simulation box does not yet exist, the *add* keyword cannot be +used; the read_data command is being used for the first time. If a +simulation box does exist, due to using the :doc:`create_box +` command, or a previous read_data command, then the *add* +keyword must be used. .. note:: @@ -571,68 +571,69 @@ appended to it, which indicate which image of a periodic simulation box the atom is in. These may be important to include for some kinds of analysis. -+------------+---------------------------------------------------------------------------+ -| angle | atom-ID molecule-ID atom-type x y z | -+------------+---------------------------------------------------------------------------+ -| atomic | atom-ID atom-type x y z | -+------------+---------------------------------------------------------------------------+ -| body | atom-ID atom-type bodyflag mass x y z | -+------------+---------------------------------------------------------------------------+ -| bond | atom-ID molecule-ID atom-type x y z | -+------------+---------------------------------------------------------------------------+ -| charge | atom-ID atom-type q x y z | -+------------+---------------------------------------------------------------------------+ -| dipole | atom-ID atom-type q x y z mux muy muz | -+------------+---------------------------------------------------------------------------+ -| dpd | atom-ID atom-type theta x y z | -+------------+---------------------------------------------------------------------------+ -| edpd | atom-ID atom-type edpd_temp edpd_cv x y z | -+------------+---------------------------------------------------------------------------+ -| mdpd | atom-ID atom-type rho x y z | -+------------+---------------------------------------------------------------------------+ -| tdpd | atom-ID atom-type x y z cc1 cc2 ... ccNspecies | -+------------+---------------------------------------------------------------------------+ -| electron | atom-ID atom-type q spin eradius x y z | -+------------+---------------------------------------------------------------------------+ -| ellipsoid | atom-ID atom-type ellipsoidflag density x y z | -+------------+---------------------------------------------------------------------------+ -| full | atom-ID molecule-ID atom-type q x y z | -+------------+---------------------------------------------------------------------------+ -| line | atom-ID molecule-ID atom-type lineflag density x y z | -+------------+---------------------------------------------------------------------------+ -| meso | atom-ID atom-type rho e cv x y z | -+------------+---------------------------------------------------------------------------+ -| molecular | atom-ID molecule-ID atom-type x y z | -+------------+---------------------------------------------------------------------------+ -| peri | atom-ID atom-type volume density x y z | -+------------+---------------------------------------------------------------------------+ -| smd | atom-ID atom-type molecule volume mass kernel-radius contact-radius x y z | -+------------+---------------------------------------------------------------------------+ -| sphere | atom-ID atom-type diameter density x y z | -+------------+---------------------------------------------------------------------------+ -| spin | atom-ID atom-type sp x y z spx spy spz | -+------------+---------------------------------------------------------------------------+ -| template | atom-ID molecule-ID template-index template-atom atom-type x y z | -+------------+---------------------------------------------------------------------------+ -| tri | atom-ID molecule-ID atom-type triangleflag density x y z | -+------------+---------------------------------------------------------------------------+ -| wavepacket | atom-ID atom-type charge spin eradius etag cs_re cs_im x y z | -+------------+---------------------------------------------------------------------------+ -| hybrid | atom-ID atom-type x y z sub-style1 sub-style2 ... | -+------------+---------------------------------------------------------------------------+ +.. list-table:: + + * - angle + - atom-ID molecule-ID atom-type x y z + * - atomic + - atom-ID atom-type x y z + * - body + - atom-ID atom-type bodyflag mass x y z + * - bond + - atom-ID molecule-ID atom-type x y z + * - charge + - atom-type q x y z + * - dipole + - atom-ID atom-type q x y z mux muy muz + * - dpd + - atom-ID atom-type theta x y z + * - edpd + - atom-ID atom-type edpd_temp edpd_cv x y z + * - electron + - atom-ID atom-type q spin eradius x y z + * - ellipsoid + - atom-ID atom-type ellipsoidflag density x y z + * - full + - atom-ID molecule-ID atom-type q x y z + * - line + - atom-ID molecule-ID atom-type lineflag density x y z + * - mdpd + - atom-ID atom-type rho x y z + * - molecular + - atom-ID molecule-ID atom-type x y z + * - peri + - atom-ID atom-type volume density x y z + * - smd + - atom-ID atom-type molecule volume mass kernel-radius contact-radius x0 y0 z0 x y z + * - sph + - atom-ID atom-type rho esph cv x y z + * - sphere + - atom-ID atom-type diameter density x y z + * - spin + - atom-ID atom-type x y z spx spy spz sp + * - tdpd + - atom-ID atom-type x y z cc1 cc2 ... ccNspecies + * - template + - atom-ID molecule-ID template-index template-atom atom-type x y z + * - tri + - atom-ID molecule-ID atom-type triangleflag density x y z + * - wavepacket + - atom-ID atom-type charge spin eradius etag cs_re cs_im x y z + * - hybrid + - atom-ID atom-type x y z sub-style1 sub-style2 ... The per-atom values have these meanings and units, listed alphabetically: * atom-ID = integer ID of atom * atom-type = type of atom (1-Ntype) * bodyflag = 1 for body particles, 0 for point particles -* cc = chemical concentration for tDPD particles for each species (mole/volume units) +* ccN = chemical concentration for tDPD particles for each species (mole/volume units) * contact-radius = ??? (distance units) * cs_re,cs_im = real/imaginary parts of wave packet coefficients * cv = heat capacity (need units) for SPH particles * density = density of particle (mass/distance\^3 or mass/distance\^2 or mass/distance units, depending on dimensionality of particle) * diameter = diameter of spherical atom (distance units) -* e = energy (need units) for SPH particles +* esph = energy (need units) for SPH particles * edpd_temp = temperature for eDPD particles (temperature units) * edpd_cv = volumetric heat capacity for eDPD particles (energy/temperature/volume units) * ellipsoidflag = 1 for ellipsoidal particles, 0 for point particles @@ -646,14 +647,15 @@ The per-atom values have these meanings and units, listed alphabetically: * q = charge on atom (charge units) * rho = density (need units) for SPH particles * spin = electron spin (+1/-1), 0 = nuclei, 2 = fixed-core, 3 = pseudo-cores (i.e. ECP) -* sp = norm of magnetic spin of atom (in number of Bohr magnetons) -* spx,spy,spz = components of magnetic spin of atom (adim normalized vector) +* sp = magnitude of magnetic spin of atom (Bohr magnetons) +* spx,spy,spz = components of magnetic spin of atom (unit vector) * template-atom = which atom within a template molecule the atom is * template-index = which molecule within the molecule template the atom is part of * theta = internal temperature of a DPD particle * triangleflag = 1 for triangular particles, 0 for point or spherical particles * volume = volume of Peridynamic particle (distance\^3 units) * x,y,z = coordinates of atom (distance units) +* x0,y0,z0 = original (strain-free) coordinates of atom (distance units) The units for these quantities depend on the unit style; see the :doc:`units ` command for details. @@ -715,17 +717,18 @@ as spheres when converting density to mass. However, they can also be modeled as 2d discs (circles) if the :doc:`set density/disc ` command is used to reset their mass after the read_data command is used. A *disc* keyword can also be used with time integration fixes, -such as :doc:`fix nve/sphere ` and :doc:`fix nvt/sphere ` to time integrate their motion as 2d +such as :doc:`fix nve/sphere ` and :doc:`fix +nvt/sphere ` to time integrate their motion as 2d discs (not 3d spheres), by changing their moment of inertia. -For atom_style hybrid, following the 5 initial values (ID,type,x,y,z), -specific values for each sub-style must be listed. The order of the -sub-styles is the same as they were listed in the -:doc:`atom_style ` command. The sub-style specific values -are those that are not the 5 standard ones (ID,type,x,y,z). For -example, for the "charge" sub-style, a "q" value would appear. For -the "full" sub-style, a "molecule-ID" and "q" would appear. These are -listed in the same order they appear as listed above. Thus if +For atom\_style hybrid, following the 5 initial values +(ID,type,x,y,z), specific values for each sub-style must be listed. +The order of the sub-styles is the same as they were listed in the +:doc:`atom_style ` command. The specific values for each +sub-style are those that are not the 5 standard ones (ID,type,x,y,z). +For example, for the "charge" sub-style, a "q" value would appear. +For the "full" sub-style, a "molecule-ID" and "q" would appear. These +are listed in the same order they appear as listed above. Thus if .. parsed-literal:: @@ -738,12 +741,14 @@ were used in the input script, each atom line would have these fields: atom-ID atom-type x y z q diameter density Note that if a non-standard value is defined by multiple sub-styles, -it must appear multiple times in the atom line. E.g. the atom line -for atom_style hybrid dipole full would list "q" twice: +it only appears once in the atom line. E.g. the atom line for +atom_style hybrid dipole full would list "q" only once, with the +dipole sub-style fields; "q" does not appear with the full sub-style +fields. .. parsed-literal:: - atom-ID atom-type x y z q mux muy myz molecule-ID q + atom-ID atom-type x y z q mux muy myz molecule-ID Atom lines specify the (x,y,z) coordinates of atoms. These can be inside or outside the simulation box. When the data file is read, @@ -1176,9 +1181,9 @@ pair style. See the :doc:`pair_style ` and :doc:`pair_coeff ` commands for details. Since pair coefficients for types I != J are not specified, these will be generated automatically by the pair style's mixing rule. See the -individual pair_style doc pages and the :doc:`pair_modify mix ` command for details. Pair coefficients can also -be set via the :doc:`pair_coeff ` command in the input -script. +individual pair_style doc pages and the :doc:`pair_modify mix +` command for details. Pair coefficients can also be set +via the :doc:`pair_coeff ` command in the input script. ---------- @@ -1200,15 +1205,15 @@ script. 3 3 0.022 2.35197 0.022 2.35197 3 5 0.022 2.35197 0.022 2.35197 -This section must have N\*(N+1)/2 lines where N = # of atom types. The -number and meaning of the coefficients are specific to the defined +This section must have N\*(N+1)/2 lines where N = # of atom types. +The number and meaning of the coefficients are specific to the defined pair style. See the :doc:`pair_style ` and :doc:`pair_coeff ` commands for details. Since pair coefficients for types I != J are all specified, these values will turn off the default mixing rule defined by the pair style. See the -individual pair_style doc pages and the :doc:`pair_modify mix ` command for details. Pair coefficients can also -be set via the :doc:`pair_coeff ` command in the input -script. +individual pair_style doc pages and the :doc:`pair_modify mix +` command for details. Pair coefficients can also be set +via the :doc:`pair_coeff ` command in the input script. ---------- diff --git a/doc/src/set.rst b/doc/src/set.rst index 84792d1a58..b986e4b26f 100644 --- a/doc/src/set.rst +++ b/doc/src/set.rst @@ -13,7 +13,7 @@ Syntax * style = *atom* or *type* or *mol* or *group* or *region* * ID = atom ID range or type range or mol ID range or group ID or region ID * one or more keyword/value pairs may be appended -* keyword = *type* or *type/fraction* or *type/ratio* or *type/subset* or *mol* or *x* or *y* or *z* or *charge* or *dipole* or *dipole/random* or *quat* or *spin* or *spin/random* or *quat* or *quat/random* or *diameter* or *shape* or *length* or *tri* or *theta* or *theta/random* or *angmom* or *omega* or *mass* or *density* or *density/disc* or *volume* or *image* or *bond* or *angle* or *dihedral* or *improper* or *meso/e* or *meso/cv* or *meso/rho* or *smd/contact/radius* or *smd/mass/density* or *dpd/theta* or *edpd/temp* or *edpd/cv* or *cc* or *i_name* or *d_name* +* keyword = *type* or *type/fraction* or *type/ratio* or *type/subset* or *mol* or *x* or *y* or *z* or *charge* or *dipole* or *dipole/random* or *quat* or *spin* or *spin/random* or *quat* or *quat/random* or *diameter* or *shape* or *length* or *tri* or *theta* or *theta/random* or *angmom* or *omega* or *mass* or *density* or *density/disc* or *volume* or *image* or *bond* or *angle* or *dihedral* or *improper* or *sph/e* or *sph/cv* or *sph/rho* or *smd/contact/radius* or *smd/mass/density* or *dpd/theta* or *edpd/temp* or *edpd/cv* or *cc* or *i_name* or *d_name* .. parsed-literal:: @@ -94,11 +94,11 @@ Syntax *angle* value = angle type for all angles between selected atoms *dihedral* value = dihedral type for all dihedrals between selected atoms *improper* value = improper type for all impropers between selected atoms - *meso/e* value = energy of SPH particles (need units) + *sph/e* value = energy of SPH particles (need units) value can be an atom-style variable (see below) - *meso/cv* value = heat capacity of SPH particles (need units) + *sph/cv* value = heat capacity of SPH particles (need units) value can be an atom-style variable (see below) - *meso/rho* value = density of SPH particles (need units) + *sph/rho* value = density of SPH particles (need units) value can be an atom-style variable (see below) *smd/contact/radius* = radius for short range interactions, i.e. contact and friction value can be an atom-style variable (see below) @@ -299,29 +299,31 @@ for each particle set by this command. This keyword does not allow use of an atom-style variable. Keyword *diameter* sets the size of the selected atoms. The particles -must be finite-size spheres as defined by the :doc:`atom_style sphere ` command. The diameter of a particle can be -set to 0.0, which means they will be treated as point particles. Note -that this command does not adjust the particle mass, even if it was -defined with a density, e.g. via the :doc:`read_data ` -command. +must be finite-size spheres as defined by the :doc:`atom_style sphere +` command. The diameter of a particle can be set to 0.0, +which means they will be treated as point particles. Note that this +command does not adjust the particle mass, even if it was defined with +a density, e.g. via the :doc:`read_data ` command. Keyword *shape* sets the size and shape of the selected atoms. The -particles must be ellipsoids as defined by the :doc:`atom_style ellipsoid ` command. The *Sx*\ , *Sy*\ , *Sz* settings are -the 3 diameters of the ellipsoid in each direction. All 3 can be set -to the same value, which means the ellipsoid is effectively a sphere. -They can also all be set to 0.0 which means the particle will be -treated as a point particle. Note that this command does not adjust -the particle mass, even if it was defined with a density, e.g. via the -:doc:`read_data ` command. +particles must be ellipsoids as defined by the :doc:`atom_style +ellipsoid ` command. The *Sx*\ , *Sy*\ , *Sz* settings +are the 3 diameters of the ellipsoid in each direction. All 3 can be +set to the same value, which means the ellipsoid is effectively a +sphere. They can also all be set to 0.0 which means the particle will +be treated as a point particle. Note that this command does not +adjust the particle mass, even if it was defined with a density, +e.g. via the :doc:`read_data ` command. Keyword *length* sets the length of selected atoms. The particles -must be line segments as defined by the :doc:`atom_style line ` command. If the specified value is non-zero the -line segment is (re)set to a length = the specified value, centered -around the particle position, with an orientation along the x-axis. -If the specified value is 0.0, the particle will become a point -particle. Note that this command does not adjust the particle mass, -even if it was defined with a density, e.g. via the -:doc:`read_data ` command. +must be line segments as defined by the :doc:`atom_style line +` command. If the specified value is non-zero the line +segment is (re)set to a length = the specified value, centered around +the particle position, with an orientation along the x-axis. If the +specified value is 0.0, the particle will become a point particle. +Note that this command does not adjust the particle mass, even if it +was defined with a density, e.g. via the :doc:`read_data ` +command. Keyword *tri* sets the size of selected atoms. The particles must be triangles as defined by the :doc:`atom_style tri ` command. @@ -335,7 +337,8 @@ does not adjust the particle mass, even if it was defined with a density, e.g. via the :doc:`read_data ` command. Keyword *theta* sets the orientation of selected atoms. The particles -must be line segments as defined by the :doc:`atom_style line ` command. The specified value is used to set the +must be line segments as defined by the :doc:`atom_style line +` command. The specified value is used to set the orientation angle of the line segments with respect to the x axis. Keyword *theta/random* randomizes the orientation of theta for the @@ -346,44 +349,47 @@ regardless of how many processors are being used. This keyword does not allow use of an atom-style variable. Keyword *angmom* sets the angular momentum of selected atoms. The -particles must be ellipsoids as defined by the :doc:`atom_style ellipsoid ` command or triangles as defined by the -:doc:`atom_style tri ` command. The angular momentum vector -of the particles is set to the 3 specified components. +particles must be ellipsoids as defined by the :doc:`atom_style +ellipsoid ` command or triangles as defined by the +:doc:`atom_style tri ` command. The angular momentum +vector of the particles is set to the 3 specified components. Keyword *omega* sets the angular velocity of selected atoms. The -particles must be spheres as defined by the -:doc:`atom_style sphere ` command. The angular velocity -vector of the particles is set to the 3 specified components. +particles must be spheres as defined by the :doc:`atom_style sphere +` command. The angular velocity vector of the particles +is set to the 3 specified components. Keyword *mass* sets the mass of all selected particles. The particles must have a per-atom mass attribute, as defined by the -:doc:`atom_style ` command. See the "mass" command for how -to set mass values on a per-type basis. +:doc:`atom_style ` command. See the "mass" command for +how to set mass values on a per-type basis. Keyword *density* or *density/disc* also sets the mass of all selected particles, but in a different way. The particles must have a per-atom mass attribute, as defined by the :doc:`atom_style ` -command. If the atom has a radius attribute (see :doc:`atom_style sphere `) and its radius is non-zero, its mass is set -from the density and particle volume for 3d systems (the input density -is assumed to be in mass/distance\^3 units). For 2d, the default is -for LAMMPS to model particles with a radius attribute as spheres. +command. If the atom has a radius attribute (see :doc:`atom_style +sphere `) and its radius is non-zero, its mass is set from +the density and particle volume for 3d systems (the input density is +assumed to be in mass/distance\^3 units). For 2d, the default is for +LAMMPS to model particles with a radius attribute as spheres. However, if the *density/disc* keyword is used, then they can be modeled as 2d discs (circles). Their mass is set from the density and particle area (the input density is assumed to be in mass/distance\^2 units). -If the atom has a shape attribute (see :doc:`atom_style ellipsoid `) and its 3 shape parameters are non-zero, -then its mass is set from the density and particle volume (the input -density is assumed to be in mass/distance\^3 units). The -*density/disc* keyword has no effect; it does not (yet) treat 3d -ellipsoids as 2d ellipses. +If the atom has a shape attribute (see :doc:`atom_style ellipsoid +`) and its 3 shape parameters are non-zero, then its mass +is set from the density and particle volume (the input density is +assumed to be in mass/distance\^3 units). The *density/disc* keyword +has no effect; it does not (yet) treat 3d ellipsoids as 2d ellipses. -If the atom has a length attribute (see :doc:`atom_style line `) and its length is non-zero, then its mass is -set from the density and line segment length (the input density is -assumed to be in mass/distance units). If the atom has an area -attribute (see :doc:`atom_style tri `) and its area is -non-zero, then its mass is set from the density and triangle area (the -input density is assumed to be in mass/distance\^2 units). +If the atom has a length attribute (see :doc:`atom_style line +`) and its length is non-zero, then its mass is set from +the density and line segment length (the input density is assumed to +be in mass/distance units). If the atom has an area attribute (see +:doc:`atom_style tri `) and its area is non-zero, then its +mass is set from the density and triangle area (the input density is +assumed to be in mass/distance\^2 units). If none of these cases are valid, then the mass is set to the density value directly (the input density is assumed to be in mass units). @@ -399,14 +405,15 @@ defined. A value of 2 means add 2 box lengths to get the true value. A value of -1 means subtract 1 box length to get the true value. LAMMPS updates these flags as atoms cross periodic boundaries during the simulation. The flags can be output with atom snapshots via the -:doc:`dump ` command. If a value of NULL is specified for any of -nx,ny,nz, then the current image value for that dimension is unchanged. -For non-periodic dimensions only a value of 0 can be specified. -This command can be useful after a system has been equilibrated and -atoms have diffused one or more box lengths in various directions. -This command can then reset the image values for atoms so that they -are effectively inside the simulation box, e.g if a diffusion -coefficient is about to be measured via the :doc:`compute msd ` command. Care should be taken not to reset the +:doc:`dump ` command. If a value of NULL is specified for any +of nx,ny,nz, then the current image value for that dimension is +unchanged. For non-periodic dimensions only a value of 0 can be +specified. This command can be useful after a system has been +equilibrated and atoms have diffused one or more box lengths in +various directions. This command can then reset the image values for +atoms so that they are effectively inside the simulation box, e.g if a +diffusion coefficient is about to be measured via the :doc:`compute +msd ` command. Care should be taken not to reset the image flags of two atoms in a bond to the same value if the bond straddles a periodic boundary (rather they should be different by +/- 1). This will not affect the dynamics of a simulation, but may mess @@ -423,10 +430,10 @@ etc) was set by the *bond types* (\ *angle types*\ , etc) field in the header of the data file read by the :doc:`read_data ` command. These keywords do not allow use of an atom-style variable. -Keywords *meso/e*\ , *meso/cv*\ , and *meso/rho* set the energy, heat +Keywords *sph/e*\ , *sph/cv*\ , and *sph/rho* set the energy, heat capacity, and density of smoothed particle hydrodynamics (SPH) -particles. See `this PDF guide `_ to -using SPH in LAMMPS. +particles. See `this PDF guide `_ +to using SPH in LAMMPS. Keyword *smd/mass/density* sets the mass of all selected particles, but it is only applicable to the Smooth Mach Dynamics package diff --git a/doc/src/tad.rst b/doc/src/tad.rst index 7f4845dc3c..c3cd3b825b 100644 --- a/doc/src/tad.rst +++ b/doc/src/tad.rst @@ -58,18 +58,21 @@ Run a temperature accelerated dynamics (TAD) simulation. This method requires two or more partitions to perform NEB transition state searches. -TAD is described in :ref:`this paper ` by Art Voter. It is a method -that uses accelerated dynamics at an elevated temperature to generate -results at a specified lower temperature. A good overview of -accelerated dynamics methods for such systems is given in :ref:`this review paper ` from the same group. In general, these methods assume -that the long-time dynamics is dominated by infrequent events i.e. the -system is confined to low energy basins for long periods, -punctuated by brief, randomly-occurring transitions to adjacent -basins. TAD is suitable for infrequent-event systems, where in +TAD is described in :ref:`this paper ` by Art Voter. It is +a method that uses accelerated dynamics at an elevated temperature to +generate results at a specified lower temperature. A good overview of +accelerated dynamics methods (AMD) for such systems is given in +:ref:`this review paper ` from the same group. To quote +from the review paper: "The dynamical evolution is characterized by +vibrational excursions within a potential basin, punctuated by +occasional transitions between basins. The transition probability is +characterized by p(t) = k\*exp(-kt) where k is the rate constant." + +TAD is a suitable AMD method for infrequent-event systems, where in addition, the transition kinetics are well-approximated by harmonic -transition state theory (hTST). In hTST, the temperature dependence of -transition rates follows the Arrhenius relation. As a consequence a -set of event times generated in a high-temperature simulation can be +transition state theory (hTST). In hTST, the temperature dependence +of transition rates follows the Arrhenius relation. As a consequence +a set of event times generated in a high-temperature simulation can be mapped to a set of much longer estimated times in the low-temperature system. However, because this mapping involves the energy barrier of the transition event, which is different for each event, the first @@ -79,8 +82,8 @@ events from the current basin. After each event, the simulation is reflected backwards into the current basin. This is repeated until the stopping criterion is satisfied, at which point the event with the earliest low-temperature occurrence time is selected. The stopping -criterion is that the confidence measure be greater than -1-\ *delta*\ . The confidence measure is the probability that no earlier +criterion is that the confidence measure be greater than 1-\ *delta*\ +. The confidence measure is the probability that no earlier low-temperature event will occur at some later time in the high-temperature simulation. hTST provides an lower bound for this probability, based on the user-specified minimum pre-exponential diff --git a/doc/utils/check-packages.py b/doc/utils/check-packages.py index 99968f9a22..52d527fd95 100755 --- a/doc/utils/check-packages.py +++ b/doc/utils/check-packages.py @@ -45,7 +45,7 @@ usrpkg = [] # folder, and is not called 'MAKE' is a package for d in pkgdirs: - pkg = dirs.match(d)[1] + pkg = dirs.match(d).group(1) if not os.path.isdir(os.path.join(src,pkg)): continue if pkg in ['DEPEND','MAKE','STUBS']: continue if user.match(pkg): diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 883d4bf44a..29eca85842 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -171,6 +171,7 @@ ba Babadi Babaei backcolor +backends Baczewski Bagi Bagnold @@ -265,6 +266,7 @@ Boltzman BondAngle BondBond bondchk +BondingIDs bondmax bondtype Bonet @@ -328,6 +330,7 @@ ccache ccachepiecewise ccl ccmake +ccN ccNspecies CCu cd @@ -371,6 +374,8 @@ Chemnitz Cheng Chenoweth chiral +ChiralIDs +chiralIDs chirality Cho chris @@ -437,6 +442,7 @@ cond conda Conda Condens +Connor conf config configfile @@ -491,12 +497,14 @@ cstdlib cstring cstyle csvr +ctrl Ctypes ctypes cuda Cuda CUDA CuH +cuFFT Cummins Curk customIDs @@ -568,6 +576,7 @@ defn deformable del delaystep +DeleteIDs deleteIDs Dellago delocalization @@ -582,12 +591,14 @@ dephasing dequidt Dequidt der +dereference derekt Derjagin Derjaguin Derlet Deserno Destree +destructor detils Devanathan devel @@ -718,10 +729,12 @@ Ec ecoul ecp Ecut +EdgeIDs edgeIDs edihed edim edip +edn edpd eDPD edu @@ -1056,6 +1069,7 @@ gpu gpuID gpus gradV +GradVidottan graining Graining Grama @@ -1109,6 +1123,7 @@ Harting Hartree Hartrees Hasan +Hashtable Haswell Haugk Hayoun @@ -1162,6 +1177,7 @@ Houlle howto Howto Hoyt +Hs hstyle html hTST @@ -1371,6 +1387,7 @@ Kai Kalia Kamberaj Kapfer +Karls Karlsruhe Karniadakis Karplus @@ -1393,6 +1410,7 @@ KDevelop ke KE Keblinski +Keefe keflag Keir Kelchner @@ -1415,6 +1433,7 @@ Klapp Kloss kmax Kmax +KMP Knizhnik knl Kofke @@ -1536,6 +1555,7 @@ libmeam libmessage libmpi libmpich +libnuma libplumed libplumedKernel libpng @@ -1583,6 +1603,7 @@ lmpqst lmpsdata Lmpsdata lmptype +LMT ln localTemp localvectors @@ -1598,6 +1619,7 @@ Lookups LoopVar Lorant lorenz +Los lossless lossy Lozovik @@ -1653,12 +1675,15 @@ manybody MANYBODY Maras Marchetti +Marchi Mariella Marrink Marroquin Marsaglia Marseille Martyna +mary +marys Masaglia Mashayak Massimilliano @@ -1725,6 +1750,7 @@ Melchor Meloni Melrose Mem +mem memalign MEMALIGN membered @@ -1772,6 +1798,7 @@ Militzer Minary mincap Mindlin +minhbonds mingw minima minimizations @@ -2100,7 +2127,6 @@ nvc nvcc nve nvidia -Nvidia nvk nvt Nwait @@ -2435,6 +2461,7 @@ Ravelo rc Rc Rcm +rcmax Rcmx Rcmy Rcut @@ -2553,6 +2580,7 @@ Rockett Rodrigues Rohart Ronchetti +Ronevich Rosati Rosenberger Rossky @@ -2666,6 +2694,8 @@ Shiga Shinoda Shiomi shlib +SHM +shm shockvel Shugaev si @@ -2835,6 +2865,7 @@ Synechococcus sys sysdim Syst +systemd Sz Tabbernor tabinner @@ -2958,6 +2989,7 @@ traceless tradeoff traj Tranchida +transactional transferability translational Translational @@ -3189,6 +3221,7 @@ Westview wget Whelan whitesmoke +whitespace Wi Wicaksono Wijk @@ -3257,6 +3290,7 @@ xzhou yaff YAFF Yamada +Yaser Yazdani Ybar ybox diff --git a/examples/ASPHERE/line/in.line b/examples/ASPHERE/line/in.line index cd7fed1639..815eacfa35 100644 --- a/examples/ASPHERE/line/in.line +++ b/examples/ASPHERE/line/in.line @@ -10,7 +10,7 @@ velocity all create 1.44 320984 loop geom neighbor 0.3 bin neigh_modify delay 0 every 1 check yes -neigh_modify exclude molecule all +neigh_modify exclude molecule/intra all pair_style line/lj 2.5 pair_coeff * * 1.0 1.0 1.0 0.25 2.5 @@ -33,8 +33,10 @@ compute 2 all ke compute 3 all pe variable toteng equal (c_1+c_2+c_3)/atoms +compute_modify thermo_temp extra/dof -350 + thermo 1000 -thermo_style custom step temp f_2 pe ke c_1 c_2 c_3 v_toteng +thermo_style custom step f_2 pe ke c_1 c_2 c_3 v_toteng run 10000 diff --git a/examples/ASPHERE/line/in.line.srd b/examples/ASPHERE/line/in.line.srd index 4b89b14b4b..6ec1998d31 100644 --- a/examples/ASPHERE/line/in.line.srd +++ b/examples/ASPHERE/line/in.line.srd @@ -36,7 +36,7 @@ velocity small create 1.44 87287 loop geom neighbor 0.3 bin neigh_modify delay 0 every 1 check yes -neigh_modify exclude molecule big include big +neigh_modify exclude molecule/intra big include big comm_modify mode multi group big vel yes neigh_modify include big @@ -68,13 +68,15 @@ compute tbig big temp variable pebig equal pe*atoms/count(big) variable ebig equal etotal*atoms/count(big) +compute_modify tbig extra/dof -350 + compute 1 big erotate/asphere compute 2 all ke compute 3 all pe variable toteng equal (c_1+c_2+c_3)/atoms thermo 1000 -thermo_style custom step temp c_tsmall f_2[9] c_1 etotal & +thermo_style custom step c_tsmall f_2[9] c_1 etotal & v_pebig v_ebig press thermo_modify temp tbig diff --git a/examples/ASPHERE/tri/in.tri.srd b/examples/ASPHERE/tri/in.tri.srd index caff7f60dd..e7ef6ef59a 100644 --- a/examples/ASPHERE/tri/in.tri.srd +++ b/examples/ASPHERE/tri/in.tri.srd @@ -35,7 +35,7 @@ velocity small create 1.44 87287 loop geom neighbor 0.3 bin neigh_modify delay 0 every 1 check yes -neigh_modify exclude molecule big include big +neigh_modify exclude molecule/intra big include big comm_modify mode multi group big vel yes neigh_modify include big @@ -66,6 +66,8 @@ compute tbig big temp variable pebig equal pe*atoms/count(big) variable ebig equal etotal*atoms/count(big) +compute_modify tbig extra/dof -4500 + compute 1 big erotate/asphere compute 2 all ke compute 3 all pe diff --git a/examples/COUPLE/simple/CMakeLists.txt b/examples/COUPLE/simple/CMakeLists.txt new file mode 100644 index 0000000000..4112eaa4e7 --- /dev/null +++ b/examples/COUPLE/simple/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) +project(simple CXX) + +set(LAMMPS_SRC_DIRECTORY "" CACHE PATH "Path for lammps source") +if(NOT LAMMPS_SRC_DIRECTORY STREQUAL "" AND EXISTS ${LAMMPS_SRC_DIRECTORY}/cmake/CMakeLists.txt) + option(BUILD_LIB "Build LAMMPS library" ON) + add_subdirectory(${LAMMPS_SRC_DIRECTORY}/cmake lammps) +else() + find_package(LAMMPS REQUIRED) +endif() + +add_executable(simpleCC simple.cpp) +target_link_libraries(simpleCC LAMMPS::lammps) + +enable_language(C) +add_executable(simpleC simple.c) +target_link_libraries(simpleC LAMMPS::lammps) diff --git a/examples/COUPLE/simple/simple.c b/examples/COUPLE/simple/simple.c index ad603e09a6..f4623470b7 100644 --- a/examples/COUPLE/simple/simple.c +++ b/examples/COUPLE/simple/simple.c @@ -23,7 +23,7 @@ #include "stdlib.h" #include "string.h" #include "mpi.h" -#include "library.h" /* this is a LAMMPS include file */ +#include "lammps/library.h" /* this is a LAMMPS include file */ int main(int narg, char **arg) { diff --git a/examples/SPIN/bfo/in.spin.bfo b/examples/SPIN/bfo/in.spin.bfo index b97f7e2d61..47ba535ab6 100644 --- a/examples/SPIN/bfo/in.spin.bfo +++ b/examples/SPIN/bfo/in.spin.bfo @@ -27,7 +27,8 @@ pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 neighbor 0.1 bin neigh_modify every 10 check yes delay 20 -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.1 21 fix 3 all nve/spin lattice frozen @@ -43,9 +44,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] diff --git a/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 new file mode 100644 index 0000000000..c5ad4e70ac --- /dev/null +++ b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.1 @@ -0,0 +1,129 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# layer sc iron atoms (in the [001] plane) in bismuth oxide + +units metal +atom_style spin +dimension 3 +boundary p p f + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice sc 3.96 +Lattice spacing in x,y,z = 3.96 3.96 3.96 +region box block 0.0 34.0 0.0 34.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 5780 atoms + create_atoms CPU = 0.00263691 secs + +# setting mass, mag. moments, and interactions for bfo + +mass 1 1.0 +set group all spin/random 11 2.50 + 5780 settings made for spin/random + +#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 +pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 +pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 +pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 +pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.1 21 +fix 3 all nve/spin lattice frozen + +timestep 0.0002 + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 500 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.1 + ghost atom cutoff = 6.1 + binsize = 3.05, bins = 45 45 7 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair spin/magelec, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none + (3) pair spin/dmi, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none +Per MPI rank memory allocation (min/avg/max) = 8.154 | 8.154 | 8.154 Mbytes +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.010071723 -0.69133656 -0.69133656 0 -0.69133656 + 50 0.01 0.0098309931 -1.3200811 -1.3200811 0 -1.3200811 + 100 0.02 0.0095903925 -1.9446727 -1.9446727 0 -1.9446727 + 150 0.03 0.0093489139 -2.5653446 -2.5653446 0 -2.5653446 + 200 0.04 0.0091051407 -3.1824298 -3.1824298 0 -3.1824298 + 250 0.05 0.0088575394 -3.7962506 -3.7962506 0 -3.7962506 + 300 0.06 0.0086053184 -4.4070501 -4.4070501 0 -4.4070501 + 350 0.07 0.0083490959 -5.0149813 -5.0149813 0 -5.0149813 + 400 0.08 0.0080907742 -5.6201417 -5.6201417 0 -5.6201417 + 450 0.09 0.0078327124 -6.222622 -6.222622 0 -6.222622 + 500 0.1 0.0075768488 -6.8225497 -6.8225497 0 -6.8225497 +Loop time of 12.9141 on 1 procs for 500 steps with 5780 atoms + +Performance: 0.669 ns/day, 35.873 hours/ns, 38.717 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 3.1844 | 3.1844 | 3.1844 | 0.0 | 24.66 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.012542 | 0.012542 | 0.012542 | 0.0 | 0.10 +Output | 0.048014 | 0.048014 | 0.048014 | 0.0 | 0.37 +Modify | 9.6569 | 9.6569 | 9.6569 | 0.0 | 74.78 +Other | | 0.01233 | | | 0.10 + +Nlocal: 5780 ave 5780 max 5780 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1065 ave 1065 max 1065 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 92480 ave 92480 max 92480 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 92480 +Ave neighs/atom = 16 +Neighbor list builds = 0 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:12 diff --git a/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 new file mode 100644 index 0000000000..655bbaef8e --- /dev/null +++ b/examples/SPIN/bfo/log.14Apr20.spin.bfo.g++.4 @@ -0,0 +1,129 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# layer sc iron atoms (in the [001] plane) in bismuth oxide + +units metal +atom_style spin +dimension 3 +boundary p p f + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice sc 3.96 +Lattice spacing in x,y,z = 3.96 3.96 3.96 +region box block 0.0 34.0 0.0 34.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 5780 atoms + create_atoms CPU = 0.000847816 secs + +# setting mass, mag. moments, and interactions for bfo + +mass 1 1.0 +set group all spin/random 11 2.50 + 5780 settings made for spin/random + +#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 +pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 +pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 +pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 +pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 anisotropy 0.00033 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.1 21 +fix 3 all nve/spin lattice frozen + +timestep 0.0002 + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 500 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.1 + ghost atom cutoff = 6.1 + binsize = 3.05, bins = 45 45 7 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair spin/magelec, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none + (3) pair spin/dmi, perpetual, copy from (1) + attributes: full, newton on + pair build: copy + stencil: none + bin: none +Per MPI rank memory allocation (min/avg/max) = 7.744 | 7.744 | 7.744 Mbytes +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.010071723 -0.69133656 -0.69133656 0 -0.69133656 + 50 0.01 0.009830993 -1.3200811 -1.3200811 0 -1.3200811 + 100 0.02 0.0095903924 -1.9446727 -1.9446727 0 -1.9446727 + 150 0.03 0.0093489138 -2.5653446 -2.5653446 0 -2.5653446 + 200 0.04 0.0091051405 -3.1824298 -3.1824298 0 -3.1824298 + 250 0.05 0.0088575392 -3.7962506 -3.7962506 0 -3.7962506 + 300 0.06 0.0086053183 -4.4070501 -4.4070501 0 -4.4070501 + 350 0.07 0.0083490958 -5.0149813 -5.0149813 0 -5.0149813 + 400 0.08 0.008090774 -5.6201417 -5.6201417 0 -5.6201417 + 450 0.09 0.0078327123 -6.222622 -6.222622 0 -6.222622 + 500 0.1 0.0075768487 -6.8225497 -6.8225497 0 -6.8225497 +Loop time of 3.92182 on 4 procs for 500 steps with 5780 atoms + +Performance: 2.203 ns/day, 10.894 hours/ns, 127.492 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.80837 | 0.82745 | 0.8485 | 1.6 | 21.10 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.045699 | 0.067487 | 0.090503 | 6.1 | 1.72 +Output | 0.01387 | 0.0139 | 0.01394 | 0.0 | 0.35 +Modify | 3.0065 | 3.0105 | 3.0138 | 0.2 | 76.76 +Other | | 0.002516 | | | 0.06 + +Nlocal: 1445 ave 1445 max 1445 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 555 ave 555 max 555 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +FullNghs: 23120 ave 23120 max 23120 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 92480 +Ave neighs/atom = 16 +Neighbor list builds = 0 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:03 diff --git a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 b/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 deleted file mode 100644 index b14210e9d0..0000000000 --- a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.1 +++ /dev/null @@ -1,167 +0,0 @@ -LAMMPS (30 Oct 2019) -# layer sc iron atoms (in the [001] plane) in bismuth oxide - -units metal -atom_style spin -dimension 3 -boundary p p f - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice sc 3.96 -Lattice spacing in x,y,z = 3.96 3.96 3.96 -region box block 0.0 34.0 0.0 34.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 5780 atoms - create_atoms CPU = 0.00226784 secs - -# setting mass, mag. moments, and interactions for bfo - -mass 1 1.0 -set group all spin/random 11 2.50 - 5780 settings made for spin/random - -#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 -pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 -pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 -pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 -pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice frozen - -timestep 0.0002 - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 500 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.1 - ghost atom cutoff = 6.1 - binsize = 3.05, bins = 45 45 7 - 3 neighbor lists, perpetual/occasional/extra = 3 0 0 - (1) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard - (2) pair spin/magelec, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none - (3) pair spin/dmi, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none -Per MPI rank memory allocation (min/avg/max) = 8.154 | 8.154 | 8.154 Mbytes -Step Time v_magnorm PotEng KinEng v_emag Temp TotEng - 0 0 0.010071723 -0.059343109 0 -0.13132609 0 -0.059343109 - 10 0.002 0.01003044 -0.18537022 0 -0.38338861 0 -0.18537022 - 20 0.004 0.0099890716 -0.31121926 0 -0.63509581 0 -0.31121926 - 30 0.006 0.0099475919 -0.43689013 0 -0.88644739 0 -0.43689013 - 40 0.008 0.0099059782 -0.5623833 0 -1.1374442 0 -0.5623833 - 50 0.01 0.0098642085 -0.68769978 0 -1.388088 0 -0.68769978 - 60 0.012 0.0098222618 -0.81284106 0 -1.6383818 0 -0.81284106 - 70 0.014 0.0097801186 -0.93780907 0 -1.8883294 0 -0.93780907 - 80 0.016 0.0097377603 -1.0626062 0 -2.1379352 0 -1.0626062 - 90 0.018 0.0096951693 -1.187235 0 -2.3872045 0 -1.187235 - 100 0.02 0.0096523288 -1.3116986 0 -2.6361432 0 -1.3116986 - 110 0.022 0.0096092227 -1.4360002 0 -2.8847577 0 -1.4360002 - 120 0.024 0.009565836 -1.5601431 0 -3.1330547 0 -1.5601431 - 130 0.026 0.0095221542 -1.6841309 0 -3.3810411 0 -1.6841309 - 140 0.028 0.0094781635 -1.8079673 0 -3.6287241 0 -1.8079673 - 150 0.03 0.0094338509 -1.9316557 0 -3.8761109 0 -1.9316557 - 160 0.032 0.0093892044 -2.0551997 0 -4.1232085 0 -2.0551997 - 170 0.034 0.0093442126 -2.178603 0 -4.370024 0 -2.178603 - 180 0.036 0.0092988654 -2.3018687 0 -4.6165639 0 -2.3018687 - 190 0.038 0.0092531537 -2.4250002 0 -4.8628348 0 -2.4250002 - 200 0.04 0.0092070698 -2.5480003 0 -5.1088426 0 -2.5480003 - 210 0.042 0.0091606073 -2.670872 0 -5.3545929 0 -2.670872 - 220 0.044 0.0091137617 -2.7936178 0 -5.6000909 0 -2.7936178 - 230 0.046 0.0090665298 -2.9162399 0 -5.8453412 0 -2.9162399 - 240 0.048 0.0090189108 -3.0387405 0 -6.0903478 0 -3.0387405 - 250 0.05 0.0089709056 -3.1611214 0 -6.3351146 0 -3.1611214 - 260 0.052 0.0089225173 -3.2833841 0 -6.5796445 0 -3.2833841 - 270 0.054 0.0088737511 -3.4055299 0 -6.8239403 0 -3.4055299 - 280 0.056 0.0088246147 -3.52756 0 -7.0680043 0 -3.52756 - 290 0.058 0.0087751176 -3.6494754 0 -7.3118383 0 -3.6494754 - 300 0.06 0.008725272 -3.7712768 0 -7.5554438 0 -3.7712768 - 310 0.062 0.0086750916 -3.8929648 0 -7.7988222 0 -3.8929648 - 320 0.064 0.0086245927 -4.0145399 0 -8.0419744 0 -4.0145399 - 330 0.066 0.0085737928 -4.1360026 0 -8.2849013 0 -4.1360026 - 340 0.068 0.0085227116 -4.2573532 0 -8.5276035 0 -4.2573532 - 350 0.07 0.0084713698 -4.378592 0 -8.7700818 0 -4.378592 - 360 0.072 0.0084197895 -4.4997194 0 -9.0123367 0 -4.4997194 - 370 0.074 0.0083679936 -4.6207358 0 -9.2543688 0 -4.6207358 - 380 0.076 0.0083160058 -4.7416414 0 -9.496179 0 -4.7416414 - 390 0.078 0.0082638503 -4.8624367 0 -9.7377681 0 -4.8624367 - 400 0.08 0.0082115512 -4.9831222 0 -9.9791371 0 -4.9831222 - 410 0.082 0.0081591329 -5.1036986 0 -10.220287 0 -5.1036986 - 420 0.084 0.0081066195 -5.2241665 0 -10.46122 0 -5.2241665 - 430 0.086 0.0080540347 -5.3445267 0 -10.701936 0 -5.3445267 - 440 0.088 0.008001402 -5.4647802 0 -10.942439 0 -5.4647802 - 450 0.09 0.0079487439 -5.5849281 0 -11.18273 0 -5.5849281 - 460 0.092 0.0078960829 -5.7049716 0 -11.422811 0 -5.7049716 - 470 0.094 0.0078434404 -5.824912 0 -11.662686 0 -5.824912 - 480 0.096 0.0077908378 -5.9447508 0 -11.902357 0 -5.9447508 - 490 0.098 0.0077382955 -6.0644896 0 -12.141828 0 -6.0644896 - 500 0.1 0.0076858338 -6.1841301 0 -12.381101 0 -6.1841301 -Loop time of 13.543 on 1 procs for 500 steps with 5780 atoms - -Performance: 0.638 ns/day, 37.619 hours/ns, 36.919 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 3.8138 | 3.8138 | 3.8138 | 0.0 | 28.16 -Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.011875 | 0.011875 | 0.011875 | 0.0 | 0.09 -Output | 0.049726 | 0.049726 | 0.049726 | 0.0 | 0.37 -Modify | 9.655 | 9.655 | 9.655 | 0.0 | 71.29 -Other | | 0.01262 | | | 0.09 - -Nlocal: 5780 ave 5780 max 5780 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1065 ave 1065 max 1065 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 0 ave 0 max 0 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 92480 ave 92480 max 92480 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 92480 -Ave neighs/atom = 16 -Neighbor list builds = 0 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:13 diff --git a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 b/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 deleted file mode 100644 index 8c701c93c1..0000000000 --- a/examples/SPIN/bfo/log.19Nov19.spin.bfo.g++.4 +++ /dev/null @@ -1,167 +0,0 @@ -LAMMPS (30 Oct 2019) -# layer sc iron atoms (in the [001] plane) in bismuth oxide - -units metal -atom_style spin -dimension 3 -boundary p p f - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice sc 3.96 -Lattice spacing in x,y,z = 3.96 3.96 3.96 -region box block 0.0 34.0 0.0 34.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) - 2 by 2 by 1 MPI processor grid -create_atoms 1 box -Created 5780 atoms - create_atoms CPU = 0.00149798 secs - -# setting mass, mag. moments, and interactions for bfo - -mass 1 1.0 -set group all spin/random 11 2.50 - 5780 settings made for spin/random - -#pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 -pair_style hybrid/overlay spin/exchange 6.0 spin/magelec 4.5 spin/dmi 4.5 -pair_coeff * * spin/exchange exchange 6.0 -0.01575 0.0 1.965 -pair_coeff * * spin/magelec magelec 4.5 0.000109 1.0 1.0 1.0 -pair_coeff * * spin/dmi dmi 4.5 0.00005 1.0 1.0 1.0 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin anisotropy 0.0000033 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice frozen - -timestep 0.0002 - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -#thermo_style custom step time v_magnorm v_emag temp etotal -thermo_style custom step time v_magnorm pe ke v_emag temp etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_bfo.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 500 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.1 - ghost atom cutoff = 6.1 - binsize = 3.05, bins = 45 45 7 - 3 neighbor lists, perpetual/occasional/extra = 3 0 0 - (1) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard - (2) pair spin/magelec, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none - (3) pair spin/dmi, perpetual, copy from (1) - attributes: full, newton on - pair build: copy - stencil: none - bin: none -Per MPI rank memory allocation (min/avg/max) = 7.744 | 7.744 | 7.744 Mbytes -Step Time v_magnorm PotEng KinEng v_emag Temp TotEng - 0 0 0.010071723 -0.11868622 0 -0.12966919 0 -0.11868622 - 10 0.002 0.010030399 -0.37068593 0 -0.38171598 0 -0.37068593 - 20 0.004 0.0099889925 -0.6223216 0 -0.6334048 0 -0.6223216 - 30 0.006 0.0099474775 -0.87359359 0 -0.8847354 0 -0.87359359 - 40 0.008 0.0099058307 -1.1245034 0 -1.1357086 0 -1.1245034 - 50 0.01 0.0098640297 -1.3750538 0 -1.3863265 0 -1.3750538 - 60 0.012 0.0098220535 -1.6252482 0 -1.6365919 0 -1.6252482 - 70 0.014 0.0097798823 -1.8750914 0 -1.8865086 0 -1.8750914 - 80 0.016 0.0097374973 -2.1245886 0 -2.1360814 0 -2.1245886 - 90 0.018 0.0096948808 -2.3737458 0 -2.3853155 0 -2.3737458 - 100 0.02 0.0096520159 -2.6225698 0 -2.6342168 0 -2.6225698 - 110 0.022 0.0096088866 -2.8710677 0 -2.8827919 0 -2.8710677 - 120 0.024 0.0095654776 -3.1192469 0 -3.1310475 0 -3.1192469 - 130 0.026 0.0095217746 -3.367115 0 -3.3789906 0 -3.367115 - 140 0.028 0.0094777638 -3.61468 0 -3.6266285 0 -3.61468 - 150 0.03 0.0094334323 -3.8619496 0 -3.8739683 0 -3.8619496 - 160 0.032 0.0093887679 -4.1089316 0 -4.1210173 0 -4.1089316 - 170 0.034 0.0093437596 -4.3556335 0 -4.3677824 0 -4.3556335 - 180 0.036 0.0092983972 -4.6020625 0 -4.6142704 0 -4.6020625 - 190 0.038 0.0092526717 -4.8482255 0 -4.8604877 0 -4.8482255 - 200 0.04 0.0092065755 -5.0941291 0 -5.1064403 0 -5.0941291 - 210 0.042 0.0091601024 -5.3397792 0 -5.3521339 0 -5.3397792 - 220 0.044 0.0091132478 -5.5851813 0 -5.5975736 0 -5.5851813 - 230 0.046 0.0090660089 -5.8303404 0 -5.842764 0 -5.8303404 - 240 0.048 0.0090183847 -6.0752609 0 -6.0877092 0 -6.0752609 - 250 0.05 0.0089703764 -6.3199467 0 -6.3324129 0 -6.3199467 - 260 0.052 0.0089219873 -6.5644011 0 -6.5768782 0 -6.5644011 - 270 0.054 0.0088732228 -6.808627 0 -6.8211078 0 -6.808627 - 280 0.056 0.0088240906 -7.0526266 0 -7.0651038 0 -7.0526266 - 290 0.058 0.0087746006 -7.296402 0 -7.3088682 0 -7.296402 - 300 0.06 0.0087247648 -7.5399545 0 -7.5524024 0 -7.5399545 - 310 0.062 0.0086745976 -7.7832854 0 -7.7957077 0 -7.7832854 - 320 0.064 0.0086241149 -8.0263956 0 -8.038785 0 -8.0263956 - 330 0.066 0.008573335 -8.2692858 0 -8.281635 0 -8.2692858 - 340 0.068 0.0085222772 -8.5119564 0 -8.5242586 0 -8.5119564 - 350 0.07 0.0084709627 -8.7544078 0 -8.7666562 0 -8.7544078 - 360 0.072 0.0084194136 -8.9966403 0 -9.0088285 0 -8.9966403 - 370 0.074 0.008367653 -9.2386543 0 -9.2507761 0 -9.2386543 - 380 0.076 0.0083157046 -9.4804501 0 -9.4924997 0 -9.4804501 - 390 0.078 0.0082635925 -9.7220281 0 -9.7340001 0 -9.7220281 - 400 0.08 0.0082113412 -9.9633888 0 -9.9752784 0 -9.9633888 - 410 0.082 0.0081589747 -10.204533 0 -10.216336 0 -10.204533 - 420 0.084 0.0081065173 -10.445462 0 -10.457173 0 -10.445462 - 430 0.086 0.0080539925 -10.686176 0 -10.697793 0 -10.686176 - 440 0.088 0.0080014235 -10.926676 0 -10.938197 0 -10.926676 - 450 0.09 0.0079488329 -11.166966 0 -11.178387 0 -11.166966 - 460 0.092 0.0078962427 -11.407045 0 -11.418366 0 -11.407045 - 470 0.094 0.0078436743 -11.646917 0 -11.658136 0 -11.646917 - 480 0.096 0.0077911486 -11.886583 0 -11.8977 0 -11.886583 - 490 0.098 0.007738686 -12.126047 0 -12.137063 0 -12.126047 - 500 0.1 0.0076863062 -12.365311 0 -12.376226 0 -12.365311 -Loop time of 3.94852 on 4 procs for 500 steps with 5780 atoms - -Performance: 2.188 ns/day, 10.968 hours/ns, 126.630 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.97416 | 0.98668 | 1.0022 | 1.0 | 24.99 -Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.032618 | 0.04948 | 0.062614 | 5.0 | 1.25 -Output | 0.014166 | 0.014229 | 0.014374 | 0.1 | 0.36 -Modify | 2.8947 | 2.8957 | 2.8965 | 0.0 | 73.34 -Other | | 0.002385 | | | 0.06 - -Nlocal: 1445 ave 1445 max 1445 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Nghost: 555 ave 555 max 555 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -Neighs: 0 ave 0 max 0 min -Histogram: 4 0 0 0 0 0 0 0 0 0 -FullNghs: 23120 ave 23120 max 23120 min -Histogram: 4 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 92480 -Ave neighs/atom = 16 -Neighbor list builds = 0 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:03 diff --git a/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc b/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc index dd9ed890ee..f81a962e3f 100644 --- a/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc +++ b/examples/SPIN/cobalt_fcc/in.spin.cobalt_fcc @@ -45,8 +45,6 @@ compute out_pe all pe compute out_ke all ke compute out_temp all temp -thermo_style custom f_1 - variable magx equal c_out_mag[1] variable magy equal c_out_mag[2] variable magz equal c_out_mag[3] @@ -54,7 +52,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal thermo 50 # compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 new file mode 100644 index 0000000000..f233c5e9ea --- /dev/null +++ b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.1 @@ -0,0 +1,144 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.54 +Lattice spacing in x,y,z = 3.54 3.54 3.54 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00057292 secs + +# setting mass, mag. moments, and interactions for fcc cobalt + +mass 1 58.93 + +#set group all spin/random 31 1.72 +set group all spin 1.72 0.0 0.0 1.0 + 500 settings made for spin +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 +fix_modify 1 energy yes + +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magx equal c_out_mag[1] +variable magy equal c_out_mag[2] +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal +thermo 50 + +# compute outsp all property/atom spx spy spz sp fmx fmy fmz +# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.718 | 5.718 | 5.718 Mbytes +Step Time v_magx v_magy v_magnorm PotEng v_emag Temp TotEng + 0 0 0 0 1 -2285.0679 -94.095041 100.00543 -2278.6175 + 50 0.005 0 0 1 -2284.7512 -94.095025 95.095146 -2278.6175 + 100 0.01 0 0 1 -2283.8798 -94.094815 81.584879 -2278.6175 + 150 0.015 0 0 1 -2282.6701 -94.09401 62.831647 -2278.6175 + 200 0.02 0 0 1 -2281.4185 -94.092175 43.426664 -2278.6174 + 250 0.025 0 0 1 -2280.4165 -94.089084 27.892653 -2278.6174 + 300 0.03 0 0 1 -2279.8662 -94.084906 19.36049 -2278.6174 + 350 0.035 0 0 1 -2279.8238 -94.080244 18.702888 -2278.6174 + 400 0.04 0 0 1 -2280.1962 -94.076013 24.47725 -2278.6174 + 450 0.045 0 0 1 -2280.7898 -94.073181 33.678697 -2278.6175 + 500 0.05 0 0 1 -2281.3871 -94.072475 42.940446 -2278.6175 + 550 0.055 0 0 1 -2281.8215 -94.074138 49.675032 -2278.6175 + 600 0.06 0 0 1 -2282.0209 -94.077829 52.765336 -2278.6175 + 650 0.065 0 0 1 -2282.0136 -94.082703 52.652501 -2278.6175 + 700 0.07 0 0 1 -2281.8997 -94.087644 50.88749 -2278.6174 + 750 0.075 0 0 1 -2281.8013 -94.091583 49.361723 -2278.6174 + 800 0.08 0 0 1 -2281.8108 -94.093792 49.508108 -2278.6174 + 850 0.085 0 0 1 -2281.9558 -94.094064 51.756992 -2278.6174 + 900 0.09 0 0 1 -2282.1927 -94.092723 55.428834 -2278.6175 + 950 0.095 0 0 1 -2282.4277 -94.090469 59.072774 -2278.6175 + 1000 0.1 0 0 1 -2282.5578 -94.088121 61.090565 -2278.6175 +Loop time of 4.4001 on 1 procs for 1000 steps with 500 atoms + +Performance: 1.964 ns/day, 12.222 hours/ns, 227.268 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.0815 | 2.0815 | 2.0815 | 0.0 | 47.31 +Neigh | 0.011835 | 0.011835 | 0.011835 | 0.0 | 0.27 +Comm | 0.033331 | 0.033331 | 0.033331 | 0.0 | 0.76 +Output | 0.00045395 | 0.00045395 | 0.00045395 | 0.0 | 0.01 +Modify | 2.2677 | 2.2677 | 2.2677 | 0.0 | 51.54 +Other | | 0.005309 | | | 0.12 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 24088 ave 24088 max 24088 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 48176 ave 48176 max 48176 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 48176 +Ave neighs/atom = 96.352 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 new file mode 100644 index 0000000000..3cbcd94268 --- /dev/null +++ b/examples/SPIN/cobalt_fcc/log.14Apr20.spin.cobalt_fcc.g++.4 @@ -0,0 +1,144 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.54 +Lattice spacing in x,y,z = 3.54 3.54 3.54 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000688791 secs + +# setting mass, mag. moments, and interactions for fcc cobalt + +mass 1 58.93 + +#set group all spin/random 31 1.72 +set group all spin 1.72 0.0 0.0 1.0 + 500 settings made for spin +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 +fix_modify 1 energy yes + +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magx equal c_out_mag[1] +variable magy equal c_out_mag[2] +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magx v_magy v_magnorm pe v_emag temp etotal +thermo 50 + +# compute outsp all property/atom spx spy spz sp fmx fmy fmz +# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.664 | 5.664 | 5.664 Mbytes +Step Time v_magx v_magy v_magnorm PotEng v_emag Temp TotEng + 0 0 0 0 1 -2285.0679 -94.095041 100.00543 -2278.6175 + 50 0.005 0 0 1 -2284.7564 -94.094967 95.176149 -2278.6175 + 100 0.01 0 0 1 -2283.8977 -94.094606 81.863661 -2278.6175 + 150 0.015 0 0 1 -2282.7006 -94.09362 63.303924 -2278.6175 + 200 0.02 0 0 1 -2281.452 -94.091648 43.94572 -2278.6174 + 250 0.025 0 0 1 -2280.4374 -94.08853 28.216468 -2278.6174 + 300 0.03 0 0 1 -2279.859 -94.084466 19.248729 -2278.6174 + 350 0.035 0 0 1 -2279.78 -94.080048 18.024631 -2278.6174 + 400 0.04 0 0 1 -2280.1196 -94.076134 23.288369 -2278.6174 + 450 0.045 0 0 1 -2280.697 -94.073612 32.240137 -2278.6175 + 500 0.05 0 0 1 -2281.305 -94.073124 41.666374 -2278.6175 + 550 0.055 0 0 1 -2281.7792 -94.074853 49.019175 -2278.6175 + 600 0.06 0 0 1 -2282.0409 -94.078438 53.075365 -2278.6175 + 650 0.065 0 0 1 -2282.1031 -94.083056 54.040647 -2278.6175 + 700 0.07 0 0 1 -2282.0468 -94.087642 53.167271 -2278.6175 + 750 0.075 0 0 1 -2281.9765 -94.09119 52.076935 -2278.6174 + 800 0.08 0 0 1 -2281.9742 -94.093023 52.041607 -2278.6174 + 850 0.085 0 0 1 -2282.0669 -94.09296 53.478909 -2278.6174 + 900 0.09 0 0 1 -2282.2193 -94.091331 55.842481 -2278.6175 + 950 0.095 0 0 1 -2282.354 -94.088827 57.930866 -2278.6175 + 1000 0.1 0 0 1 -2282.3885 -94.086262 58.464579 -2278.6175 +Loop time of 2.75 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.142 ns/day, 7.639 hours/ns, 363.637 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.57529 | 0.60652 | 0.63664 | 3.2 | 22.06 +Neigh | 0.0026889 | 0.0029447 | 0.0030761 | 0.3 | 0.11 +Comm | 0.10943 | 0.14471 | 0.17847 | 7.5 | 5.26 +Output | 0.00045419 | 0.00050056 | 0.00060511 | 0.0 | 0.02 +Modify | 1.9865 | 1.9926 | 1.998 | 0.3 | 72.46 +Other | | 0.002738 | | | 0.10 + +Nlocal: 125 ave 132 max 116 min +Histogram: 1 0 0 0 0 1 1 0 0 1 +Nghost: 1099 ave 1108 max 1092 min +Histogram: 1 0 0 1 1 0 0 0 0 1 +Neighs: 6033 ave 6372 max 5495 min +Histogram: 1 0 0 0 0 0 0 2 0 1 +FullNghs: 12066 ave 12964 max 10977 min +Histogram: 1 0 0 0 0 1 1 0 0 1 + +Total # of neighbors = 48264 +Ave neighs/atom = 96.528 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 b/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 deleted file mode 100644 index 06774d0858..0000000000 --- a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.1 +++ /dev/null @@ -1,142 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.54 -Lattice spacing in x,y,z = 3.54 3.54 3.54 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000470161 secs - -# setting mass, mag. moments, and interactions for fcc cobalt - -mass 1 58.93 - -#set group all spin/random 31 1.72 -set group all spin 1.72 0.0 0.0 1.0 - 500 settings made for spin -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix_modify 1 energy yes - -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -thermo_style custom f_1 - -variable magx equal c_out_mag[1] -variable magy equal c_out_mag[2] -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal -thermo 50 - -# compute outsp all property/atom spx spy spz sp fmx fmy fmz -# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 5.718 | 5.718 | 5.718 Mbytes -Step Time f_1 v_magx v_magy v_magnorm v_emag Temp TotEng - 0 0 -0.099570972 0 0 1 -188.09051 100.00543 -2278.6175 - 50 0.005 -0.099570972 0 0 1 -188.09048 95.094679 -2278.6175 - 100 0.01 -0.099570972 0 0 1 -188.09007 81.578321 -2278.6177 - 150 0.015 -0.099570972 0 0 1 -188.08848 62.802727 -2278.6185 - 200 0.02 -0.099570972 0 0 1 -188.08487 43.35108 -2278.6203 - 250 0.025 -0.099570972 0 0 1 -188.07877 27.749821 -2278.6233 - 300 0.03 -0.099570972 0 0 1 -188.07054 19.149389 -2278.6274 - 350 0.035 -0.099570972 0 0 1 -188.06135 18.453387 -2278.632 - 400 0.04 -0.099570972 0 0 1 -188.053 24.249423 -2278.6362 - 450 0.045 -0.099570972 0 0 1 -188.04742 33.548008 -2278.639 - 500 0.05 -0.099570972 0 0 1 -188.04604 42.973172 -2278.6397 - 550 0.055 -0.099570972 0 0 1 -188.04935 49.902539 -2278.638 - 600 0.06 -0.099570972 0 0 1 -188.0567 53.166772 -2278.6344 - 650 0.065 -0.099570972 0 0 1 -188.06642 53.153416 -2278.6295 - 700 0.07 -0.099570972 0 0 1 -188.07628 51.377187 -2278.6246 - 750 0.075 -0.099570972 0 0 1 -188.08415 49.725449 -2278.6206 - 800 0.08 -0.099570972 0 0 1 -188.08857 49.663576 -2278.6184 - 850 0.085 -0.099570972 0 0 1 -188.0891 51.681567 -2278.6182 - 900 0.09 -0.099570972 0 0 1 -188.08636 55.166554 -2278.6195 - 950 0.095 -0.099570972 0 0 1 -188.08174 58.718232 -2278.6218 - 1000 0.1 -0.099570972 0 0 1 -188.0769 60.75567 -2278.6243 -Loop time of 4.6196 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.870 ns/day, 12.832 hours/ns, 216.469 timesteps/s -99.9% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.3116 | 2.3116 | 2.3116 | 0.0 | 50.04 -Neigh | 0.011227 | 0.011227 | 0.011227 | 0.0 | 0.24 -Comm | 0.032837 | 0.032837 | 0.032837 | 0.0 | 0.71 -Output | 0.00039411 | 0.00039411 | 0.00039411 | 0.0 | 0.01 -Modify | 2.2584 | 2.2584 | 2.2584 | 0.0 | 48.89 -Other | | 0.005152 | | | 0.11 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 24065 ave 24065 max 24065 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 48130 ave 48130 max 48130 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 48130 -Ave neighs/atom = 96.26 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 b/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 deleted file mode 100644 index 331ed60315..0000000000 --- a/examples/SPIN/cobalt_fcc/log.19Nov19.spin.cobalt_fcc.g++.4 +++ /dev/null @@ -1,142 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.54 -Lattice spacing in x,y,z = 3.54 3.54 3.54 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.7 17.7 17.7) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000808001 secs - -# setting mass, mag. moments, and interactions for fcc cobalt - -mass 1 58.93 - -#set group all spin/random 31 1.72 -set group all spin 1.72 0.0 0.0 1.0 - 500 settings made for spin -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 0.0446928 0.003496 1.4885 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix_modify 1 energy yes - -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -thermo_style custom f_1 - -variable magx equal c_out_mag[1] -variable magy equal c_out_mag[2] -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time f_1 v_magx v_magy v_magnorm v_emag temp etotal -thermo 50 - -# compute outsp all property/atom spx spy spz sp fmx fmy fmz -# dump 1 all custom 100 dump_cobalt_fcc.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 5.664 | 5.664 | 5.664 Mbytes -Step Time f_1 v_magx v_magy v_magnorm v_emag Temp TotEng - 0 0 -0.099570972 0 0 1 -188.09051 100.00543 -2372.6129 - 50 0.005 -0.099570972 0 0 1 -188.09036 95.174807 -2372.6129 - 100 0.01 -0.099570972 0 0 1 -188.08965 81.854304 -2372.6129 - 150 0.015 -0.099570972 0 0 1 -188.0877 63.270938 -2372.6129 - 200 0.02 -0.099570972 0 0 1 -188.08381 43.867262 -2372.6129 - 250 0.025 -0.099570972 0 0 1 -188.07767 28.075261 -2372.6129 - 300 0.03 -0.099570972 0 0 1 -188.06966 19.046222 -2372.6129 - 350 0.035 -0.099570972 0 0 1 -188.06096 17.79071 -2372.6129 - 400 0.04 -0.099570972 0 0 1 -188.05326 23.079994 -2372.6129 - 450 0.045 -0.099570972 0 0 1 -188.04831 32.127316 -2372.6129 - 500 0.05 -0.099570972 0 0 1 -188.04737 41.709644 -2372.6129 - 550 0.055 -0.099570972 0 0 1 -188.05082 49.246292 -2372.6129 - 600 0.06 -0.099570972 0 0 1 -188.05795 53.465535 -2372.6129 - 650 0.065 -0.099570972 0 0 1 -188.06713 54.522857 -2372.6129 - 700 0.07 -0.099570972 0 0 1 -188.07626 53.635521 -2372.6129 - 750 0.075 -0.099570972 0 0 1 -188.08332 52.419678 -2372.6129 - 800 0.08 -0.099570972 0 0 1 -188.08696 52.176558 -2372.6129 - 850 0.085 -0.099570972 0 0 1 -188.0868 53.380592 -2372.6129 - 900 0.09 -0.099570972 0 0 1 -188.08348 55.551378 -2372.6129 - 950 0.095 -0.099570972 0 0 1 -188.07838 57.540047 -2372.6129 - 1000 0.1 -0.099570972 0 0 1 -188.07314 58.088674 -2372.6129 -Loop time of 2.54753 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.392 ns/day, 7.076 hours/ns, 392.538 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.62017 | 0.6485 | 0.66275 | 2.1 | 25.46 -Neigh | 0.0027115 | 0.0029724 | 0.0030868 | 0.3 | 0.12 -Comm | 0.095047 | 0.1102 | 0.13819 | 5.0 | 4.33 -Output | 0.00039029 | 0.00042999 | 0.00049996 | 0.0 | 0.02 -Modify | 1.7801 | 1.7834 | 1.7852 | 0.1 | 70.01 -Other | | 0.002028 | | | 0.08 - -Nlocal: 125 ave 133 max 116 min -Histogram: 1 0 0 0 0 2 0 0 0 1 -Nghost: 1099 ave 1108 max 1091 min -Histogram: 1 0 0 0 2 0 0 0 0 1 -Neighs: 6032.5 ave 6417 max 5489 min -Histogram: 1 0 0 0 0 0 1 1 0 1 -FullNghs: 12065 ave 13062 max 10970 min -Histogram: 1 0 0 0 0 2 0 0 0 1 - -Total # of neighbors = 48260 -Ave neighs/atom = 96.52 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp b/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp index dd114202cb..2bfa8393f3 100644 --- a/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp +++ b/examples/SPIN/cobalt_hcp/in.spin.cobalt_hcp @@ -32,10 +32,9 @@ pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 neighbor 0.1 bin neigh_modify every 10 check yes delay 20 -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving timestep 0.0001 @@ -51,8 +50,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] diff --git a/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 new file mode 100644 index 0000000000..b26892f6e9 --- /dev/null +++ b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.1 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# hcp cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice hcp 2.5071 +Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00121403 secs + +# setting mass, mag. moments, and interactions for hcp cobalt + +mass 1 58.93 + +set group all spin/random 31 1.72 + 500 settings made for spin/random +#set group all spin 1.72 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 +#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 +fix 3 all nve/spin lattice moving + +timestep 0.0001 + + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 4 7 7 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.902 | 7.902 | 7.902 Mbytes +Step Time v_magnorm PotEng v_emag Temp Press TotEng + 0 0 0.076558814 -2197.5536 -2.5536882 100.00543 -552.75983 -2191.1032 + 50 0.005 0.079437931 -2197.3113 -2.6177795 96.18776 -337.75504 -2191.1071 + 100 0.01 0.079575823 -2196.5768 -2.7375927 84.740309 571.91195 -2191.1109 + 150 0.015 0.078526145 -2195.4996 -2.8719243 67.984081 1845.185 -2191.1146 + 200 0.02 0.077001318 -2194.3502 -3.019723 50.107518 3008.9709 -2191.1182 + 250 0.025 0.077628454 -2193.403 -3.1832392 35.364524 4018.0217 -2191.122 + 300 0.03 0.077407462 -2192.8618 -3.3557644 26.910583 4535.9542 -2191.126 + 350 0.035 0.078090775 -2192.8052 -3.5305639 25.971561 4733.0322 -2191.13 + 400 0.04 0.078594494 -2193.135 -3.6772939 31.026665 4309.2088 -2191.1338 + 450 0.045 0.079898162 -2193.6965 -3.8044501 39.672566 3591.9593 -2191.1376 + 500 0.05 0.079885039 -2194.3293 -3.9435795 49.423774 2698.4519 -2191.1414 + 550 0.055 0.077196547 -2194.8554 -4.0868278 57.523322 1934.2444 -2191.1451 + 600 0.06 0.075430904 -2195.1797 -4.2355252 62.494025 1597.2543 -2191.1488 + 650 0.065 0.076510964 -2195.2854 -4.3752086 64.080496 1656.2312 -2191.1522 + 700 0.07 0.07649426 -2195.2723 -4.5226349 63.825926 1521.7541 -2191.1555 + 750 0.075 0.076254777 -2195.2746 -4.6937954 63.804162 1505.1323 -2191.1592 + 800 0.08 0.074211447 -2195.3567 -4.8567561 65.022623 1203.5409 -2191.1627 + 850 0.085 0.072034236 -2195.5531 -5.0007443 68.003461 900.03381 -2191.1668 + 900 0.09 0.071097702 -2195.8563 -5.1391578 72.641879 548.08834 -2191.1709 + 950 0.095 0.072642434 -2196.2007 -5.2853353 77.926596 194.45928 -2191.1743 + 1000 0.1 0.07306537 -2196.46 -5.4169261 81.891897 -404.70871 -2191.1779 +Loop time of 4.78824 on 1 procs for 1000 steps with 500 atoms + +Performance: 1.804 ns/day, 13.301 hours/ns, 208.845 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.3284 | 2.3284 | 2.3284 | 0.0 | 48.63 +Neigh | 0.01273 | 0.01273 | 0.01273 | 0.0 | 0.27 +Comm | 0.042432 | 0.042432 | 0.042432 | 0.0 | 0.89 +Output | 0.0083201 | 0.0083201 | 0.0083201 | 0.0 | 0.17 +Modify | 2.3895 | 2.3895 | 2.3895 | 0.0 | 49.90 +Other | | 0.006823 | | | 0.14 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 2444 ave 2444 max 2444 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 27036 ave 27036 max 27036 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 54072 ave 54072 max 54072 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 54072 +Ave neighs/atom = 108.144 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 new file mode 100644 index 0000000000..ad8bf1807f --- /dev/null +++ b/examples/SPIN/cobalt_hcp/log.14Apr20.spin.cobalt_hcp.g++.4 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# hcp cobalt in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice hcp 2.5071 +Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000710964 secs + +# setting mass, mag. moments, and interactions for hcp cobalt + +mass 1 58.93 + +set group all spin/random 31 1.72 + 500 settings made for spin/random +#set group all spin 1.72 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co +pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 +#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 +fix 3 all nve/spin lattice moving + +timestep 0.0001 + + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.59954 + ghost atom cutoff = 6.59954 + binsize = 3.29977, bins = 4 7 7 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.814 | 7.814 | 7.815 Mbytes +Step Time v_magnorm PotEng v_emag Temp Press TotEng + 0 0 0.076558814 -2197.5536 -2.5536882 100.00543 -552.75983 -2191.1032 + 50 0.005 0.079452711 -2197.3115 -2.6266704 96.190558 -328.47868 -2191.1071 + 100 0.01 0.079678568 -2196.5823 -2.7759025 84.826338 585.21827 -2191.1109 + 150 0.015 0.078665787 -2195.5034 -2.9426881 68.043637 1872.5465 -2191.1146 + 200 0.02 0.076875906 -2194.3466 -3.1163511 50.052941 2986.3962 -2191.1181 + 250 0.025 0.076865073 -2193.3857 -3.2792912 35.095608 4003.4846 -2191.122 + 300 0.03 0.07675751 -2192.8611 -3.4452905 26.902302 4561.1429 -2191.1259 + 350 0.035 0.077351833 -2192.8579 -3.6193072 26.792986 4523.2641 -2191.1297 + 400 0.04 0.077672952 -2193.275 -3.7845654 33.199609 4108.3226 -2191.1336 + 450 0.045 0.077553541 -2193.9028 -3.9247064 42.874729 3267.3626 -2191.1373 + 500 0.05 0.076992612 -2194.5433 -4.0593738 52.743363 2317.5276 -2191.1413 + 550 0.055 0.074971927 -2195.0364 -4.1961092 60.332059 1620.5766 -2191.145 + 600 0.06 0.072652113 -2195.3154 -4.3458839 64.600641 1265.4418 -2191.1486 + 650 0.065 0.071405665 -2195.392 -4.5049778 65.734457 1221.4637 -2191.1521 + 700 0.07 0.072030336 -2195.337 -4.6535106 64.831697 1224.9583 -2191.1553 + 750 0.075 0.072468553 -2195.2702 -4.7829549 63.746912 1220.2392 -2191.1585 + 800 0.08 0.071546019 -2195.3271 -4.9161885 64.581676 1134.3858 -2191.1616 + 850 0.085 0.071414723 -2195.5618 -5.0652271 68.168922 864.52044 -2191.1648 + 900 0.09 0.073428472 -2195.9219 -5.219789 73.702937 556.14868 -2191.168 + 950 0.095 0.0745891 -2196.3065 -5.3782699 79.616238 -64.458151 -2191.1711 + 1000 0.1 0.074027925 -2196.6049 -5.5446493 84.189424 -640.80166 -2191.1746 +Loop time of 2.86028 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.021 ns/day, 7.945 hours/ns, 349.616 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.6266 | 0.66621 | 0.68577 | 3.0 | 23.29 +Neigh | 0.0030921 | 0.0031813 | 0.0032392 | 0.1 | 0.11 +Comm | 0.1184 | 0.14023 | 0.18062 | 6.6 | 4.90 +Output | 0.0031779 | 0.0032207 | 0.0033176 | 0.1 | 0.11 +Modify | 2.0432 | 2.0445 | 2.0467 | 0.1 | 71.48 +Other | | 0.002987 | | | 0.10 + +Nlocal: 125 ave 128 max 121 min +Histogram: 1 0 1 0 0 0 0 0 0 2 +Nghost: 1324 ave 1331 max 1318 min +Histogram: 2 0 0 0 0 0 0 0 1 1 +Neighs: 6756.5 ave 6978 max 6543 min +Histogram: 1 1 0 0 0 0 0 0 1 1 +FullNghs: 13513 ave 13915 max 13042 min +Histogram: 1 1 0 0 0 0 0 0 0 2 + +Total # of neighbors = 54052 +Ave neighs/atom = 108.104 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 b/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 deleted file mode 100644 index 781e2264a7..0000000000 --- a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.1 +++ /dev/null @@ -1,219 +0,0 @@ -LAMMPS (30 Oct 2019) -# hcp cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice hcp 2.5071 -Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00105 secs - -# setting mass, mag. moments, and interactions for hcp cobalt - -mass 1 58.93 - -set group all spin/random 31 1.72 - 500 settings made for spin/random -#set group all spin 1.72 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 -#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice moving - -timestep 0.0001 - - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 4 7 7 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.902 | 7.902 | 7.902 Mbytes -Step Time v_magnorm v_emag Temp Press TotEng - 0 0 0.076558814 -5.1073764 100.00543 -552.75983 -2189.4486 - 10 0.001 0.074494403 -6.2746901 100.01038 -1571.7966 -2190.0317 - 20 0.002 0.072366265 -7.4280779 99.885587 -2535.9845 -2190.5874 - 30 0.003 0.070127018 -8.5667999 99.611653 -3445.9872 -2191.1169 - 40 0.004 0.067755946 -9.6899272 99.164813 -4302.5715 -2191.6215 - 50 0.005 0.065261592 -10.79648 98.520535 -5107.2841 -2192.103 - 60 0.006 0.062676613 -11.885341 97.657148 -5862.7198 -2192.5638 - 70 0.007 0.060046709 -12.955115 96.558718 -6572.0571 -2193.0064 - 80 0.008 0.057417313 -14.004096 95.216748 -7238.1396 -2193.4327 - 90 0.009 0.054822275 -15.030416 93.630634 -7862.5226 -2193.8437 - 100 0.01 0.052277835 -16.032345 91.80711 -8445.2646 -2194.2391 - 110 0.011 0.049783153 -17.008652 89.759163 -8985.5937 -2194.6181 - 120 0.012 0.047326373 -17.958895 87.504922 -9483.1141 -2194.98 - 130 0.013 0.044893289 -18.883574 85.066818 -9938.8838 -2195.3247 - 140 0.014 0.042474822 -19.784052 82.471014 -10355.911 -2195.6533 - 150 0.015 0.040070404 -20.662271 79.746901 -10739.081 -2195.9677 - 160 0.016 0.037686856 -21.520294 76.926428 -11094.793 -2196.2709 - 170 0.017 0.035334961 -22.359822 74.043181 -11430.247 -2196.5659 - 180 0.018 0.033026799 -23.181822 71.131269 -11752.268 -2196.8556 - 190 0.019 0.030775544 -23.986406 68.224204 -12065.774 -2197.1412 - 200 0.02 0.028597121 -24.773013 65.353995 -12372.712 -2197.4226 - 210 0.021 0.026511775 -25.540835 62.55053 -12672.055 -2197.6975 - 220 0.022 0.02454383 -26.289327 59.841288 -12961.112 -2197.9631 - 230 0.023 0.02271918 -27.018625 57.251361 -13237.544 -2198.2165 - 240 0.024 0.021061271 -27.729714 54.80373 -13501.028 -2198.4564 - 250 0.025 0.019587072 -28.42449 52.519717 -13754.325 -2198.6833 - 260 0.026 0.018304494 -29.105398 50.419388 -14002.718 -2198.8994 - 270 0.027 0.017211977 -29.775134 48.521812 -14253.089 -2199.1079 - 280 0.028 0.016300002 -30.436204 46.845075 -14512.437 -2199.3119 - 290 0.029 0.015553519 -31.090499 45.405985 -14786.53 -2199.5143 - 300 0.03 0.014954102 -31.739026 44.219544 -15079.165 -2199.7168 - 310 0.031 0.014481189 -32.381585 43.298175 -15391.531 -2199.9198 - 320 0.032 0.014112494 -33.016984 42.650874 -15722.828 -2200.1226 - 330 0.033 0.013824206 -33.643289 42.282535 -16070.874 -2200.324 - 340 0.034 0.013591568 -34.258323 42.19365 -16433.065 -2200.5226 - 350 0.035 0.013390035 -34.860184 42.380506 -16807.186 -2200.7174 - 360 0.036 0.01319679 -35.447655 42.835832 -17191.816 -2200.9077 - 370 0.037 0.012992271 -36.020512 43.549656 -17586.676 -2201.0935 - 380 0.038 0.012761486 -36.579332 44.510078 -17991.857 -2201.2754 - 390 0.039 0.012494918 -37.125414 45.703757 -18407.738 -2201.4538 - 400 0.04 0.0121888 -37.660321 47.115967 -18834.276 -2201.6292 - 410 0.041 0.011844474 -38.185489 48.730291 -19270.674 -2201.8019 - 420 0.042 0.011466715 -38.70192 50.528119 -19715.276 -2201.9716 - 430 0.043 0.011061388 -39.21005 52.488204 -20165.66 -2202.1377 - 440 0.044 0.010633241 -39.709778 54.586528 -20618.997 -2202.2998 - 450 0.045 0.010184696 -40.200724 56.79654 -21072.538 -2202.4571 - 460 0.046 0.0097161044 -40.682449 59.089699 -21523.873 -2202.6094 - 470 0.047 0.0092271788 -41.154614 61.436133 -21970.922 -2202.7565 - 480 0.048 0.0087187266 -41.617256 63.805414 -22412.32 -2202.8989 - 490 0.049 0.0081937768 -42.070708 66.167399 -22847.061 -2203.037 - 500 0.05 0.0076576327 -42.51563 68.493235 -23274.619 -2203.172 - 510 0.051 0.0071170477 -42.952841 70.756444 -23694.559 -2203.3046 - 520 0.052 0.006579078 -43.383338 72.933996 -24106.717 -2203.4358 - 530 0.053 0.006050144 -43.807962 75.007131 -24510.338 -2203.5662 - 540 0.054 0.0055354475 -44.227552 76.961803 -24904.495 -2203.6957 - 550 0.055 0.0050386503 -44.64268 78.788647 -25287.341 -2203.8241 - 560 0.056 0.0045617699 -45.053996 80.4825 -25657.11 -2203.9504 - 570 0.057 0.0041054334 -45.461923 82.041527 -26011.443 -2204.0737 - 580 0.058 0.003669689 -45.866895 83.466142 -26348.265 -2204.1931 - 590 0.059 0.0032553824 -46.269219 84.757926 -26665.834 -2204.3077 - 600 0.06 0.0028655752 -46.669125 85.918711 -26963.24 -2204.4173 - 610 0.061 0.0025060765 -47.066641 86.95 -27240.331 -2204.5218 - 620 0.062 0.0021839971 -47.461566 87.852838 -27497.728 -2204.6218 - 630 0.063 0.0019039581 -47.853462 88.628142 -27736.503 -2204.7177 - 640 0.064 0.0016633855 -48.241747 89.277364 -27957.91 -2204.81 - 650 0.065 0.0014502904 -48.625803 89.803307 -28163.11 -2204.899 - 660 0.066 0.0012463786 -49.005026 90.210807 -28352.881 -2204.9847 - 670 0.067 0.0010345087 -49.378935 90.507107 -28527.721 -2205.0668 - 680 0.068 0.00080788134 -49.747325 90.701795 -28688.395 -2205.1453 - 690 0.069 0.000586442 -50.110227 90.80636 -28836.094 -2205.22 - 700 0.07 0.00046855102 -50.467799 90.833539 -28972.361 -2205.2911 - 710 0.071 0.00061091693 -50.82044 90.796649 -29099.44 -2205.3592 - 720 0.072 0.00094960177 -51.168606 90.709122 -29219.676 -2205.4249 - 730 0.073 0.0013742455 -51.512913 90.584346 -29335.643 -2205.4887 - 740 0.074 0.0018397629 -51.853957 90.435783 -29449.521 -2205.5511 - 750 0.075 0.0023216474 -52.192407 90.277231 -29563.316 -2205.6124 - 760 0.076 0.0028000512 -52.528883 90.123061 -29678.726 -2205.6729 - 770 0.077 0.0032569295 -52.863859 89.98824 -29797.079 -2205.7329 - 780 0.078 0.0036765431 -53.197843 89.888047 -29919.964 -2205.7925 - 790 0.079 0.0040467094 -53.530921 89.837568 -30048.271 -2205.8521 - 800 0.08 0.0043597837 -53.862938 89.850978 -30182.622 -2205.9119 - 810 0.081 0.0046129296 -54.193489 89.940884 -30323.293 -2205.9718 - 820 0.082 0.0048076151 -54.522077 90.117797 -30470.468 -2206.0321 - 830 0.083 0.004948533 -54.84813 90.389814 -30624.056 -2206.0926 - 840 0.084 0.0050423324 -55.171024 90.762454 -30783.658 -2206.1532 - 850 0.085 0.0050965581 -55.490357 91.238681 -30949.141 -2206.2139 - 860 0.086 0.0051190641 -55.805904 91.818973 -31120.5 -2206.2745 - 870 0.087 0.0051180301 -56.117429 92.501449 -31297.412 -2206.3349 - 880 0.088 0.0051024116 -56.424751 93.281992 -31479.436 -2206.3949 - 890 0.089 0.005082454 -56.727832 94.154367 -31666.293 -2206.4544 - 900 0.09 0.0050697645 -57.026442 95.110386 -31857.043 -2206.513 - 910 0.091 0.0050765431 -57.320291 96.140056 -32050.436 -2206.5703 - 920 0.092 0.0051139309 -57.609075 97.231838 -32245.079 -2206.6257 - 930 0.093 0.0051899535 -57.89236 98.372982 -32439.141 -2206.6788 - 940 0.094 0.0053078572 -58.169742 99.54995 -32630.727 -2206.7288 - 950 0.095 0.0054654923 -58.44083 100.74893 -32817.882 -2206.7752 - 960 0.096 0.0056558757 -58.705483 101.95638 -32999.116 -2206.8176 - 970 0.097 0.0058685513 -58.963698 103.15953 -33173.159 -2206.8557 - 980 0.098 0.0060912487 -59.215624 104.34681 -33338.961 -2206.8893 - 990 0.099 0.0063114886 -59.461806 105.50819 -33496.345 -2206.9188 - 1000 0.1 0.0065179843 -59.702883 106.63524 -33645.259 -2206.9444 -Loop time of 5.20295 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.661 ns/day, 14.453 hours/ns, 192.199 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.6241 | 2.6241 | 2.6241 | 0.0 | 50.43 -Neigh | 0.01424 | 0.01424 | 0.01424 | 0.0 | 0.27 -Comm | 0.041207 | 0.041207 | 0.041207 | 0.0 | 0.79 -Output | 0.0090086 | 0.0090086 | 0.0090086 | 0.0 | 0.17 -Modify | 2.5084 | 2.5084 | 2.5084 | 0.0 | 48.21 -Other | | 0.006008 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 2442 ave 2442 max 2442 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 27581 ave 27581 max 27581 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 55162 ave 55162 max 55162 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 55162 -Ave neighs/atom = 110.324 -Neighbor list builds = 7 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:05 diff --git a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 b/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 deleted file mode 100644 index de00d5be32..0000000000 --- a/examples/SPIN/cobalt_hcp/log.19Nov19.spin.cobalt_hcp.g++.4 +++ /dev/null @@ -1,219 +0,0 @@ -LAMMPS (30 Oct 2019) -# hcp cobalt in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice hcp 2.5071 -Lattice spacing in x,y,z = 2.5071 4.34242 4.09408 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00101995 secs - -# setting mass, mag. moments, and interactions for hcp cobalt - -mass 1 58.93 - -set group all spin/random 31 1.72 - 500 settings made for spin/random -#set group all spin 1.72 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -#pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Co_PurjaPun_2012.eam.alloy Co -pair_coeff * * spin/exchange exchange 4.0 -0.3593 1.135028015e-05 1.064568567 -#pair_coeff * * spin/neel neel 4.0 0.0048 0.234 1.168 2.6905 0.705 0.652 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -#fix 1 all precession/spin zeeman 1.0 0.0 0.0 1.0 -fix 1 all precession/spin anisotropy 0.01 0.0 0.0 1.0 -#fix 2 all langevin/spin 0.0 0.0 21 -fix 2 all langevin/spin 0.0 0.1 21 -fix 3 all nve/spin lattice moving - -timestep 0.0001 - - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp press etotal -thermo 10 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_cobalt_hcp.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 6.59954 - ghost atom cutoff = 6.59954 - binsize = 3.29977, bins = 4 7 7 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.814 | 7.814 | 7.815 Mbytes -Step Time v_magnorm v_emag Temp Press TotEng - 0 0 0.076558814 -5.1073764 100.00543 -552.75983 -2190.3478 - 10 0.001 0.074494512 -6.2728301 99.980769 -1570.0726 -2191.5261 - 20 0.002 0.072367013 -7.4259977 99.847801 -2531.5119 -2192.6655 - 30 0.003 0.070129365 -8.566306 99.586282 -3438.1309 -2193.7672 - 40 0.004 0.067761178 -9.6929189 99.171132 -4291.017 -2194.8323 - 50 0.005 0.065270916 -10.8048 98.575397 -5091.9111 -2195.8628 - 60 0.006 0.062690557 -11.900573 97.773618 -5843.4528 -2196.8612 - 70 0.007 0.060064592 -12.978381 96.745047 -6548.726 -2197.8306 - 80 0.008 0.05743694 -14.035923 95.476292 -7210.2954 -2198.773 - 90 0.009 0.054839883 -15.07074 93.963026 -7829.4252 -2199.689 - 100 0.01 0.052288504 -16.08066 92.210482 -8405.9983 -2200.5773 - 110 0.011 0.049782155 -17.064251 90.232741 -8939.3051 -2201.4357 - 120 0.012 0.047311759 -18.021135 88.051042 -9429.1353 -2202.2626 - 130 0.013 0.044869196 -18.952065 85.691573 -9876.5628 -2203.0575 - 140 0.014 0.042453961 -19.858739 83.18315 -10284.249 -2203.8215 - 150 0.015 0.040074171 -20.743348 80.555177 -10656.417 -2204.5569 - 160 0.016 0.037742459 -21.608 77.836156 -10998.818 -2205.2677 - 170 0.017 0.035470168 -22.454209 75.052994 -11318.525 -2205.9587 - 180 0.018 0.033263447 -23.282658 72.231211 -11623.118 -2206.6354 - 190 0.019 0.031122821 -24.093311 69.395936 -11919.248 -2207.3023 - 200 0.02 0.029045634 -24.88579 66.573223 -12211.306 -2207.9613 - 210 0.021 0.027029857 -25.659817 63.791041 -12500.812 -2208.6115 - 220 0.022 0.025077742 -26.415541 61.079413 -12787.018 -2209.2498 - 230 0.023 0.023198048 -27.153652 58.469604 -13068.277 -2209.8722 - 240 0.024 0.02140599 -27.875313 55.992687 -13343.621 -2210.4756 - 250 0.025 0.019720922 -28.581973 53.678031 -13613.86 -2211.0588 - 260 0.026 0.018162738 -29.275283 51.552191 -13882.15 -2211.6232 - 270 0.027 0.016748514 -29.956802 49.638467 -14153.137 -2212.1718 - 280 0.028 0.01549075 -30.628043 47.957071 -14432.246 -2212.7087 - 290 0.029 0.014397611 -31.290177 46.525552 -14724.005 -2213.2371 - 300 0.03 0.013474315 -31.943984 45.359085 -15031.315 -2213.759 - 310 0.031 0.012723957 -32.589853 44.47023 -15355.595 -2214.275 - 320 0.032 0.012146358 -33.227585 43.868153 -15696.845 -2214.7851 - 330 0.033 0.011734827 -33.856656 43.557623 -16054.887 -2215.289 - 340 0.034 0.011472508 -34.476313 43.538346 -16429.77 -2215.7871 - 350 0.035 0.011330772 -35.085716 43.805034 -16821.627 -2216.2802 - 360 0.036 0.011271169 -35.684147 44.348312 -17230.21 -2216.7687 - 370 0.037 0.01125027 -36.271215 45.156046 -17654.485 -2217.2524 - 380 0.038 0.011225354 -36.847053 46.214576 -18092.623 -2217.7301 - 390 0.039 0.011159026 -37.412284 47.509345 -18542.156 -2218.2003 - 400 0.04 0.011022073 -37.967916 49.024843 -19000.554 -2218.6614 - 410 0.041 0.01079477 -38.515123 50.744046 -19465.713 -2219.1128 - 420 0.042 0.010467095 -39.054921 52.647653 -19935.873 -2219.5544 - 430 0.043 0.010038219 -39.588034 54.713405 -20409.666 -2219.9869 - 440 0.044 0.0095155267 -40.114703 56.915658 -20885.556 -2220.4109 - 450 0.045 0.0089134996 -40.634722 59.225397 -21361.621 -2220.8268 - 460 0.046 0.0082528918 -41.147681 61.610799 -21835.762 -2221.2347 - 470 0.047 0.0075606723 -41.653088 64.038349 -22305.687 -2221.6343 - 480 0.048 0.0068707613 -42.150486 66.474377 -22768.948 -2222.0253 - 490 0.049 0.0062249854 -42.639704 68.886721 -23223.418 -2222.4076 - 500 0.05 0.0056723593 -43.120772 71.24617 -23667.077 -2222.7814 - 510 0.051 0.00526312 -43.59404 73.527392 -24098.459 -2223.147 - 520 0.052 0.0050342241 -44.059917 75.709206 -24516.163 -2223.5051 - 530 0.053 0.0049906301 -44.518898 77.774314 -24919.192 -2223.8564 - 540 0.054 0.0050976586 -44.971364 79.708763 -25306.611 -2224.2014 - 550 0.055 0.0052941974 -45.417577 81.501347 -25677.67 -2224.5405 - 560 0.056 0.0055157717 -45.857628 83.143173 -26031.673 -2224.8736 - 570 0.057 0.0057113414 -46.291426 84.627457 -26367.904 -2225.2003 - 580 0.058 0.0058493207 -46.718709 85.949497 -26685.6 -2225.52 - 590 0.059 0.0059162201 -47.139052 87.10679 -26984.124 -2225.8316 - 600 0.06 0.0059118584 -47.551892 88.099176 -27263.145 -2226.1347 - 610 0.061 0.005843747 -47.956571 88.928929 -27522.773 -2226.4287 - 620 0.062 0.0057222223 -48.352422 89.600763 -27763.549 -2226.7139 - 630 0.063 0.0055570967 -48.738876 90.12173 -27986.321 -2226.9905 - 640 0.064 0.0053558993 -49.115723 90.501081 -28192.238 -2227.2593 - 650 0.065 0.0051233209 -49.483122 90.750056 -28382.3 -2227.5205 - 660 0.066 0.0048614512 -49.841791 90.881635 -28557.623 -2227.7746 - 670 0.067 0.0045706003 -50.192974 90.910245 -28719.422 -2228.0219 - 680 0.068 0.0042506564 -50.538196 90.851397 -28868.809 -2228.2627 - 690 0.069 0.0039028575 -50.879364 90.721317 -29007.619 -2228.4973 - 700 0.07 0.0035319814 -51.218193 90.536521 -29137.623 -2228.7265 - 710 0.071 0.0031491486 -51.556251 90.313501 -29261.193 -2228.9511 - 720 0.072 0.0027758205 -51.894643 90.068503 -29380.924 -2229.1724 - 730 0.073 0.002449449 -52.233987 89.817462 -29499.606 -2229.3917 - 740 0.074 0.0022276613 -52.574465 89.57612 -29620.196 -2229.6103 - 750 0.075 0.0021767124 -52.915641 89.360246 -29744.882 -2229.829 - 760 0.076 0.0023310362 -53.256843 89.185838 -29875.573 -2230.0485 - 770 0.077 0.0026637349 -53.597197 89.069228 -30013.477 -2230.2685 - 780 0.078 0.0031129938 -53.93565 89.026943 -30158.812 -2230.4882 - 790 0.079 0.0036204667 -54.271339 89.075322 -30311.602 -2230.7066 - 800 0.08 0.0041448552 -54.603455 89.229912 -30471.244 -2230.9226 - 810 0.081 0.0046613106 -54.931421 89.504766 -30636.938 -2231.1352 - 820 0.082 0.0051580947 -55.255056 89.911726 -30808.087 -2231.3434 - 830 0.083 0.0056329652 -55.574491 90.459766 -30984.153 -2231.5469 - 840 0.084 0.0060893356 -55.890024 91.154456 -31164.372 -2231.7452 - 850 0.085 0.0065324419 -56.202052 91.997528 -31347.792 -2231.9379 - 860 0.086 0.0069661977 -56.511206 92.986622 -31533.977 -2232.1249 - 870 0.087 0.0073913051 -56.817814 94.115192 -31721.92 -2232.306 - 880 0.088 0.0078048547 -57.122061 95.372548 -31910.795 -2232.4809 - 890 0.089 0.008201165 -57.423984 96.744135 -32100.108 -2232.65 - 900 0.09 0.0085732702 -57.723377 98.212046 -32289.532 -2232.8136 - 910 0.091 0.0089144724 -58.019938 99.755667 -32479.154 -2232.9728 - 920 0.092 0.0092194916 -58.313266 101.35254 -32669.227 -2233.1285 - 930 0.093 0.0094849872 -58.602956 102.97932 -32860.091 -2233.2822 - 940 0.094 0.0097093572 -58.888668 104.61271 -33051.981 -2233.4348 - 950 0.095 0.0098920175 -59.169925 106.23045 -33244.279 -2233.5871 - 960 0.096 0.01003244 -59.44662 107.81212 -33436.562 -2233.7396 - 970 0.097 0.010129313 -59.718668 109.33976 -33627.714 -2233.8925 - 980 0.098 0.010180127 -59.986126 110.79823 -33816.218 -2234.0455 - 990 0.099 0.010181304 -60.24929 112.17528 -34000.522 -2234.1984 - 1000 0.1 0.01012881 -60.508632 113.46137 -34179.052 -2234.3508 -Loop time of 2.93788 on 4 procs for 1000 steps with 500 atoms - -Performance: 2.941 ns/day, 8.161 hours/ns, 340.381 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.72349 | 0.73783 | 0.7554 | 1.3 | 25.11 -Neigh | 0.00353 | 0.0036981 | 0.0038559 | 0.2 | 0.13 -Comm | 0.12285 | 0.14476 | 0.16041 | 3.6 | 4.93 -Output | 0.0046515 | 0.0047909 | 0.0050418 | 0.2 | 0.16 -Modify | 2.0407 | 2.0439 | 2.0482 | 0.2 | 69.57 -Other | | 0.00288 | | | 0.10 - -Nlocal: 125 ave 136 max 119 min -Histogram: 1 1 1 0 0 0 0 0 0 1 -Nghost: 1324 ave 1331 max 1310 min -Histogram: 1 0 0 0 0 0 0 0 2 1 -Neighs: 6897.25 ave 7552 max 6604 min -Histogram: 2 1 0 0 0 0 0 0 0 1 -FullNghs: 13794.5 ave 15117 max 13164 min -Histogram: 2 0 1 0 0 0 0 0 0 1 - -Total # of neighbors = 55178 -Ave neighs/atom = 110.356 -Neighbor list builds = 7 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 similarity index 75% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 index c2af23d167..db58eec01d 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.000741005 secs + create_atoms CPU = 0.00106382 secs # setting mass, mag. moments, and interactions for bcc iron @@ -88,23 +92,23 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 13.4 | 13.4 | 13.4 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 1.0737264e-35 -768.37511 -15175.868 -15131.207 - 50 0.005 -1 -2.7722752e-10 -2.1828666e-10 1 6.8846921e-09 -768.35793 -15174.244 -15131.215 - 100 0.01 -1 -2.0983066e-09 -1.7330951e-09 1 1.0038885e-08 -768.30868 -15169.656 -15131.24 -Loop time of 7.86359 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 1.0737264e-35 -384.18755 -15175.868 -15131.207 + 50 0.005 -1 -2.7725069e-10 -2.182903e-10 1 6.8851185e-09 -384.17896 -15174.244 -15131.207 + 100 0.01 -1 -2.0990209e-09 -1.7332235e-09 1 1.0040825e-08 -384.15433 -15169.655 -15131.207 +Loop time of 7.47017 on 1 procs for 100 steps with 3456 atoms -Performance: 0.110 ns/day, 218.433 hours/ns, 12.717 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.116 ns/day, 207.505 hours/ns, 13.387 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 3.6134 | 3.6134 | 3.6134 | 0.0 | 45.95 +Pair | 3.1998 | 3.1998 | 3.1998 | 0.0 | 42.83 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.014062 | 0.014062 | 0.014062 | 0.0 | 0.18 -Output | 0.006057 | 0.006057 | 0.006057 | 0.0 | 0.08 -Modify | 4.226 | 4.226 | 4.226 | 0.0 | 53.74 -Other | | 0.004064 | | | 0.05 +Comm | 0.015094 | 0.015094 | 0.015094 | 0.0 | 0.20 +Output | 0.006531 | 0.006531 | 0.006531 | 0.0 | 0.09 +Modify | 4.2443 | 4.2443 | 4.2443 | 0.0 | 56.82 +Other | | 0.004467 | | | 0.06 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 similarity index 75% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 index 1f8157bb29..a014aae3cd 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_cut.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_cut.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00090003 secs + create_atoms CPU = 0.00132084 secs # setting mass, mag. moments, and interactions for bcc iron @@ -88,23 +92,23 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 9.217 | 9.217 | 9.217 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 1.0737264e-35 -768.37511 -15560.055 -15515.394 - 50 0.005 -1 9.6204015e-11 -3.3767807e-10 1 6.6905249e-09 -768.35767 -15558.438 -15515.394 - 100 0.01 -1 7.8881609e-10 -2.7017321e-09 1 9.8111281e-09 -768.30769 -15553.868 -15515.394 -Loop time of 2.29116 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 1.0737264e-35 -384.18755 -15175.868 -15131.207 + 50 0.005 -1 9.6205501e-11 -3.3769045e-10 1 6.6909444e-09 -384.17884 -15174.259 -15131.207 + 100 0.01 -1 7.8887025e-10 -2.7021386e-09 1 9.8130686e-09 -384.15383 -15169.712 -15131.207 +Loop time of 2.27865 on 4 procs for 100 steps with 3456 atoms -Performance: 0.377 ns/day, 63.643 hours/ns, 43.646 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.379 ns/day, 63.296 hours/ns, 43.886 timesteps/s +99.5% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.92259 | 0.92963 | 0.93393 | 0.4 | 40.57 +Pair | 0.80959 | 0.8184 | 0.82996 | 0.9 | 35.92 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.02284 | 0.027597 | 0.035185 | 2.8 | 1.20 -Output | 0.0018489 | 0.0018544 | 0.0018642 | 0.0 | 0.08 -Modify | 1.3296 | 1.3303 | 1.3308 | 0.0 | 58.06 -Other | | 0.001818 | | | 0.08 +Comm | 0.035061 | 0.046857 | 0.055443 | 3.9 | 2.06 +Output | 0.0018592 | 0.0018642 | 0.0018783 | 0.0 | 0.08 +Modify | 1.4085 | 1.4095 | 1.41 | 0.1 | 61.86 +Other | | 0.002023 | | | 0.09 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 similarity index 72% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 index 9a38445af5..f6c015bb20 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00166988 secs + create_atoms CPU = 0.00187302 secs # setting mass, mag. moments, and interactions for bcc iron @@ -67,10 +71,10 @@ dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_o run 100 EwaldDipoleSpin initialization ... - using 12-bit tables for long-range coulomb (../kspace.cpp:323) + using 12-bit tables for long-range coulomb (../kspace.cpp:332) G vector (1/distance) = 0.324623 - estimated absolute RMS force accuracy = 9.55526e-84 - estimated relative force accuracy = 6.63576e-85 + estimated absolute RMS force accuracy = 1.69788e-59 + estimated relative force accuracy = 1.17911e-60 KSpace vectors: actual max1d max3d = 2084 10 4630 kxmax kymax kzmax = 10 10 10 Neighbor list info ... @@ -97,24 +101,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 30.07 | 30.07 | 30.07 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 2.5872886e-37 -767.88567 -15175.39 -15130.729 - 50 0.005 -1 4.3660916e-09 -2.1918692e-09 1 5.3480999e-10 -767.86847 -15173.766 -15130.738 - 100 0.01 -1 9.9854966e-09 -4.2823677e-09 1 2.3267629e-09 -767.81917 -15169.178 -15130.762 -Loop time of 24.9345 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 2.5872886e-37 -383.94283 -15175.635 -15130.974 + 50 0.005 -1 4.3660908e-09 -2.1918693e-09 1 5.3484784e-10 -383.93423 -15174.011 -15130.974 + 100 0.01 -1 9.9854743e-09 -4.282369e-09 1 2.3273467e-09 -383.90957 -15169.421 -15130.974 +Loop time of 24.8682 on 1 procs for 100 steps with 3456 atoms -Performance: 0.035 ns/day, 692.624 hours/ns, 4.011 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.035 ns/day, 690.783 hours/ns, 4.021 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.8022 | 4.8022 | 4.8022 | 0.0 | 19.26 -Kspace | 10.337 | 10.337 | 10.337 | 0.0 | 41.46 +Pair | 4.6097 | 4.6097 | 4.6097 | 0.0 | 18.54 +Kspace | 10.303 | 10.303 | 10.303 | 0.0 | 41.43 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.013856 | 0.013856 | 0.013856 | 0.0 | 0.06 -Output | 0.007138 | 0.007138 | 0.007138 | 0.0 | 0.03 -Modify | 9.7705 | 9.7705 | 9.7705 | 0.0 | 39.18 -Other | | 0.004077 | | | 0.02 +Comm | 0.015258 | 0.015258 | 0.015258 | 0.0 | 0.06 +Output | 0.006583 | 0.006583 | 0.006583 | 0.0 | 0.03 +Modify | 9.9285 | 9.9285 | 9.9285 | 0.0 | 39.92 +Other | | 0.004751 | | | 0.02 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 similarity index 72% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 index 306799f576..b70221f603 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_ewald.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_ewald.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00088191 secs + create_atoms CPU = 0.000773907 secs # setting mass, mag. moments, and interactions for bcc iron @@ -67,10 +71,10 @@ dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_o run 100 EwaldDipoleSpin initialization ... - using 12-bit tables for long-range coulomb (../kspace.cpp:323) + using 12-bit tables for long-range coulomb (../kspace.cpp:332) G vector (1/distance) = 0.324623 - estimated absolute RMS force accuracy = 9.29828e-84 - estimated relative force accuracy = 6.4573e-85 + estimated absolute RMS force accuracy = 2.94041e-64 + estimated relative force accuracy = 2.042e-65 KSpace vectors: actual max1d max3d = 2084 10 4630 kxmax kymax kzmax = 10 10 10 Neighbor list info ... @@ -97,24 +101,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 25.89 | 25.89 | 25.89 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 3.5107565e-37 -767.88567 -15559.577 -15514.916 - 50 0.005 -1 4.3196063e-09 -2.1966927e-09 1 5.1719577e-10 -767.86822 -15557.96 -15514.916 - 100 0.01 -1 9.7636593e-09 -4.3236953e-09 1 2.2443181e-09 -767.81819 -15553.39 -15514.916 -Loop time of 6.80139 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 3.5107565e-37 -383.94283 -15175.635 -15130.974 + 50 0.005 -1 4.3196054e-09 -2.1966927e-09 1 5.1723249e-10 -383.93411 -15174.026 -15130.974 + 100 0.01 -1 9.7636345e-09 -4.3236965e-09 1 2.2448849e-09 -383.90908 -15169.479 -15130.974 +Loop time of 7.03264 on 4 procs for 100 steps with 3456 atoms -Performance: 0.127 ns/day, 188.927 hours/ns, 14.703 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.123 ns/day, 195.351 hours/ns, 14.219 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.248 | 1.2649 | 1.2816 | 1.1 | 18.60 -Kspace | 2.523 | 2.5743 | 2.6505 | 3.0 | 37.85 +Pair | 1.1743 | 1.207 | 1.2416 | 2.2 | 17.16 +Kspace | 2.6173 | 2.6542 | 2.7273 | 2.7 | 37.74 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.029461 | 0.087268 | 0.13754 | 13.0 | 1.28 -Output | 0.0018618 | 0.001869 | 0.0018811 | 0.0 | 0.03 -Modify | 2.8692 | 2.8709 | 2.8741 | 0.1 | 42.21 -Other | | 0.002119 | | | 0.03 +Comm | 0.042837 | 0.11362 | 0.1882 | 16.9 | 1.62 +Output | 0.0018778 | 0.0018882 | 0.0019088 | 0.0 | 0.03 +Modify | 3.0484 | 3.0535 | 3.0606 | 0.3 | 43.42 +Other | | 0.002387 | | | 0.03 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 @@ -132,4 +136,4 @@ Dangerous builds = 0 Please see the log.cite file for references relevant to this simulation -Total wall time: 0:00:06 +Total wall time: 0:00:07 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 similarity index 77% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 index b0e87d786d..3b2c36b40c 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.1 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00166583 secs + create_atoms CPU = 0.00192595 secs # setting mass, mag. moments, and interactions for bcc iron @@ -99,24 +103,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 16.27 | 16.27 | 16.27 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 3.7996771e-37 -767.89759 -15175.402 -15130.741 - 50 0.005 -1 3.6585337e-09 -1.9445403e-09 1 5.1405121e-10 -767.88039 -15173.779 -15130.75 - 100 0.01 -1 7.3585728e-09 -3.8640878e-09 1 2.0194927e-09 -767.83109 -15169.191 -15130.774 -Loop time of 15.3615 on 1 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 3.7996771e-37 -383.94879 -15175.641 -15130.98 + 50 0.005 -1 3.6585337e-09 -1.9445403e-09 1 5.1408909e-10 -383.94019 -15174.017 -15130.98 + 100 0.01 -1 7.3585736e-09 -3.8640869e-09 1 2.0200831e-09 -383.91553 -15169.428 -15130.98 +Loop time of 15.1465 on 1 procs for 100 steps with 3456 atoms -Performance: 0.056 ns/day, 426.709 hours/ns, 6.510 timesteps/s -99.9% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.057 ns/day, 420.737 hours/ns, 6.602 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.8418 | 4.8418 | 4.8418 | 0.0 | 31.52 -Kspace | 0.66626 | 0.66626 | 0.66626 | 0.0 | 4.34 +Pair | 4.6008 | 4.6008 | 4.6008 | 0.0 | 30.38 +Kspace | 0.65995 | 0.65995 | 0.65995 | 0.0 | 4.36 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.014248 | 0.014248 | 0.014248 | 0.0 | 0.09 -Output | 0.0064788 | 0.0064788 | 0.0064788 | 0.0 | 0.04 -Modify | 9.8279 | 9.8279 | 9.8279 | 0.0 | 63.98 -Other | | 0.00478 | | | 0.03 +Comm | 0.01495 | 0.01495 | 0.01495 | 0.0 | 0.10 +Output | 0.0065951 | 0.0065951 | 0.0065951 | 0.0 | 0.04 +Modify | 9.8589 | 9.8589 | 9.8589 | 0.0 | 65.09 +Other | | 0.005332 | | | 0.04 Nlocal: 3456 ave 3456 max 3456 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 similarity index 76% rename from examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 rename to examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 index 4fbd0eb52b..1c3394dfbf 100644 --- a/examples/SPIN/dipole_spin/log.19Nov19.spin.iron_dipole_pppm.g++.4 +++ b/examples/SPIN/dipole_spin/log.14Apr20.spin.iron_dipole_pppm.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (34.398 34.398 34.398) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 3456 atoms - create_atoms CPU = 0.00123286 secs + create_atoms CPU = 0.0007658 secs # setting mass, mag. moments, and interactions for bcc iron @@ -99,24 +103,24 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 10.42 | 10.42 | 10.42 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 2.3173191e-37 -767.89759 -15559.59 -15514.929 - 50 0.005 -1 3.6593054e-09 -1.9379563e-09 1 4.9747018e-10 -767.88014 -15557.972 -15514.929 - 100 0.01 -1 7.3731919e-09 -3.8151563e-09 1 1.9544299e-09 -767.8301 -15553.402 -15514.929 -Loop time of 4.4084 on 4 procs for 100 steps with 3456 atoms + 0 0 -1 0 0 1 2.3173191e-37 -383.94879 -15175.641 -15130.98 + 50 0.005 -1 3.6593053e-09 -1.9379563e-09 1 4.9750695e-10 -383.94007 -15174.032 -15130.98 + 100 0.01 -1 7.3731899e-09 -3.8151552e-09 1 1.9550037e-09 -383.91504 -15169.485 -15130.98 +Loop time of 4.3717 on 4 procs for 100 steps with 3456 atoms -Performance: 0.196 ns/day, 122.455 hours/ns, 22.684 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.198 ns/day, 121.436 hours/ns, 22.874 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.2326 | 1.2513 | 1.2693 | 1.3 | 28.38 -Kspace | 0.22823 | 0.24585 | 0.26385 | 2.8 | 5.58 +Pair | 1.1624 | 1.1869 | 1.2125 | 1.8 | 27.15 +Kspace | 0.24468 | 0.26758 | 0.29157 | 3.6 | 6.12 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.025352 | 0.028409 | 0.032299 | 1.6 | 0.64 -Output | 0.001868 | 0.0018761 | 0.0018861 | 0.0 | 0.04 -Modify | 2.8753 | 2.8788 | 2.8818 | 0.1 | 65.30 -Other | | 0.002175 | | | 0.05 +Comm | 0.027149 | 0.030758 | 0.033902 | 1.7 | 0.70 +Output | 0.0030079 | 0.0030248 | 0.0030622 | 0.0 | 0.07 +Modify | 2.8782 | 2.8806 | 2.8837 | 0.1 | 65.89 +Other | | 0.002793 | | | 0.06 Nlocal: 864 ave 864 max 864 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/iron/in.spin.iron b/examples/SPIN/iron/in.spin.iron index d60e6b86f5..58c0537af7 100644 --- a/examples/SPIN/iron/in.spin.iron +++ b/examples/SPIN/iron/in.spin.iron @@ -31,6 +31,7 @@ neighbor 0.1 bin neigh_modify every 10 check yes delay 20 fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving diff --git a/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 new file mode 100644 index 0000000000..0f3257bc05 --- /dev/null +++ b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.1 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# bcc iron in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 250 atoms + create_atoms CPU = 0.00103498 secs + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 + +set group all spin/random 31 2.2 + 250 settings made for spin/random +# set group all spin 2.2 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.77337 + ghost atom cutoff = 5.77337 + binsize = 2.88668, bins = 5 5 5 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes +Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng + 0 0 0.076456975 4554.5462 100.00358 -0.42895634 3.2186929 -1070.429 394.43342 -1067.2103 + 50 0.005 0.076456974 4671.3033 96.520068 -0.43128393 3.1065733 -1070.3169 707.2166 -1067.2103 + 100 0.01 0.076456983 4793.2674 86.525198 -0.43550567 2.7848806 -1069.9952 1456.8628 -1067.2103 + 150 0.015 0.076456973 4894.1924 71.664875 -0.44035101 2.3065896 -1069.5169 2510.4447 -1067.2103 + 200 0.02 0.076456944 4859.9379 54.610496 -0.44591875 1.7576812 -1068.968 3686.7124 -1067.2103 + 250 0.025 0.076456953 4636.3427 38.560198 -0.45258831 1.2410899 -1068.4514 4757.3215 -1067.2103 + 300 0.03 0.076457027 4333.5734 26.459387 -0.45961266 0.85161592 -1068.0619 5505.1963 -1067.2103 + 350 0.035 0.076457102 4124.1453 20.205123 -0.46569388 0.65031758 -1067.8606 5768.9329 -1067.2103 + 400 0.04 0.076457116 4064.6843 20.142986 -0.47038245 0.64831763 -1067.8586 5521.4162 -1067.2103 + 450 0.045 0.076457072 4076.2576 25.084719 -0.4741092 0.80737114 -1068.0176 4890.8714 -1067.2103 + 500 0.05 0.076457001 4105.5788 32.863873 -0.47765283 1.0577493 -1068.268 4093.2603 -1067.2103 + 550 0.055 0.076456962 4109.0613 41.126745 -0.48188168 1.3236962 -1068.534 3337.5111 -1067.2103 + 600 0.06 0.076456996 3993.3891 47.996188 -0.48790867 1.5447946 -1068.7551 2767.4353 -1067.2103 + 650 0.065 0.076457077 3776.2342 52.386407 -0.49621451 1.6860972 -1068.8964 2458.9058 -1067.2103 + 700 0.07 0.076457137 3586.7963 54.031337 -0.50582777 1.7390405 -1068.9493 2397.5333 -1067.2103 + 750 0.075 0.076457135 3535.1994 53.389736 -0.51542639 1.7183901 -1068.9287 2514.1889 -1067.2103 + 800 0.08 0.076457118 3585.6546 51.428399 -0.52423597 1.6552629 -1068.8655 2732.6669 -1067.2103 + 850 0.085 0.076457118 3634.4891 49.293001 -0.53165471 1.5865335 -1068.7968 2977.6259 -1067.2103 + 900 0.09 0.076457126 3660.3333 47.999274 -0.53780479 1.5448939 -1068.7552 3176.4112 -1067.2103 + 950 0.095 0.076457129 3671.3259 48.180409 -0.54376787 1.5507239 -1068.761 3283.2242 -1067.2103 + 1000 0.1 0.076457139 3628.6038 49.917341 -0.55029097 1.6066284 -1068.8169 3283.2361 -1067.2103 +Loop time of 1.66624 on 1 procs for 1000 steps with 250 atoms + +Performance: 5.185 ns/day, 4.628 hours/ns, 600.152 timesteps/s +99.7% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.73884 | 0.73884 | 0.73884 | 0.0 | 44.34 +Neigh | 0.0045731 | 0.0045731 | 0.0045731 | 0.0 | 0.27 +Comm | 0.023681 | 0.023681 | 0.023681 | 0.0 | 1.42 +Output | 0.0042441 | 0.0042441 | 0.0042441 | 0.0 | 0.25 +Modify | 0.89131 | 0.89131 | 0.89131 | 0.0 | 53.49 +Other | | 0.003589 | | | 0.22 + +Nlocal: 250 ave 250 max 250 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1407 ave 1407 max 1407 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 7868 ave 7868 max 7868 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 15736 ave 15736 max 15736 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 15736 +Ave neighs/atom = 62.944 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 new file mode 100644 index 0000000000..1cfa74559b --- /dev/null +++ b/examples/SPIN/iron/log.14Apr20.spin.iron.g++.4 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# bcc iron in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 250 atoms + create_atoms CPU = 0.000688791 secs + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 + +set group all spin/random 31 2.2 + 250 settings made for spin/random +# set group all spin 2.2 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.77337 + ghost atom cutoff = 5.77337 + binsize = 2.88668, bins = 5 5 5 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes +Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng + 0 0 0.076456975 4554.5462 100.00358 -0.42895634 3.2186929 -1070.429 394.43342 -1067.2103 + 50 0.005 0.076456995 4714.366 96.279315 -0.42705836 3.0988245 -1070.3091 714.16563 -1067.2103 + 100 0.01 0.076457028 4844.7708 86.007787 -0.43034795 2.7682274 -1069.9785 1479.8537 -1067.2103 + 150 0.015 0.076457073 4938.5943 70.888778 -0.43554708 2.2816103 -1069.4919 2538.6386 -1067.2103 + 200 0.02 0.076457107 4910.2627 53.612031 -0.44069391 1.7255448 -1068.9358 3702.0713 -1067.2103 + 250 0.025 0.07645713 4705.3075 37.374184 -0.44525189 1.2029171 -1068.4132 4749.8323 -1067.2103 + 300 0.03 0.076457162 4418.4389 25.117093 -0.44873668 0.80841314 -1068.0187 5473.2266 -1067.2103 + 350 0.035 0.07645722 4233.0963 18.792985 -0.45135838 0.60486682 -1067.8151 5710.4039 -1067.2103 + 400 0.04 0.07645726 4204.002 18.876267 -0.4546575 0.60754729 -1067.8178 5437.712 -1067.2103 + 450 0.045 0.076457242 4221.8277 24.290539 -0.45989731 0.78180986 -1067.9921 4788.1384 -1067.2103 + 500 0.05 0.076457208 4226.337 32.849365 -0.46668115 1.0572823 -1068.2676 3961.2587 -1067.2103 + 550 0.055 0.076457211 4195.1414 42.028524 -0.47379263 1.3527207 -1068.563 3159.4718 -1067.2103 + 600 0.06 0.076457222 4043.6401 49.722783 -0.48155951 1.6003664 -1068.8106 2554.4057 -1067.2103 + 650 0.065 0.07645721 3784.8692 54.624328 -0.49066003 1.7581264 -1068.9684 2266.0816 -1067.2103 + 700 0.07 0.076457187 3576.7472 56.274292 -0.49932577 1.8112318 -1069.0215 2298.3908 -1067.2103 + 750 0.075 0.07645717 3531.6724 55.083486 -0.50591093 1.7729047 -1068.9832 2557.6667 -1067.2103 + 800 0.08 0.076457176 3593.0894 52.172747 -0.51103604 1.6792204 -1068.8895 2933.0411 -1067.2103 + 850 0.085 0.076457206 3688.4988 48.957423 -0.51573858 1.5757327 -1068.786 3313.8291 -1067.2103 + 900 0.09 0.076457231 3788.943 46.719714 -0.52043742 1.5037103 -1068.714 3600.8734 -1067.2103 + 950 0.095 0.076457251 3854.0552 46.272425 -0.52460019 1.4893139 -1068.6996 3718.2987 -1067.2103 + 1000 0.1 0.076457302 3859.0984 47.806309 -0.52719778 1.5386831 -1068.749 3641.2287 -1067.2103 +Loop time of 1.55258 on 4 procs for 1000 steps with 250 atoms + +Performance: 5.565 ns/day, 4.313 hours/ns, 644.089 timesteps/s +99.8% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.2109 | 0.21798 | 0.22568 | 1.1 | 14.04 +Neigh | 0.0011308 | 0.0011812 | 0.0012279 | 0.1 | 0.08 +Comm | 0.074407 | 0.082247 | 0.090297 | 2.0 | 5.30 +Output | 0.0019011 | 0.0019355 | 0.0020187 | 0.1 | 0.12 +Modify | 1.2468 | 1.2477 | 1.249 | 0.1 | 80.36 +Other | | 0.001532 | | | 0.10 + +Nlocal: 62.5 ave 66 max 60 min +Histogram: 1 1 0 0 0 1 0 0 0 1 +Nghost: 846.25 ave 861 max 831 min +Histogram: 1 0 1 0 0 0 0 1 0 1 +Neighs: 1962.25 ave 2091 max 1866 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 3924.5 ave 4138 max 3771 min +Histogram: 1 1 0 0 0 1 0 0 0 1 + +Total # of neighbors = 15698 +Ave neighs/atom = 62.792 +Neighbor list builds = 6 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 similarity index 65% rename from examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 rename to examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 index 22957c8498..5d07f1b71a 100644 --- a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.1 +++ b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.1 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.00101709 secs + create_atoms CPU = 0.000530005 secs # setting mass, mag. moments, and interactions for bcc iron @@ -81,53 +85,53 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 0 -55.58269 -1097.7914 -1094.5727 - 50 0.005 -1 0 0 1 0 -55.581417 -1097.6764 -1094.5733 - 100 0.01 -1 0 0 1 0 -55.577759 -1097.35 -1094.5751 - 150 0.015 -1 0 0 1 0 -55.57219 -1096.8677 -1094.5779 - 200 0.02 -1 0 0 1 0 -55.565438 -1096.3163 -1094.5813 - 250 0.025 -1 0 0 1 0 -55.558379 -1095.7987 -1094.5848 - 300 0.03 -1 0 0 1 0 -55.551886 -1095.4103 -1094.5881 - 350 0.035 -1 0 0 1 0 -55.546675 -1095.2124 -1094.5907 - 400 0.04 -1 0 0 1 0 -55.543187 -1095.2153 -1094.5924 - 450 0.045 -1 0 0 1 0 -55.54154 -1095.379 -1094.5932 - 500 0.05 -1 0 0 1 0 -55.541574 -1095.633 -1094.5932 - 550 0.055 -1 0 0 1 0 -55.542941 -1095.9006 -1094.5925 - 600 0.06 -1 0 0 1 0 -55.545209 -1096.1205 -1094.5914 - 650 0.065 -1 0 0 1 0 -55.547951 -1096.2575 -1094.59 - 700 0.07 -1 0 0 1 0 -55.550801 -1096.3044 -1094.5886 - 750 0.075 -1 0 0 1 0 -55.553483 -1096.2778 -1094.5873 - 800 0.08 -1 0 0 1 0 -55.555816 -1096.2098 -1094.5861 - 850 0.085 -1 0 0 1 0 -55.557706 -1096.1372 -1094.5852 - 900 0.09 -1 0 0 1 0 -55.55913 -1096.0919 -1094.5844 - 950 0.095 -1 0 0 1 0 -55.560111 -1096.0925 -1094.584 - 1000 0.1 -1 0 0 1 0 -55.560705 -1096.1411 -1094.5837 -Loop time of 1.74825 on 1 procs for 1000 steps with 250 atoms + 0 0 -1 0 0 1 0 -27.791345 -1097.7914 -1094.5727 + 50 0.005 -1 0 0 1 0 -27.790708 -1097.6764 -1094.5727 + 100 0.01 -1 0 0 1 0 -27.788879 -1097.3499 -1094.5727 + 150 0.015 -1 0 0 1 0 -27.78609 -1096.8672 -1094.5727 + 200 0.02 -1 0 0 1 0 -27.782705 -1096.3147 -1094.5727 + 250 0.025 -1 0 0 1 0 -27.779157 -1095.7952 -1094.5727 + 300 0.03 -1 0 0 1 0 -27.775883 -1095.4038 -1094.5727 + 350 0.035 -1 0 0 1 0 -27.773241 -1095.2023 -1094.5727 + 400 0.04 -1 0 0 1 0 -27.771451 -1095.201 -1094.5727 + 450 0.045 -1 0 0 1 0 -27.770578 -1095.3608 -1094.5727 + 500 0.05 -1 0 0 1 0 -27.770546 -1095.6113 -1094.5727 + 550 0.055 -1 0 0 1 0 -27.771185 -1095.8764 -1094.5727 + 600 0.06 -1 0 0 1 0 -27.772282 -1096.0948 -1094.5727 + 650 0.065 -1 0 0 1 0 -27.773629 -1096.2313 -1094.5727 + 700 0.07 -1 0 0 1 0 -27.775042 -1096.2787 -1094.5727 + 750 0.075 -1 0 0 1 0 -27.776384 -1096.2534 -1094.5727 + 800 0.08 -1 0 0 1 0 -27.777564 -1096.1872 -1094.5727 + 850 0.085 -1 0 0 1 0 -27.778533 -1096.117 -1094.5727 + 900 0.09 -1 0 0 1 0 -27.779276 -1096.0741 -1094.5727 + 950 0.095 -1 0 0 1 0 -27.779802 -1096.0771 -1094.5727 + 1000 0.1 -1 0 0 1 0 -27.780134 -1096.1278 -1094.5727 +Loop time of 1.70062 on 1 procs for 1000 steps with 250 atoms -Performance: 4.942 ns/day, 4.856 hours/ns, 571.999 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 5.080 ns/day, 4.724 hours/ns, 588.019 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.80384 | 0.80384 | 0.80384 | 0.0 | 45.98 -Neigh | 0.004528 | 0.004528 | 0.004528 | 0.0 | 0.26 -Comm | 0.022954 | 0.022954 | 0.022954 | 0.0 | 1.31 -Output | 0.0034568 | 0.0034568 | 0.0034568 | 0.0 | 0.20 -Modify | 0.91007 | 0.91007 | 0.91007 | 0.0 | 52.06 -Other | | 0.003404 | | | 0.19 +Pair | 0.72617 | 0.72617 | 0.72617 | 0.0 | 42.70 +Neigh | 0.0045958 | 0.0045958 | 0.0045958 | 0.0 | 0.27 +Comm | 0.023132 | 0.023132 | 0.023132 | 0.0 | 1.36 +Output | 0.0035856 | 0.0035856 | 0.0035856 | 0.0 | 0.21 +Modify | 0.93966 | 0.93966 | 0.93966 | 0.0 | 55.25 +Other | | 0.003483 | | | 0.20 Nlocal: 250 ave 250 max 250 min Histogram: 1 0 0 0 0 0 0 0 0 0 Nghost: 1415 ave 1415 max 1415 min Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 7873 ave 7873 max 7873 min +Neighs: 7872 ave 7872 max 7872 min Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 15746 ave 15746 max 15746 min +FullNghs: 15744 ave 15744 max 15744 min Histogram: 1 0 0 0 0 0 0 0 0 0 -Total # of neighbors = 15746 -Ave neighs/atom = 62.984 +Total # of neighbors = 15744 +Ave neighs/atom = 62.976 Neighbor list builds = 6 Dangerous builds = 0 # min_style spin diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 similarity index 65% rename from examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 rename to examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 index 60db1064e0..d1d0868e2f 100644 --- a/examples/SPIN/iron/log.19Nov19.spin.iron_cubic.g++.4 +++ b/examples/SPIN/iron/log.14Apr20.spin.iron_cubic.g++.4 @@ -1,7 +1,11 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal atom_style spin @@ -19,7 +23,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000651121 secs + create_atoms CPU = 0.00071311 secs # setting mass, mag. moments, and interactions for bcc iron @@ -81,53 +85,53 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes Step Time v_magx v_magy v_magz v_magnorm v_tmag v_emag PotEng TotEng - 0 0 -1 0 0 1 0 -55.58269 -1125.5827 -1122.364 - 50 0.005 -1 0 0 1 0 -55.581457 -1125.4635 -1122.364 - 100 0.01 -1 0 0 1 0 -55.577922 -1125.1262 -1122.364 - 150 0.015 -1 0 0 1 0 -55.572562 -1124.6305 -1122.364 - 200 0.02 -1 0 0 1 0 -55.566098 -1124.067 -1122.364 - 250 0.025 -1 0 0 1 0 -55.559384 -1123.5412 -1122.364 - 300 0.03 -1 0 0 1 0 -55.553261 -1123.1491 -1122.364 - 350 0.035 -1 0 0 1 0 -55.548413 -1122.9526 -1122.364 - 400 0.04 -1 0 0 1 0 -55.545248 -1122.9623 -1122.364 - 450 0.045 -1 0 0 1 0 -55.54387 -1123.1395 -1122.364 - 500 0.05 -1 0 0 1 0 -55.544101 -1123.4126 -1122.364 - 550 0.055 -1 0 0 1 0 -55.54558 -1123.7021 -1122.364 - 600 0.06 -1 0 0 1 0 -55.547857 -1123.9414 -1122.364 - 650 0.065 -1 0 0 1 0 -55.550495 -1124.0897 -1122.364 - 700 0.07 -1 0 0 1 0 -55.553127 -1124.136 -1122.364 - 750 0.075 -1 0 0 1 0 -55.555497 -1124.0961 -1122.364 - 800 0.08 -1 0 0 1 0 -55.557466 -1124.0053 -1122.364 - 850 0.085 -1 0 0 1 0 -55.559001 -1123.9069 -1122.364 - 900 0.09 -1 0 0 1 0 -55.560147 -1123.8404 -1122.364 - 950 0.095 -1 0 0 1 0 -55.560992 -1123.8312 -1122.364 - 1000 0.1 -1 0 0 1 0 -55.561635 -1123.8853 -1122.364 -Loop time of 1.5074 on 4 procs for 1000 steps with 250 atoms + 0 0 -1 0 0 1 0 -27.791345 -1097.7914 -1094.5727 + 50 0.005 -1 0 0 1 0 -27.790728 -1097.6727 -1094.5727 + 100 0.01 -1 0 0 1 0 -27.78896 -1097.3371 -1094.5727 + 150 0.015 -1 0 0 1 0 -27.786276 -1096.8437 -1094.5727 + 200 0.02 -1 0 0 1 0 -27.783035 -1096.2824 -1094.5727 + 250 0.025 -1 0 0 1 0 -27.779661 -1095.758 -1094.5727 + 300 0.03 -1 0 0 1 0 -27.776574 -1095.3661 -1094.5727 + 350 0.035 -1 0 0 1 0 -27.774114 -1095.1684 -1094.5727 + 400 0.04 -1 0 0 1 0 -27.772489 -1095.1758 -1094.5727 + 450 0.045 -1 0 0 1 0 -27.771753 -1095.3498 -1094.5727 + 500 0.05 -1 0 0 1 0 -27.771823 -1095.6196 -1094.5727 + 550 0.055 -1 0 0 1 0 -27.772521 -1095.9061 -1094.5727 + 600 0.06 -1 0 0 1 0 -27.773627 -1096.1431 -1094.5727 + 650 0.065 -1 0 0 1 0 -27.774925 -1096.2899 -1094.5727 + 700 0.07 -1 0 0 1 0 -27.776234 -1096.3356 -1094.5727 + 750 0.075 -1 0 0 1 0 -27.777423 -1096.2961 -1094.5727 + 800 0.08 -1 0 0 1 0 -27.778424 -1096.2063 -1094.5727 + 850 0.085 -1 0 0 1 0 -27.779215 -1096.1093 -1094.5727 + 900 0.09 -1 0 0 1 0 -27.779817 -1096.0443 -1094.5727 + 950 0.095 -1 0 0 1 0 -27.780271 -1096.0363 -1094.5727 + 1000 0.1 -1 0 0 1 0 -27.780622 -1096.0914 -1094.5727 +Loop time of 1.54663 on 4 procs for 1000 steps with 250 atoms -Performance: 5.732 ns/day, 4.187 hours/ns, 663.393 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 5.586 ns/day, 4.296 hours/ns, 646.566 timesteps/s +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.22156 | 0.23223 | 0.24219 | 1.5 | 15.41 -Neigh | 0.0011292 | 0.0011852 | 0.0012362 | 0.1 | 0.08 -Comm | 0.067507 | 0.076341 | 0.087237 | 2.6 | 5.06 -Output | 0.0015073 | 0.0015442 | 0.0015914 | 0.1 | 0.10 -Modify | 1.1934 | 1.1947 | 1.1955 | 0.1 | 79.25 -Other | | 0.001434 | | | 0.10 +Pair | 0.2086 | 0.21716 | 0.22473 | 1.2 | 14.04 +Neigh | 0.0011575 | 0.0011975 | 0.001235 | 0.1 | 0.08 +Comm | 0.070766 | 0.080324 | 0.088558 | 2.3 | 5.19 +Output | 0.0016837 | 0.0017157 | 0.0017838 | 0.1 | 0.11 +Modify | 1.2424 | 1.2446 | 1.2467 | 0.1 | 80.47 +Other | | 0.001639 | | | 0.11 Nlocal: 62.5 ave 66 max 60 min Histogram: 1 1 0 0 0 1 0 0 0 1 Nghost: 848.25 ave 861 max 834 min Histogram: 1 0 0 0 1 0 1 0 0 1 -Neighs: 1962.75 ave 2087 max 1870 min +Neighs: 1962.25 ave 2085 max 1870 min Histogram: 1 1 0 0 0 0 1 0 0 1 -FullNghs: 3925.5 ave 4138 max 3776 min +FullNghs: 3924.5 ave 4136 max 3776 min Histogram: 1 1 0 0 0 1 0 0 0 1 -Total # of neighbors = 15702 -Ave neighs/atom = 62.808 +Total # of neighbors = 15698 +Ave neighs/atom = 62.792 Neighbor list builds = 6 Dangerous builds = 0 # min_style spin diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 b/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 deleted file mode 100644 index cbc749a1f7..0000000000 --- a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.1 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# bcc iron in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice bcc 2.8665 -Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 250 atoms - create_atoms CPU = 0.000422955 secs - -# setting mass, mag. moments, and interactions for bcc iron - -mass 1 55.845 - -set group all spin/random 31 2.2 - 250 settings made for spin/random -# set group all spin 2.2 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 3.5 -pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe -pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.77337 - ghost atom cutoff = 5.77337 - binsize = 2.88668, bins = 5 5 5 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.82 | 7.82 | 7.82 Mbytes -Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng - 0 0 0.076456975 4554.5462 100.00358 -0.85791269 3.2186929 -1070.429 394.43342 -1067.2103 - 50 0.005 0.076456974 4658.383 96.663685 -0.86504718 3.1111957 -1070.3179 709.50826 -1067.2067 - 100 0.01 0.076456983 4744.1872 86.965803 -0.88035771 2.7990619 -1069.9981 1466.6938 -1067.1991 - 150 0.015 0.076456973 4794.5283 72.421197 -0.8996913 2.3309324 -1069.5203 2534.3867 -1067.1894 - 200 0.02 0.076456944 4707.6548 55.633188 -0.921682 1.7905973 -1068.969 3732.183 -1067.1784 - 250 0.025 0.076456953 4439.4697 39.802206 -0.94649004 1.2810649 -1068.447 4831.5559 -1067.166 - 300 0.03 0.076457027 4101.6694 27.882295 -0.97253854 0.8974133 -1068.0504 5612.0928 -1067.153 - 350 0.035 0.076457103 3860.1545 21.776538 -0.99708692 0.70089477 -1067.8416 5906.3057 -1067.1407 - 400 0.04 0.076457117 3765.5341 21.857102 -1.0190244 0.70348778 -1067.8332 5682.0053 -1067.1297 - 450 0.045 0.076457072 3739.9037 26.959407 -1.0389343 0.86770942 -1067.9875 5066.5077 -1067.1198 - 500 0.05 0.076457001 3730.8342 34.92521 -1.0582008 1.124095 -1068.2342 4279.2424 -1067.1101 - 550 0.055 0.076456962 3698.0556 43.405912 -1.0785156 1.397053 -1068.497 3533.4153 -1067.1 - 600 0.06 0.076456997 3560.947 50.544844 -1.102048 1.626825 -1068.715 2975.8479 -1067.0882 - 650 0.065 0.076457079 3341.7402 55.261218 -1.1296588 1.7786252 -1068.853 2683.3023 -1067.0744 - 700 0.07 0.076457136 3156.8448 57.25083 -1.1595102 1.8426624 -1068.9021 2640.5967 -1067.0595 - 750 0.075 0.076457132 3099.5181 56.934336 -1.1893875 1.8324758 -1068.877 2778.3261 -1067.0445 - 800 0.08 0.076457116 3132.9985 55.266343 -1.2181223 1.7787901 -1068.809 3020.1175 -1067.0302 - 850 0.085 0.076457116 3163.2943 53.376453 -1.2443326 1.7179626 -1068.735 3287.9042 -1067.0171 - 900 0.09 0.076457121 3168.063 52.279557 -1.2676425 1.6826581 -1068.6881 3504.7334 -1067.0054 - 950 0.095 0.076457122 3144.2102 52.667743 -1.2902335 1.6951522 -1068.6893 3622.1382 -1066.9941 - 1000 0.1 0.076457135 3061.0811 54.684094 -1.314147 1.76005 -1068.7422 3625.2935 -1066.9822 -Loop time of 1.6779 on 1 procs for 1000 steps with 250 atoms - -Performance: 5.149 ns/day, 4.661 hours/ns, 595.982 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.78285 | 0.78285 | 0.78285 | 0.0 | 46.66 -Neigh | 0.004487 | 0.004487 | 0.004487 | 0.0 | 0.27 -Comm | 0.022926 | 0.022926 | 0.022926 | 0.0 | 1.37 -Output | 0.003927 | 0.003927 | 0.003927 | 0.0 | 0.23 -Modify | 0.86033 | 0.86033 | 0.86033 | 0.0 | 51.27 -Other | | 0.003381 | | | 0.20 - -Nlocal: 250 ave 250 max 250 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1399 ave 1399 max 1399 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 7855 ave 7855 max 7855 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 15710 ave 15710 max 15710 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 15710 -Ave neighs/atom = 62.84 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:01 diff --git a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 b/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 deleted file mode 100644 index a5cc9d7f0b..0000000000 --- a/examples/SPIN/iron/log.19Nov19.spin.iron.g++.4 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# bcc iron in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice bcc 2.8665 -Lattice spacing in x,y,z = 2.8665 2.8665 2.8665 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 250 atoms - create_atoms CPU = 0.000705957 secs - -# setting mass, mag. moments, and interactions for bcc iron - -mass 1 55.845 - -set group all spin/random 31 2.2 - 250 settings made for spin/random -# set group all spin 2.2 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 3.5 -pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe -pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_tmag temp v_emag ke pe press etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump_iron.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.77337 - ghost atom cutoff = 5.77337 - binsize = 2.88668, bins = 5 5 5 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.766 | 7.766 | 7.766 Mbytes -Step Time v_magnorm v_tmag Temp v_emag KinEng PotEng Press TotEng - 0 0 0.076456975 4554.5462 100.00358 -0.85791269 3.2186929 -1070.8579 394.43342 -1067.6392 - 50 0.005 0.076456995 4701.2004 96.298333 -0.85659448 3.0994366 -1070.7387 714.37866 -1067.6392 - 100 0.01 0.076457028 4794.5923 86.330828 -0.87003341 2.7786247 -1070.4179 1484.2951 -1067.6392 - 150 0.015 0.076457074 4836.9634 71.603402 -0.89006992 2.3046111 -1069.9438 2551.9258 -1067.6392 - 200 0.02 0.076457106 4754.5574 54.648817 -0.91124541 1.7589146 -1069.3981 3731.1494 -1067.6392 - 250 0.025 0.076457128 4502.135 38.599515 -0.93187522 1.2423553 -1068.8816 4804.619 -1067.6392 - 300 0.03 0.076457157 4176.7186 26.383018 -0.95082226 0.8491579 -1068.4884 5563.3287 -1067.6392 - 350 0.035 0.076457207 3955.5658 20.01039 -0.96826468 0.64404992 -1068.2833 5839.6479 -1067.6392 - 400 0.04 0.076457243 3887.9746 20.097682 -0.98706373 0.64685949 -1068.2861 5601.1255 -1067.6392 - 450 0.045 0.076457231 3868.5613 25.687511 -1.0095684 0.82677249 -1068.466 4974.0031 -1067.6392 - 500 0.05 0.076457204 3838.4905 34.604697 -1.0349855 1.113779 -1068.753 4157.1837 -1067.6392 - 550 0.055 0.076457196 3775.1404 44.251809 -1.0609123 1.4242788 -1069.0635 3357.1 -1067.6392 - 600 0.06 0.076457188 3604.8828 52.475202 -1.0880854 1.6889551 -1069.3282 2752.0424 -1067.6392 - 650 0.065 0.07645718 3345.5894 57.926479 -1.1179657 1.8644087 -1069.5036 2467.7403 -1067.6392 - 700 0.07 0.076457185 3138.2001 60.030548 -1.1469999 1.9321298 -1069.5714 2510.1752 -1067.6392 - 750 0.075 0.07645719 3074.9626 59.122504 -1.1721939 1.9029037 -1069.5421 2788.7489 -1067.6392 - 800 0.08 0.076457195 3103.5294 56.349146 -1.1949365 1.813641 -1069.4529 3192.5158 -1067.6392 - 850 0.085 0.076457199 3164.2317 53.154464 -1.2164642 1.7108177 -1069.35 3602.931 -1067.6392 - 900 0.09 0.076457199 3228.1358 50.837416 -1.2366018 1.6362417 -1069.2755 3917.0758 -1067.6392 - 950 0.095 0.076457222 3247.5532 50.234549 -1.2539657 1.6168379 -1069.2561 4059.9275 -1067.6392 - 1000 0.1 0.076457266 3208.3875 51.592727 -1.2671834 1.6605519 -1069.2998 4001.4995 -1067.6392 -Loop time of 1.47769 on 4 procs for 1000 steps with 250 atoms - -Performance: 5.847 ns/day, 4.105 hours/ns, 676.731 timesteps/s -100.0% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.21791 | 0.22724 | 0.23568 | 1.4 | 15.38 -Neigh | 0.001137 | 0.0011771 | 0.0012221 | 0.1 | 0.08 -Comm | 0.066727 | 0.074288 | 0.083826 | 2.3 | 5.03 -Output | 0.0017431 | 0.0017657 | 0.0018256 | 0.1 | 0.12 -Modify | 1.1707 | 1.1718 | 1.1725 | 0.1 | 79.30 -Other | | 0.001427 | | | 0.10 - -Nlocal: 62.5 ave 66 max 60 min -Histogram: 1 0 0 2 0 0 0 0 0 1 -Nghost: 844 ave 857 max 829 min -Histogram: 1 0 0 1 0 0 0 0 1 1 -Neighs: 1962.5 ave 2096 max 1855 min -Histogram: 1 0 1 0 0 1 0 0 0 1 -FullNghs: 3925 ave 4139 max 3766 min -Histogram: 1 0 0 2 0 0 0 0 0 1 - -Total # of neighbors = 15700 -Ave neighs/atom = 62.8 -Neighbor list builds = 6 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:01 diff --git a/examples/SPIN/nickel/in.spin.nickel b/examples/SPIN/nickel/in.spin.nickel index 1d62188d8f..0fd2e5f345 100644 --- a/examples/SPIN/nickel/in.spin.nickel +++ b/examples/SPIN/nickel/in.spin.nickel @@ -31,6 +31,7 @@ neighbor 0.1 bin neigh_modify every 10 check yes delay 20 fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes fix 2 all langevin/spin 0.0 0.0 21 fix 3 all nve/spin lattice moving @@ -48,7 +49,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/nickel/in.spin.nickel_cubic b/examples/SPIN/nickel/in.spin.nickel_cubic index 1ae069a64f..88c477132e 100644 --- a/examples/SPIN/nickel/in.spin.nickel_cubic +++ b/examples/SPIN/nickel/in.spin.nickel_cubic @@ -50,7 +50,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal thermo 50 compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 new file mode 100644 index 0000000000..e4bcd538c8 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.1 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.00111699 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7591 0.20498788 100.03408 -4775.0671 -2218.3068 + 50 0.005 0.028733807 -2224.6271 0.093798492 97.987469 -10540.61 -2218.3068 + 100 0.01 0.028733815 -2224.1217 -0.12620081 90.153107 8107.9288 -2218.3068 + 150 0.015 0.028733823 -2223.2984 -0.433762 77.388213 2482.6323 -2218.3068 + 200 0.02 0.028733827 -2222.3037 -0.80329408 61.9663 1426.722 -2218.3068 + 250 0.025 0.028733825 -2221.3495 -1.21118 47.173828 1010.1027 -2218.3068 + 300 0.03 0.028733813 -2220.6532 -1.6389575 36.378126 789.93596 -2218.3068 + 350 0.035 0.028733795 -2220.3656 -2.0724228 31.919862 648.39236 -2218.3068 + 400 0.04 0.028733778 -2220.5225 -2.498882 34.352305 545.8856 -2218.3068 + 450 0.045 0.02873377 -2221.0414 -2.9067963 42.396008 468.97305 -2218.3068 + 500 0.05 0.02873377 -2221.7614 -3.2881407 53.559839 411.03247 -2218.3068 + 550 0.055 0.028733775 -2222.5025 -3.6391769 65.048917 366.0991 -2218.3068 + 600 0.06 0.028733784 -2223.1215 -3.9612604 74.645803 330.37583 -2218.3068 + 650 0.065 0.028733794 -2223.5495 -4.2623843 81.281536 303.0489 -2218.3068 + 700 0.07 0.028733802 -2223.795 -4.5550161 85.087649 283.0897 -2218.3068 + 750 0.075 0.028733805 -2223.9222 -4.8523832 87.059996 267.48601 -2218.3068 + 800 0.08 0.028733804 -2224.0176 -5.1656984 88.538724 253.94215 -2218.3068 + 850 0.085 0.028733799 -2224.1506 -5.4981915 90.600346 242.22988 -2218.3068 + 900 0.09 0.028733794 -2224.3425 -5.8400709 93.576582 232.32115 -2218.3068 + 950 0.095 0.028733796 -2224.5674 -6.1746129 97.063173 222.64192 -2218.3068 + 1000 0.1 0.028733805 -2224.7832 -6.4920444 100.40752 212.7708 -2218.3068 +Loop time of 4.1438 on 1 procs for 1000 steps with 500 atoms + +Performance: 2.085 ns/day, 11.511 hours/ns, 241.324 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.8846 | 1.8846 | 1.8846 | 0.0 | 45.48 +Neigh | 0.012244 | 0.012244 | 0.012244 | 0.0 | 0.30 +Comm | 0.033385 | 0.033385 | 0.033385 | 0.0 | 0.81 +Output | 0.014202 | 0.014202 | 0.014202 | 0.0 | 0.34 +Modify | 2.1938 | 2.1938 | 2.1938 | 0.0 | 52.94 +Other | | 0.005526 | | | 0.13 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 19500 ave 19500 max 19500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 39000 ave 39000 max 39000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 new file mode 100644 index 0000000000..2ef20e4789 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel.g++.4 @@ -0,0 +1,141 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000680923 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7591 0.20498788 100.03408 -4775.0671 -2218.3068 + 50 0.005 0.028733805 -2224.5402 0.18523081 96.640497 -5356.0839 -2218.3068 + 100 0.01 0.028733812 -2223.9764 0.042321266 87.900506 -24443.229 -2218.3068 + 150 0.015 0.028733818 -2223.1275 -0.20362758 74.73931 5409.1535 -2218.3068 + 200 0.02 0.028733826 -2222.1368 -0.52607498 59.379029 2258.3662 -2218.3068 + 250 0.025 0.028733831 -2221.2102 -0.90147712 45.013713 1422.5963 -2218.3068 + 300 0.03 0.028733827 -2220.558 -1.3121131 34.902868 1037.9833 -2218.3068 + 350 0.035 0.028733817 -2220.3198 -1.7411113 31.20948 805.99534 -2218.3068 + 400 0.04 0.028733807 -2220.5162 -2.1695831 34.253525 646.07984 -2218.3068 + 450 0.045 0.028733802 -2221.0547 -2.581156 42.602248 532.25918 -2218.3068 + 500 0.05 0.028733805 -2221.7784 -2.9684862 53.822318 451.09419 -2218.3068 + 550 0.055 0.028733814 -2222.5255 -3.3338614 65.405638 392.41628 -2218.3068 + 600 0.06 0.028733822 -2223.1693 -3.6823124 75.38674 349.28401 -2218.3068 + 650 0.065 0.028733828 -2223.6339 -4.0182131 82.590109 317.40382 -2218.3068 + 700 0.07 0.028733827 -2223.907 -4.3515619 86.82355 294.41794 -2218.3068 + 750 0.075 0.028733822 -2224.0293 -4.6966825 88.720045 278.64174 -2218.3068 + 800 0.08 0.02873382 -2224.064 -5.0589353 89.257884 267.43696 -2218.3068 + 850 0.085 0.028733824 -2224.0702 -5.4291065 89.354006 257.51219 -2218.3068 + 900 0.09 0.028733833 -2224.0977 -5.7927329 89.780835 246.96548 -2218.3068 + 950 0.095 0.028733835 -2224.1852 -6.1414644 91.136895 236.30442 -2218.3068 + 1000 0.1 0.028733828 -2224.3442 -6.472877 93.602509 227.18362 -2218.3068 +Loop time of 2.57923 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.350 ns/day, 7.165 hours/ns, 387.713 timesteps/s +99.4% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.52633 | 0.53745 | 0.55615 | 1.6 | 20.84 +Neigh | 0.0030599 | 0.0031699 | 0.0032969 | 0.2 | 0.12 +Comm | 0.10599 | 0.1261 | 0.13713 | 3.4 | 4.89 +Output | 0.0051079 | 0.0051572 | 0.0052865 | 0.1 | 0.20 +Modify | 1.9032 | 1.9052 | 1.9065 | 0.1 | 73.87 +Other | | 0.002202 | | | 0.09 + +Nlocal: 125 ave 138 max 115 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +Nghost: 1099 ave 1109 max 1086 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Neighs: 4875 ave 5336 max 4525 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 9750 ave 10764 max 8970 min +Histogram: 1 0 0 2 0 0 0 0 0 1 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 new file mode 100644 index 0000000000..a9c6dc1ea4 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.1 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000916958 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7478 0.21626519 100.03408 -4301.853 -2218.2955 + 50 0.005 0.028732018 -2224.6158 0.10507552 97.987527 -8506.178 -2218.2955 + 100 0.01 0.028730353 -2224.1105 -0.11492231 90.153208 9821.9545 -2218.2955 + 150 0.015 0.028729179 -2223.2871 -0.42247044 77.388252 2612.6762 -2218.2955 + 200 0.02 0.028730623 -2222.2924 -0.79199024 61.966499 1465.6684 -2218.2955 + 250 0.025 0.02873338 -2221.3383 -1.199896 47.175023 1028.0909 -2218.2955 + 300 0.03 0.028732667 -2220.6421 -1.6277461 36.381371 800.14526 -2218.2955 + 350 0.035 0.028726646 -2220.3547 -2.0613125 31.92537 654.96081 -2218.2955 + 400 0.04 0.028719587 -2220.5116 -2.4878332 34.358607 550.47697 -2218.2955 + 450 0.045 0.028713837 -2221.0303 -2.8957078 42.400515 472.35844 -2218.2955 + 500 0.05 0.028707813 -2221.7502 -3.2769128 53.560569 413.68656 -2218.2955 + 550 0.055 0.028706758 -2222.4909 -3.627757 65.045564 368.28172 -2218.2955 + 600 0.06 0.02871485 -2223.1098 -3.9496387 74.639348 332.20637 -2218.2955 + 650 0.065 0.028726143 -2223.5376 -4.2505578 81.273271 304.61479 -2218.2954 + 700 0.07 0.028732341 -2223.7831 -4.5429323 85.078158 284.47251 -2218.2954 + 750 0.075 0.028732637 -2223.9102 -4.8399235 87.049135 268.76484 -2218.2954 + 800 0.08 0.028731303 -2224.0055 -5.1527352 88.526667 255.14878 -2218.2954 + 850 0.085 0.028728379 -2224.1384 -5.4846326 90.587963 243.32753 -2218.2954 + 900 0.09 0.028721628 -2224.3304 -5.8258679 93.565089 233.2995 -2218.2954 + 950 0.095 0.028715574 -2224.5555 -6.1597678 97.054035 223.54492 -2218.2954 + 1000 0.1 0.02871582 -2224.7714 -6.4766454 100.40218 213.59853 -2218.2954 +Loop time of 4.15073 on 1 procs for 1000 steps with 500 atoms + +Performance: 2.082 ns/day, 11.530 hours/ns, 240.921 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.8514 | 1.8514 | 1.8514 | 0.0 | 44.60 +Neigh | 0.012244 | 0.012244 | 0.012244 | 0.0 | 0.29 +Comm | 0.032971 | 0.032971 | 0.032971 | 0.0 | 0.79 +Output | 0.027815 | 0.027815 | 0.027815 | 0.0 | 0.67 +Modify | 2.2209 | 2.2209 | 2.2209 | 0.0 | 53.51 +Other | | 0.005408 | | | 0.13 + +Nlocal: 500 ave 500 max 500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 1956 ave 1956 max 1956 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 19500 ave 19500 max 19500 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 39000 ave 39000 max 39000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 new file mode 100644 index 0000000000..9a25cb1fb3 --- /dev/null +++ b/examples/SPIN/nickel/log.14Apr20.spin.nickel_cubic.g++.4 @@ -0,0 +1,142 @@ +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# fcc nickel in a 3d periodic box + +clear +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice fcc 3.524 +Lattice spacing in x,y,z = 3.524 3.524 3.524 +region box block 0.0 5.0 0.0 5.0 0.0 5.0 +create_box 1 box +Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box +Created 500 atoms + create_atoms CPU = 0.000750065 secs + +# setting mass, mag. moments, and interactions for cobalt + +mass 1 58.69 + +set group all spin/random 31 0.63 + 500 settings made for spin/random +#set group all spin 0.63 0.0 0.0 1.0 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 +pair_coeff * * eam/alloy Ni99.eam.alloy Ni +pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 +fix_modify 1 energy yes +fix 2 all langevin/spin 0.0 0.0 21 + +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable magz equal c_out_mag[3] +variable magnorm equal c_out_mag[4] +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_magnorm pe v_emag temp v_tmag etotal +thermo 50 + +compute outsp all property/atom spx spy spz sp fmx fmy fmz +dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] + +run 1000 +Neighbor list info ... + update every 10 steps, delay 20 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 5.90375 + ghost atom cutoff = 5.90375 + binsize = 2.95187, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair eam/alloy, perpetual, half/full from (2) + attributes: half, newton on + pair build: halffull/newton + stencil: none + bin: none + (2) pair spin/exchange, perpetual + attributes: full, newton on + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes +Step Time v_magnorm PotEng v_emag Temp v_tmag TotEng + 0 0 0.028733803 -2224.7478 0.21626519 100.03408 -4301.853 -2218.2955 + 50 0.005 0.028732084 -2224.5289 0.19650376 96.640616 -4777.5312 -2218.2955 + 100 0.01 0.028730714 -2223.9652 0.053583455 87.900827 -16071.152 -2218.2955 + 150 0.015 0.028730068 -2223.1163 -0.19237362 74.739735 6053.4731 -2218.2955 + 200 0.02 0.028732658 -2222.1255 -0.5148243 59.37938 2354.2236 -2218.2955 + 250 0.025 0.028736824 -2221.1989 -0.89021871 45.013765 1456.8539 -2218.2955 + 300 0.03 0.028734673 -2220.5467 -1.3008412 34.902634 1054.9996 -2218.2955 + 350 0.035 0.028726346 -2220.3085 -1.7298348 31.209321 816.00699 -2218.2955 + 400 0.04 0.028721022 -2220.5048 -2.1583012 34.253626 652.5234 -2218.2955 + 450 0.045 0.028722238 -2221.0433 -2.5698412 42.602344 536.67287 -2218.2955 + 500 0.05 0.02872462 -2221.767 -2.9571093 53.822165 454.37668 -2218.2955 + 550 0.055 0.02872545 -2222.5141 -3.3223962 65.405129 395.00593 -2218.2955 + 600 0.06 0.02873171 -2223.1579 -3.6706939 75.385253 351.37664 -2218.2955 + 650 0.065 0.028744439 -2223.6223 -4.0063211 82.586428 319.12437 -2218.2954 + 700 0.07 0.028756808 -2223.8952 -4.3392538 86.816357 295.88017 -2218.2954 + 750 0.075 0.028765368 -2224.0172 -4.6838051 88.708218 279.91792 -2218.2954 + 800 0.08 0.028773583 -2224.0515 -5.0453234 89.24056 268.55561 -2218.2954 + 850 0.085 0.028781239 -2224.0574 -5.4146399 89.331407 258.50935 -2218.2954 + 900 0.09 0.028787568 -2224.0847 -5.7774277 89.755159 247.86416 -2218.2954 + 950 0.095 0.028794703 -2224.1722 -6.1254539 91.111184 237.12719 -2218.2954 + 1000 0.1 0.028801253 -2224.3313 -6.4562841 93.577969 227.97114 -2218.2954 +Loop time of 2.49458 on 4 procs for 1000 steps with 500 atoms + +Performance: 3.464 ns/day, 6.929 hours/ns, 400.868 timesteps/s +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.52895 | 0.53652 | 0.54117 | 0.6 | 21.51 +Neigh | 0.003077 | 0.0032379 | 0.003371 | 0.2 | 0.13 +Comm | 0.11315 | 0.1193 | 0.12956 | 1.8 | 4.78 +Output | 0.0093312 | 0.0093816 | 0.0094745 | 0.1 | 0.38 +Modify | 1.8206 | 1.8239 | 1.8254 | 0.1 | 73.11 +Other | | 0.00229 | | | 0.09 + +Nlocal: 125 ave 138 max 115 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +Nghost: 1099 ave 1109 max 1086 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Neighs: 4875 ave 5337 max 4528 min +Histogram: 1 0 0 2 0 0 0 0 0 1 +FullNghs: 9750 ave 10764 max 8970 min +Histogram: 1 0 0 2 0 0 0 0 0 1 + +Total # of neighbors = 39000 +Ave neighs/atom = 78 +Neighbor list builds = 7 +Dangerous builds = 0 + + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 b/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 deleted file mode 100644 index 7871d82ae9..0000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.1 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000484943 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.40997576 100.03408 -4775.0671 -2218.3068 - 50 0.005 0.028733807 0.070491717 101.47879 -28153.519 -2218.1371 - 100 0.01 0.028733815 -0.70937134 101.7311 2925.8177 -2217.7471 - 150 0.015 0.028733823 -1.853981 99.63039 1197.9339 -2217.1748 - 200 0.02 0.028733828 -3.2679239 94.850105 741.17431 -2216.4678 - 250 0.025 0.028733824 -4.863967 88.444584 550.36979 -2215.6698 - 300 0.03 0.028733807 -6.5763457 82.689581 449.78321 -2214.8136 - 350 0.035 0.028733783 -8.3489158 80.108798 384.32228 -2213.9273 - 400 0.04 0.028733763 -10.120216 82.374947 335.01545 -2213.0417 - 450 0.045 0.028733755 -11.828932 89.814597 296.88965 -2212.1873 - 500 0.05 0.028733762 -13.423712 101.39613 267.51686 -2211.39 - 550 0.055 0.028733783 -14.866724 115.07399 244.96012 -2210.6684 - 600 0.06 0.028733801 -16.135279 128.57849 229.33327 -2210.0342 - 650 0.065 0.028733804 -17.222838 140.22402 220.05718 -2209.4904 - 700 0.07 0.028733795 -18.154813 149.61295 212.95678 -2209.0244 - 750 0.075 0.028733781 -18.996903 157.5814 206.41327 -2208.6034 - 800 0.08 0.028733768 -19.804249 164.92075 203.88977 -2208.1997 - 850 0.085 0.028733752 -20.579151 171.67278 203.42363 -2207.8122 - 900 0.09 0.028733728 -21.294277 177.67238 199.84817 -2207.4547 - 950 0.095 0.028733715 -21.943945 183.2621 194.9614 -2207.1298 - 1000 0.1 0.02873374 -22.551277 188.99284 191.59796 -2206.8262 -Loop time of 4.30614 on 1 procs for 1000 steps with 500 atoms - -Performance: 2.006 ns/day, 11.961 hours/ns, 232.227 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.038 | 2.038 | 2.038 | 0.0 | 47.33 -Neigh | 0.01566 | 0.01566 | 0.01566 | 0.0 | 0.36 -Comm | 0.032802 | 0.032802 | 0.032802 | 0.0 | 0.76 -Output | 0.014146 | 0.014146 | 0.014146 | 0.0 | 0.33 -Modify | 2.2003 | 2.2003 | 2.2003 | 0.0 | 51.10 -Other | | 0.005288 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 19504 ave 19504 max 19504 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 39008 ave 39008 max 39008 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 b/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 deleted file mode 100644 index 638ab5e635..0000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel.g++.4 +++ /dev/null @@ -1,136 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000733852 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.40997576 100.03408 -4775.0671 -2218.1018 - 50 0.005 0.028733805 0.25324083 98.741633 -7863.8744 -2218.1018 - 100 0.01 0.028733812 -0.37320751 97.073875 5622.1863 -2218.1018 - 150 0.015 0.028733819 -1.3971549 94.073447 1625.0258 -2218.1018 - 200 0.02 0.028733825 -2.7238372 89.419944 919.37601 -2218.1018 - 250 0.025 0.028733829 -4.2684428 84.07494 652.18375 -2218.1018 - 300 0.03 0.028733824 -5.9636712 80.06368 512.89077 -2218.1018 - 350 0.035 0.02873381 -7.7386326 79.366702 422.24864 -2218.1018 - 400 0.04 0.028733802 -9.5148059 83.052751 357.60379 -2218.1018 - 450 0.045 0.028733806 -11.234935 91.282747 310.87776 -2218.1018 - 500 0.05 0.02873381 -12.875184 103.49836 275.0224 -2218.1018 - 550 0.055 0.028733808 -14.413473 118.16526 247.85208 -2218.1018 - 600 0.06 0.028733803 -15.812466 132.83837 230.67903 -2218.1018 - 650 0.065 0.028733808 -17.061311 145.41049 222.19476 -2218.1018 - 700 0.07 0.028733818 -18.181903 154.83414 219.42933 -2218.1018 - 750 0.075 0.028733823 -19.176259 160.58645 218.45231 -2218.1018 - 800 0.08 0.028733825 -20.035157 163.02829 214.86596 -2218.1018 - 850 0.085 0.028733825 -20.806548 164.4197 209.86881 -2218.1018 - 900 0.09 0.028733829 -21.571419 167.8571 205.79849 -2218.1018 - 950 0.095 0.028733825 -22.365879 175.00875 201.33088 -2218.1018 - 1000 0.1 0.028733821 -23.133464 184.68305 195.52912 -2218.1018 -Loop time of 2.47211 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.495 ns/day, 6.867 hours/ns, 404.513 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.5549 | 0.56708 | 0.58627 | 1.6 | 22.94 -Neigh | 0.0039728 | 0.0041007 | 0.0042598 | 0.2 | 0.17 -Comm | 0.087296 | 0.10802 | 0.11917 | 3.8 | 4.37 -Output | 0.0046923 | 0.0047188 | 0.0047917 | 0.1 | 0.19 -Modify | 1.7832 | 1.7862 | 1.7879 | 0.1 | 72.25 -Other | | 0.002016 | | | 0.08 - -Nlocal: 125 ave 139 max 112 min -Histogram: 1 0 1 0 0 0 1 0 0 1 -Nghost: 1099 ave 1112 max 1085 min -Histogram: 1 0 0 1 0 0 0 1 0 1 -Neighs: 4876 ave 5386 max 4426 min -Histogram: 1 0 0 1 0 1 0 0 0 1 -FullNghs: 9752 ave 10845 max 8737 min -Histogram: 1 0 1 0 0 0 1 0 0 1 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 b/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 deleted file mode 100644 index 4f3b94df53..0000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.1 +++ /dev/null @@ -1,138 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 1 by 1 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.00109196 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 -fix_modify 1 energy yes -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.885 | 7.885 | 7.885 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.455085 100.03408 -4301.853 -2218.2955 - 50 0.005 0.028732021 0.11535308 101.47887 -17203.944 -2218.1258 - 100 0.01 0.0287304 -0.665283 101.73105 3119.2267 -2217.7358 - 150 0.015 0.028729403 -1.8105707 99.629794 1226.3803 -2217.1636 - 200 0.02 0.028731067 -3.224763 94.849715 750.93124 -2216.4566 - 250 0.025 0.028732765 -4.8207784 88.447019 555.16456 -2215.6585 - 300 0.03 0.028728169 -6.5331538 82.697813 452.6101 -2214.8022 - 350 0.035 0.02871707 -8.3059526 80.122838 386.20109 -2213.9158 - 400 0.04 0.028706605 -10.077613 82.389555 336.36118 -2213.03 - 450 0.045 0.028701727 -11.78634 89.823176 297.91478 -2212.1758 - 500 0.05 0.028706691 -13.380919 101.39804 268.32933 -2211.3785 - 550 0.055 0.028714065 -14.824128 115.07511 245.62893 -2210.6569 - 600 0.06 0.028713691 -16.093505 128.58093 229.91054 -2210.0225 - 650 0.065 0.028713232 -17.181217 140.22137 220.57591 -2209.4787 - 700 0.07 0.02871245 -18.113035 149.60156 213.40077 -2209.0127 - 750 0.075 0.028712431 -18.954952 157.56849 206.80962 -2208.5916 - 800 0.08 0.02872489 -19.762756 164.91833 204.24742 -2208.1876 - 850 0.085 0.028733709 -20.538757 171.69348 203.73934 -2207.7993 - 900 0.09 0.028737031 -21.256095 177.71981 200.12043 -2207.4406 - 950 0.095 0.028743446 -21.908156 183.31613 195.23386 -2207.1149 - 1000 0.1 0.028751809 -22.516179 189.01672 191.90401 -2206.8111 -Loop time of 4.3661 on 1 procs for 1000 steps with 500 atoms - -Performance: 1.979 ns/day, 12.128 hours/ns, 229.037 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 2.0467 | 2.0467 | 2.0467 | 0.0 | 46.88 -Neigh | 0.015636 | 0.015636 | 0.015636 | 0.0 | 0.36 -Comm | 0.032918 | 0.032918 | 0.032918 | 0.0 | 0.75 -Output | 0.027737 | 0.027737 | 0.027737 | 0.0 | 0.64 -Modify | 2.2379 | 2.2379 | 2.2379 | 0.0 | 51.26 -Other | | 0.005233 | | | 0.12 - -Nlocal: 500 ave 500 max 500 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Nghost: 1956 ave 1956 max 1956 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -Neighs: 19504 ave 19504 max 19504 min -Histogram: 1 0 0 0 0 0 0 0 0 0 -FullNghs: 39008 ave 39008 max 39008 min -Histogram: 1 0 0 0 0 0 0 0 0 0 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:04 diff --git a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 b/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 deleted file mode 100644 index 1724ef9df8..0000000000 --- a/examples/SPIN/nickel/log.19Nov19.spin.nickel_cubic.g++.4 +++ /dev/null @@ -1,138 +0,0 @@ -LAMMPS (30 Oct 2019) -# fcc nickel in a 3d periodic box - -clear -units metal -atom_style spin - -dimension 3 -boundary p p p - -# necessary for the serial algorithm (sametag) -atom_modify map array - -lattice fcc 3.524 -Lattice spacing in x,y,z = 3.524 3.524 3.524 -region box block 0.0 5.0 0.0 5.0 0.0 5.0 -create_box 1 box -Created orthogonal box = (0 0 0) to (17.62 17.62 17.62) - 1 by 2 by 2 MPI processor grid -create_atoms 1 box -Created 500 atoms - create_atoms CPU = 0.000827074 secs - -# setting mass, mag. moments, and interactions for cobalt - -mass 1 58.69 - -set group all spin/random 31 0.63 - 500 settings made for spin/random -#set group all spin 0.63 0.0 0.0 1.0 -velocity all create 100 4928459 rot yes dist gaussian - -pair_style hybrid/overlay eam/alloy spin/exchange 4.0 -pair_coeff * * eam/alloy Ni99.eam.alloy Ni -pair_coeff * * spin/exchange exchange 4.0 0.50 0.2280246862 1.229983475 - -neighbor 0.1 bin -neigh_modify every 10 check yes delay 20 - -fix 1 all precession/spin cubic -0.0001 0.0 1.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 1.0 zeeman 0.0 0.0 0.0 1.0 -fix_modify 1 energy yes -fix 2 all langevin/spin 0.0 0.0 21 - -fix 3 all nve/spin lattice moving -timestep 0.0001 - -# compute and output options - -compute out_mag all spin -compute out_pe all pe -compute out_ke all ke -compute out_temp all temp - -variable magz equal c_out_mag[3] -variable magnorm equal c_out_mag[4] -variable emag equal c_out_mag[5] -variable tmag equal c_out_mag[6] - -thermo_style custom step time v_magnorm v_emag temp v_tmag etotal -thermo 50 - -compute outsp all property/atom spx spy spz sp fmx fmy fmz -dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3] c_outsp[4] c_outsp[5] c_outsp[6] c_outsp[7] - -run 1000 -Neighbor list info ... - update every 10 steps, delay 20 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 5.90375 - ghost atom cutoff = 5.90375 - binsize = 2.95187, bins = 6 6 6 - 2 neighbor lists, perpetual/occasional/extra = 2 0 0 - (1) pair eam/alloy, perpetual, half/full from (2) - attributes: half, newton on - pair build: halffull/newton - stencil: none - bin: none - (2) pair spin/exchange, perpetual - attributes: full, newton on - pair build: full/bin/atomonly - stencil: full/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 7.799 | 7.799 | 7.799 Mbytes -Step Time v_magnorm v_emag Temp v_tmag TotEng - 0 0 0.028733803 0.455085 100.03408 -4301.853 -2218.0905 - 50 0.005 0.028732088 0.2980989 98.74184 -6680.4308 -2218.0904 - 100 0.01 0.02873076 -0.32911738 97.074246 6374.7026 -2218.09 - 150 0.015 0.028730298 -1.3537059 94.073558 1676.9365 -2218.0897 - 200 0.02 0.028733079 -2.6807428 89.419616 934.03305 -2218.0895 - 250 0.025 0.028735725 -4.2256641 84.074249 658.72816 -2218.0893 - 300 0.03 0.028728939 -5.9209085 80.063263 516.58161 -2218.0893 - 350 0.035 0.028716731 -7.6957087 79.36782 424.59625 -2218.0893 - 400 0.04 0.02871114 -9.4720832 83.055773 359.18204 -2218.0893 - 450 0.045 0.02870879 -11.19254 91.28713 312.02076 -2218.0891 - 500 0.05 0.028708873 -12.832707 103.50343 275.92991 -2218.0892 - 550 0.055 0.028710315 -14.370603 118.16778 248.59763 -2218.0893 - 600 0.06 0.028707016 -15.769641 132.83264 231.2886 -2218.089 - 650 0.065 0.028706727 -17.018362 145.39247 222.70304 -2218.0888 - 700 0.07 0.028710482 -18.137792 154.80131 219.85838 -2218.0889 - 750 0.075 0.028705169 -19.130471 160.53663 218.83811 -2218.0892 - 800 0.08 0.028695336 -19.988452 162.95918 215.21456 -2218.089 - 850 0.085 0.028688393 -20.758389 164.33238 210.21496 -2218.0889 - 900 0.09 0.028684101 -21.521505 167.76167 206.14977 -2218.089 - 950 0.095 0.028684705 -22.314351 174.918 201.65878 -2218.0891 - 1000 0.1 0.028691284 -23.080026 184.60192 195.8385 -2218.0893 -Loop time of 2.5947 on 4 procs for 1000 steps with 500 atoms - -Performance: 3.330 ns/day, 7.207 hours/ns, 385.402 timesteps/s -99.9% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.56409 | 0.58139 | 0.605 | 2.0 | 22.41 -Neigh | 0.0039618 | 0.0041058 | 0.0042889 | 0.2 | 0.16 -Comm | 0.095324 | 0.12147 | 0.13892 | 4.8 | 4.68 -Output | 0.008945 | 0.0089793 | 0.0090477 | 0.0 | 0.35 -Modify | 1.8745 | 1.8765 | 1.8795 | 0.1 | 72.32 -Other | | 0.002217 | | | 0.09 - -Nlocal: 125 ave 139 max 112 min -Histogram: 1 0 1 0 0 0 1 0 0 1 -Nghost: 1099 ave 1112 max 1085 min -Histogram: 1 0 0 1 0 0 0 1 0 1 -Neighs: 4876 ave 5385 max 4427 min -Histogram: 1 0 0 1 0 1 0 0 0 1 -FullNghs: 9752 ave 10845 max 8737 min -Histogram: 1 0 1 0 0 0 1 0 0 1 - -Total # of neighbors = 39008 -Ave neighs/atom = 78.016 -Neighbor list builds = 9 -Dangerous builds = 0 - - -Please see the log.cite file for references relevant to this simulation - -Total wall time: 0:00:02 diff --git a/examples/SPIN/read_restart/in.spin.read_data b/examples/SPIN/read_restart/in.spin.read_data index e788ecf67e..b2b55a9fcb 100644 --- a/examples/SPIN/read_restart/in.spin.read_data +++ b/examples/SPIN/read_restart/in.spin.read_data @@ -35,8 +35,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/read_restart/in.spin.restart b/examples/SPIN/read_restart/in.spin.restart index ccce25b254..985da65eb4 100644 --- a/examples/SPIN/read_restart/in.spin.restart +++ b/examples/SPIN/read_restart/in.spin.restart @@ -39,8 +39,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/read_restart/in.spin.write_restart b/examples/SPIN/read_restart/in.spin.write_restart index c127101093..19ab8e6b30 100644 --- a/examples/SPIN/read_restart/in.spin.write_restart +++ b/examples/SPIN/read_restart/in.spin.write_restart @@ -44,7 +44,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -52,4 +52,3 @@ dump 100 all custom 1 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ run 1000 write_restart restart_hcp_cobalt.equil - diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 similarity index 51% rename from examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 index d5c38a9967..4a744700f9 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 boundary p p p @@ -12,7 +14,7 @@ read_data Norm_randXY_8x8x32.data 1 by 1 by 1 MPI processor grid reading atoms ... 8192 atoms - read_data CPU = 0.0127251 secs + read_data CPU = 0.022048 secs mass 1 58.93 @@ -41,8 +43,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -66,33 +68,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 19.68 | 19.68 | 19.68 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0177864461018737 -1323.65841279979 1274.398774669 0 -36558.7284872918 - 10 0.001 0.0177864363786085 -1323.66900862123 1270.76616762926 0.0100007025152235 -36558.7231900452 - 20 0.002 0.0177864377251544 -1323.70032173151 1259.90270462032 0.0394803272360477 -36558.7075350597 - 30 0.003 0.0177864511986563 -1323.75117991179 1243.50772254923 0.0871132837928349 -36558.6821082609 - 40 0.004 0.0177864729727686 -1323.81992477224 1223.91535595806 0.150986538096776 -36558.6477386289 - 50 0.005 0.017786495620418 -1323.90456907402 1203.45497846157 0.22877054554493 -36558.6054195788 - 60 0.006 0.0177865119365897 -1324.00293472823 1183.95496070422 0.317876389336898 -36558.556239967 - 70 0.007 0.0177865186121948 -1324.11277680481 1166.52445270059 0.415601818818485 -36558.5013220755 - 80 0.008 0.0177865171615599 -1324.23190710734 1151.59958937508 0.519276751090729 -36558.4417598279 - 90 0.009 0.0177865117923882 -1324.35831839963 1139.14485136813 0.626407059487507 -36558.3785566998 - 100 0.01 0.0177865063215865 -1324.49029089774 1128.88117273962 0.734797362055872 -36558.3125725035 -Loop time of 14.8985 on 1 procs for 100 steps with 8192 atoms +Per MPI rank memory allocation (min/avg/max) = 19.99 | 19.99 | 19.99 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 0 0 0.0177864461018737 -36558.7284872918 -661.829206399896 1274.398774669 0 -36558.7284872918 + 20 0.002 0.0177864377256184 -36558.7389378387 -661.839683504936 1259.94171978912 0.00986992693139795 -36558.7284878577 + 40 0.004 0.017786472977471 -36558.7684525639 -661.869582914286 1224.05894016152 0.0377451568363827 -36558.7284891299 + 60 0.006 0.0177865119543331 -36558.8126238543 -661.915330492427 1184.24369688088 0.0794631076347515 -36558.728490712 + 80 0.008 0.0177865172048059 -36558.8659242367 -661.972562482488 1152.05459929593 0.129803482511904 -36558.7284922233 + 100 0.01 0.0177865063752424 -36558.9229549739 -662.037138807935 1129.51470280479 0.183667498513087 -36558.7284933644 +Loop time of 14.3276 on 1 procs for 100 steps with 8192 atoms -Performance: 0.058 ns/day, 413.847 hours/ns, 6.712 timesteps/s -99.6% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.060 ns/day, 397.988 hours/ns, 6.980 timesteps/s +99.7% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 4.5996 | 4.5996 | 4.5996 | 0.0 | 30.87 -Neigh | 3.6 | 3.6 | 3.6 | 0.0 | 24.16 -Comm | 0.057512 | 0.057512 | 0.057512 | 0.0 | 0.39 -Output | 2.4463 | 2.4463 | 2.4463 | 0.0 | 16.42 -Modify | 4.1766 | 4.1766 | 4.1766 | 0.0 | 28.03 -Other | | 0.01854 | | | 0.12 +Pair | 4.0409 | 4.0409 | 4.0409 | 0.0 | 28.20 +Neigh | 3.6219 | 3.6219 | 3.6219 | 0.0 | 25.28 +Comm | 0.055327 | 0.055327 | 0.055327 | 0.0 | 0.39 +Output | 2.4259 | 2.4259 | 2.4259 | 0.0 | 16.93 +Modify | 4.1688 | 4.1688 | 4.1688 | 0.0 | 29.10 +Other | | 0.01477 | | | 0.10 Nlocal: 8192 ave 8192 max 8192 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -110,4 +107,4 @@ Dangerous builds not checked Please see the log.cite file for references relevant to this simulation -Total wall time: 0:00:15 +Total wall time: 0:00:14 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 similarity index 51% rename from examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 index decd7f66de..6ce648abf3 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.read_data.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.read_data.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 boundary p p p @@ -12,7 +14,7 @@ read_data Norm_randXY_8x8x32.data 1 by 1 by 4 MPI processor grid reading atoms ... 8192 atoms - read_data CPU = 0.0103889 secs + read_data CPU = 0.013634 secs mass 1 58.93 @@ -41,8 +43,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -66,33 +68,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 8.422 | 8.508 | 8.751 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0177864461018739 -1323.65841279979 1274.398774669 0 -37220.5576936996 - 10 0.001 0.0177864050983603 -1323.66900862096 1270.76618998865 0.0100007022583634 -37220.5576943555 - 20 0.002 0.0177863936833753 -1323.70032172864 1259.90276925185 0.0394803245313843 -37220.5576959254 - 30 0.003 0.0177864120892274 -1323.75117990111 1243.50783331745 0.087113273744231 -37220.5576982173 - 40 0.004 0.0177864533855652 -1323.8199247464 1223.91551103958 0.150986513868405 -37220.557701015 - 50 0.005 0.0177865078997919 -1323.90456902433 1203.45516787752 0.228770499151177 -37220.5577041159 - 60 0.006 0.0177865576955448 -1324.0029346455 1183.95517338662 0.317876312538184 -37220.5577073314 - 70 0.007 0.0177865860816348 -1324.11277667948 1166.52467969539 0.415601703342581 -37220.5577104775 - 80 0.008 0.0177865881669081 -1324.23190693081 1151.59982868413 0.519276589926842 -37220.557713381 - 90 0.009 0.0177865780982769 -1324.35831816525 1139.14509878533 0.626406847905203 -37220.557715901 - 100 0.01 0.017786564605084 -1324.49029060173 1128.88143013641 0.734797098519806 -37220.557717952 -Loop time of 4.32342 on 4 procs for 100 steps with 8192 atoms +Per MPI rank memory allocation (min/avg/max) = 8.961 | 9.047 | 9.29 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 0 0 0.0177864461018739 -36558.7284872997 -661.829206399894 1274.398774669 0 -36558.7284872997 + 20 0.002 0.0177863981273124 -36558.7389378386 -661.839683504262 1259.94177798388 0.00986992629371963 -36558.7284878582 + 40 0.004 0.0177864622701489 -36558.7684525586 -661.869582908114 1224.05908191331 0.0377451510479599 -36558.7284891308 + 60 0.006 0.0177865625037858 -36558.8126238326 -661.915330472361 1184.24389640891 0.0794630890177406 -36558.72849071 + 80 0.008 0.0177865898045059 -36558.8659241943 -661.972562439245 1152.05483020781 0.129803443061299 -36558.7284922226 + 100 0.01 0.017786565190115 -36558.9229549058 -662.037138735432 1129.51495182843 0.183667434061771 -36558.7284933646 +Loop time of 4.35911 on 4 procs for 100 steps with 8192 atoms -Performance: 0.200 ns/day, 120.095 hours/ns, 23.130 timesteps/s -99.8% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 0.198 ns/day, 121.086 hours/ns, 22.940 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 1.185 | 1.1925 | 1.1991 | 0.5 | 27.58 -Neigh | 0.93459 | 0.93934 | 0.94983 | 0.6 | 21.73 -Comm | 0.042462 | 0.059373 | 0.069249 | 4.1 | 1.37 -Output | 0.61823 | 0.63273 | 0.64528 | 1.3 | 14.63 -Modify | 1.4827 | 1.4955 | 1.5099 | 0.8 | 34.59 -Other | | 0.003968 | | | 0.09 +Pair | 1.0924 | 1.1043 | 1.1117 | 0.7 | 25.33 +Neigh | 0.93575 | 0.94926 | 0.98325 | 2.0 | 21.78 +Comm | 0.044663 | 0.088288 | 0.11128 | 8.7 | 2.03 +Output | 0.64199 | 0.6587 | 0.67226 | 1.4 | 15.11 +Modify | 1.5412 | 1.5535 | 1.5706 | 0.9 | 35.64 +Other | | 0.005046 | | | 0.12 Nlocal: 2048 ave 2061 max 2035 min Histogram: 1 0 0 1 0 0 1 0 0 1 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 similarity index 53% rename from examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 index c7544dedfa..652f7da474 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # start a spin-lattice simulation from a data file units metal atom_style spin @@ -10,13 +12,13 @@ boundary p p p atom_modify map array read_restart restart_hcp_cobalt.equil -WARNING: Restart file used different # of processors: 4 vs. 1 (../read_restart.cpp:742) +WARNING: Restart file used different # of processors: 4 vs. 1 (../read_restart.cpp:736) restoring atom style spin from restart orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 1 by 1 MPI processor grid restoring pair style spin/exchange from restart 500 atoms - read_restart CPU = 0.000396967 secs + read_restart CPU = 0.00179696 secs # setting mass, mag. moments, and interactions @@ -47,8 +49,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -72,33 +74,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 6.816 | 6.816 | 6.816 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 1000 0 0.108317262557656 -10.7649197733649 2538.4247868621 0 -2200.38241212222 - 1010 0.001 0.108317281393701 -10.7743543303502 2527.22692799144 0.146167392153018 -2200.3776953858 - 1020 0.002 0.108317318482233 -10.8022550516195 2509.47863584151 0.577304300153637 -2200.36374625815 - 1030 0.003 0.108317366763426 -10.8476659807571 2487.5614649682 1.27529086243277 -2200.34104256596 - 1040 0.004 0.108317415532953 -10.9092708333549 2463.97963921611 2.21443906326453 -2200.31024227618 - 1050 0.005 0.108317453851058 -10.98553406179 2440.70473642157 3.36396898978859 -2200.27211302201 - 1060 0.006 0.108317473584086 -11.0748008072977 2418.66477599297 4.68991434259399 -2200.22748216359 - 1070 0.007 0.108317471632913 -11.175325501803 2397.59274785581 6.15596240129541 -2200.17722244953 - 1080 0.008 0.108317450667394 -11.2852665400894 2376.32871275528 7.7237909750654 -2200.12225459883 - 1090 0.009 0.108317417687893 -11.4027246787047 2353.52646917989 9.35409156720424 -2200.06352807392 - 1100 0.01 0.108317381194814 -11.52585602487 2328.41541723561 11.0087303030003 -2200.0019646458 -Loop time of 0.964681 on 1 procs for 100 steps with 500 atoms +Per MPI rank memory allocation (min/avg/max) = 7.422 | 7.422 | 7.422 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 1000 0 0.108317262557656 -2200.38241212222 -5.38245988668244 2538.4247868621 0 -2200.38241212222 + 1020 0.002 0.108317318495042 -2200.39172132133 -5.39179331134703 2513.42968070374 0.144319963844279 -2200.38241256643 + 1040 0.004 0.108317415558744 -2200.41811580407 -5.418541526637 2478.87571728648 0.553516420254567 -2200.38241354532 + 1060 0.006 0.108317473592946 -2200.45801216332 -5.45990062771403 2449.77257658726 1.17203792179707 -2200.38241476526 + 1080 0.008 0.108317450745396 -2200.5068824087 -5.51245983698347 2427.25022669715 1.92968606059505 -2200.3824160902 + 1100 0.01 0.108317381572202 -2200.55976028827 -5.57250071024394 2400.86131889957 2.74946927499959 -2200.38241728649 +Loop time of 0.954493 on 1 procs for 100 steps with 500 atoms -Performance: 0.896 ns/day, 26.797 hours/ns, 103.661 timesteps/s -100.0% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 0.905 ns/day, 26.514 hours/ns, 104.768 timesteps/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.29842 | 0.29842 | 0.29842 | 0.0 | 30.93 -Neigh | 0.25359 | 0.25359 | 0.25359 | 0.0 | 26.29 -Comm | 0.0069926 | 0.0069926 | 0.0069926 | 0.0 | 0.72 -Output | 0.14398 | 0.14398 | 0.14398 | 0.0 | 14.93 -Modify | 0.26045 | 0.26045 | 0.26045 | 0.0 | 27.00 -Other | | 0.001249 | | | 0.13 +Pair | 0.27043 | 0.27043 | 0.27043 | 0.0 | 28.33 +Neigh | 0.26148 | 0.26148 | 0.26148 | 0.0 | 27.40 +Comm | 0.0071123 | 0.0071123 | 0.0071123 | 0.0 | 0.75 +Output | 0.14169 | 0.14169 | 0.14169 | 0.0 | 14.84 +Modify | 0.2726 | 0.2726 | 0.2726 | 0.0 | 28.56 +Other | | 0.001178 | | | 0.12 Nlocal: 500 ave 500 max 500 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 similarity index 53% rename from examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 index 6443d341cd..c22c672f6b 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.restart.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.restart.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # start a spin-lattice simulation from a data file units metal atom_style spin @@ -15,7 +17,7 @@ read_restart restart_hcp_cobalt.equil 1 by 2 by 2 MPI processor grid restoring pair style spin/exchange from restart 500 atoms - read_restart CPU = 0.000922918 secs + read_restart CPU = 0.00173593 secs # setting mass, mag. moments, and interactions @@ -46,8 +48,8 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo 10 -thermo_style custom step time v_magnorm v_emag v_tmag temp etotal +thermo 20 +thermo_style custom step time v_magnorm pe v_emag v_tmag temp etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,33 +73,28 @@ Neighbor list info ... pair build: full/bin/atomonly stencil: full/bin/3d bin: standard -Per MPI rank memory allocation (min/avg/max) = 6.704 | 6.704 | 6.704 Mbytes -Step Time v_magnorm v_emag v_tmag Temp TotEng - 1000 0 0.108317262557656 -10.7649197733649 2538.4247868621 0 -2205.7648720085 - 1010 0.001 0.108317290362529 -10.7743543303489 2527.22680531097 0.146167392137698 -2205.76487255096 - 1020 0.002 0.108317316207642 -10.8022550521284 2509.47840782645 0.577304308061779 -2205.76487378396 - 1030 0.003 0.108317335980455 -10.8476659832667 2487.56119588937 1.27529090130452 -2205.76487555634 - 1040 0.004 0.108317347902639 -10.9092708400684 2463.97936529674 2.21443916694928 -2205.76487769286 - 1050 0.005 0.108317349786635 -10.9855340757384 2440.7044253165 3.36396920446814 -2205.76488005291 - 1060 0.006 0.108317342445881 -11.0748008315013 2418.66438763214 4.68991471343994 -2205.76488256723 - 1070 0.007 0.10831733355314 -11.1753255362286 2397.59228728929 6.15596292529133 -2205.76488520046 - 1080 0.008 0.108317320750099 -11.2852665775656 2376.32820919279 7.7237915384778 -2205.76488786888 - 1090 0.009 0.108317304079233 -11.402724701646 2353.52588586648 9.35409189724323 -2205.7648904133 - 1100 0.01 0.108317284409678 -11.5258560062539 2328.41472376239 11.0087299868288 -2205.76489265829 -Loop time of 0.410707 on 4 procs for 100 steps with 500 atoms +Per MPI rank memory allocation (min/avg/max) = 7.324 | 7.324 | 7.324 Mbytes +Step Time v_magnorm PotEng v_emag v_tmag Temp TotEng + 1000 0 0.108317262557656 -2200.38241212182 -5.38245988668244 2538.4247868621 0 -2200.38241212182 + 1020 0.002 0.108317316216432 -2200.39172132147 -5.39179331147409 2513.42945241007 0.14431996581917 -2200.38241256644 + 1040 0.004 0.108317347939802 -2200.41811580574 -5.41854152831072 2478.87544274124 0.553516446104432 -2200.38241354532 + 1060 0.006 0.108317342440309 -2200.45801216927 -5.45990063373049 2449.77218633122 1.17203801398165 -2200.38241476526 + 1080 0.008 0.108317320345284 -2200.50688241767 -5.51245984623572 2427.2497145488 1.92968619968329 -2200.3824160902 + 1100 0.01 0.10831728372281 -2200.55976028296 -5.57250070536486 2400.86059511731 2.74946919265255 -2200.38241728649 +Loop time of 0.405615 on 4 procs for 100 steps with 500 atoms -Performance: 2.104 ns/day, 11.409 hours/ns, 243.483 timesteps/s -99.7% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 2.130 ns/day, 11.267 hours/ns, 246.539 timesteps/s +99.7% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.083043 | 0.083714 | 0.084202 | 0.1 | 20.38 -Neigh | 0.063158 | 0.064429 | 0.065314 | 0.3 | 15.69 -Comm | 0.012237 | 0.013588 | 0.014798 | 0.8 | 3.31 -Output | 0.039088 | 0.040653 | 0.042176 | 0.6 | 9.90 -Modify | 0.20645 | 0.20795 | 0.20945 | 0.2 | 50.63 -Other | | 0.0003724 | | | 0.09 +Pair | 0.075661 | 0.076798 | 0.077343 | 0.2 | 18.93 +Neigh | 0.063154 | 0.064974 | 0.066991 | 0.5 | 16.02 +Comm | 0.012538 | 0.013787 | 0.015151 | 0.8 | 3.40 +Output | 0.039155 | 0.040842 | 0.042502 | 0.6 | 10.07 +Modify | 0.20709 | 0.20883 | 0.21036 | 0.3 | 51.49 +Other | | 0.0003826 | | | 0.09 Nlocal: 125 ave 127 max 122 min Histogram: 1 0 0 0 1 0 0 0 0 2 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 similarity index 59% rename from examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 rename to examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 index 9dbe1a3548..8ad14c55c0 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.1 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # fcc cobalt in a 3d periodic box units metal @@ -18,7 +20,7 @@ Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 500 atoms - create_atoms CPU = 0.000552893 secs + create_atoms CPU = 0.000952005 secs # setting mass, mag. moments, and interactions for cobalt @@ -51,7 +53,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,32 +73,32 @@ Neighbor list info ... stencil: full/bin/3d bin: standard Per MPI rank memory allocation (min/avg/max) = 6.947 | 6.947 | 6.947 Mbytes -Step Time v_magnorm v_emag Temp TotEng - 0 0 0.076558814 1.7982359 0 0.89911794 - 100 0.01 0.077628154 0.73387834 0 0.36693917 - 200 0.02 0.076678996 -0.4048463 0 -0.20242315 - 300 0.03 0.079174837 -1.3519103 0 -0.67595514 - 400 0.04 0.085031632 -3.0345702 0 -1.5172851 - 500 0.05 0.08702747 -4.0853256 0 -2.0426628 - 600 0.06 0.087066482 -5.259549 0 -2.6297745 - 700 0.07 0.089788894 -6.629076 0 -3.314538 - 800 0.08 0.091699611 -8.0574087 0 -4.0287043 - 900 0.09 0.090038899 -9.2012019 0 -4.600601 - 1000 0.1 0.093257309 -10.470452 0 -5.2352261 -Loop time of 3.37852 on 1 procs for 1000 steps with 500 atoms +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.076558814 0.89911794 0.89911794 0 0.89911794 + 100 0.01 0.077628154 0.36693917 0.36693917 0 0.36693917 + 200 0.02 0.076678996 -0.20242315 -0.20242315 0 -0.20242315 + 300 0.03 0.079174837 -0.67595514 -0.67595514 0 -0.67595514 + 400 0.04 0.085031632 -1.5172851 -1.5172851 0 -1.5172851 + 500 0.05 0.08702747 -2.0426628 -2.0426628 0 -2.0426628 + 600 0.06 0.087066482 -2.6297745 -2.6297745 0 -2.6297745 + 700 0.07 0.089788894 -3.314538 -3.314538 0 -3.314538 + 800 0.08 0.091699611 -4.0287043 -4.0287043 0 -4.0287043 + 900 0.09 0.090038899 -4.600601 -4.600601 0 -4.600601 + 1000 0.1 0.093257309 -5.2352261 -5.2352261 0 -5.2352261 +Loop time of 3.30071 on 1 procs for 1000 steps with 500 atoms -Performance: 2.557 ns/day, 9.385 hours/ns, 295.987 timesteps/s -99.6% CPU use with 1 MPI tasks x no OpenMP threads +Performance: 2.618 ns/day, 9.169 hours/ns, 302.965 timesteps/s +99.3% CPU use with 1 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.45808 | 0.45808 | 0.45808 | 0.0 | 13.56 +Pair | 0.3844 | 0.3844 | 0.3844 | 0.0 | 11.65 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.019707 | 0.019707 | 0.019707 | 0.0 | 0.58 -Output | 1.3865 | 1.3865 | 1.3865 | 0.0 | 41.04 -Modify | 1.5106 | 1.5106 | 1.5106 | 0.0 | 44.71 -Other | | 0.003624 | | | 0.11 +Comm | 0.019863 | 0.019863 | 0.019863 | 0.0 | 0.60 +Output | 1.3844 | 1.3844 | 1.3844 | 0.0 | 41.94 +Modify | 1.5084 | 1.5084 | 1.5084 | 0.0 | 45.70 +Other | | 0.00367 | | | 0.11 Nlocal: 500 ave 500 max 500 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -113,7 +115,6 @@ Neighbor list builds = 0 Dangerous builds = 0 write_restart restart_hcp_cobalt.equil - Please see the log.cite file for references relevant to this simulation Total wall time: 0:00:03 diff --git a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 similarity index 59% rename from examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 rename to examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 index cb98274603..c0c1ec130c 100644 --- a/examples/SPIN/read_restart/log.19Nov19.spin.write_restart.g++.4 +++ b/examples/SPIN/read_restart/log.14Apr20.spin.write_restart.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # fcc cobalt in a 3d periodic box units metal @@ -18,7 +20,7 @@ Created orthogonal box = (0 0 0) to (12.5355 21.7121 20.4704) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 500 atoms - create_atoms CPU = 0.000751972 secs + create_atoms CPU = 0.000663042 secs # setting mass, mag. moments, and interactions for cobalt @@ -51,7 +53,7 @@ variable magnorm equal c_out_mag[4] variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] -thermo_style custom step time v_magnorm v_emag temp etotal +thermo_style custom step time v_magnorm pe v_emag temp etotal thermo 100 compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -71,32 +73,32 @@ Neighbor list info ... stencil: full/bin/3d bin: standard Per MPI rank memory allocation (min/avg/max) = 6.868 | 6.868 | 6.868 Mbytes -Step Time v_magnorm v_emag Temp TotEng - 0 0 0.076558814 1.7982359 0 1.7982359 - 100 0.01 0.078299981 0.88259584 0 0.88259584 - 200 0.02 0.081260508 -0.43484722 0 -0.43484722 - 300 0.03 0.081195603 -1.7408209 0 -1.7408209 - 400 0.04 0.087298495 -3.4139038 0 -3.4139038 - 500 0.05 0.087663924 -4.3766089 0 -4.3766089 - 600 0.06 0.091713683 -5.8534921 0 -5.8534921 - 700 0.07 0.093779119 -6.706628 0 -6.706628 - 800 0.08 0.097960611 -7.8688568 0 -7.8688568 - 900 0.09 0.10193463 -9.5888008 0 -9.5888008 - 1000 0.1 0.10831726 -10.76492 0 -10.76492 -Loop time of 1.70473 on 4 procs for 1000 steps with 500 atoms +Step Time v_magnorm PotEng v_emag Temp TotEng + 0 0 0.076558814 0.89911794 0.89911794 0 0.89911794 + 100 0.01 0.078299981 0.44129792 0.44129792 0 0.44129792 + 200 0.02 0.081260508 -0.21742361 -0.21742361 0 -0.21742361 + 300 0.03 0.081195603 -0.87041046 -0.87041046 0 -0.87041046 + 400 0.04 0.087298495 -1.7069519 -1.7069519 0 -1.7069519 + 500 0.05 0.087663924 -2.1883045 -2.1883045 0 -2.1883045 + 600 0.06 0.091713683 -2.9267461 -2.9267461 0 -2.9267461 + 700 0.07 0.093779119 -3.353314 -3.353314 0 -3.353314 + 800 0.08 0.097960611 -3.9344284 -3.9344284 0 -3.9344284 + 900 0.09 0.10193463 -4.7944004 -4.7944004 0 -4.7944004 + 1000 0.1 0.10831726 -5.3824599 -5.3824599 0 -5.3824599 +Loop time of 1.7839 on 4 procs for 1000 steps with 500 atoms -Performance: 5.068 ns/day, 4.735 hours/ns, 586.602 timesteps/s -99.6% CPU use with 4 MPI tasks x no OpenMP threads +Performance: 4.843 ns/day, 4.955 hours/ns, 560.569 timesteps/s +99.5% CPU use with 4 MPI tasks x 1 OpenMP threads MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.11636 | 0.11927 | 0.12069 | 0.5 | 7.00 +Pair | 0.10068 | 0.10749 | 0.11461 | 1.5 | 6.03 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.049208 | 0.052445 | 0.057424 | 1.4 | 3.08 -Output | 0.38579 | 0.40345 | 0.4199 | 2.0 | 23.67 -Modify | 1.1138 | 1.1282 | 1.1436 | 1.1 | 66.18 -Other | | 0.001322 | | | 0.08 +Comm | 0.052378 | 0.062171 | 0.07177 | 2.8 | 3.49 +Output | 0.4054 | 0.42334 | 0.44025 | 2.0 | 23.73 +Modify | 1.174 | 1.1893 | 1.2043 | 1.1 | 66.67 +Other | | 0.001558 | | | 0.09 Nlocal: 125 ave 125 max 125 min Histogram: 4 0 0 0 0 0 0 0 0 0 @@ -113,7 +115,6 @@ Neighbor list builds = 0 Dangerous builds = 0 write_restart restart_hcp_cobalt.equil - Please see the log.cite file for references relevant to this simulation Total wall time: 0:00:01 diff --git a/examples/SPIN/read_restart/restart_hcp_cobalt.equil b/examples/SPIN/read_restart/restart_hcp_cobalt.equil new file mode 100644 index 0000000000..d7f92285db Binary files /dev/null and b/examples/SPIN/read_restart/restart_hcp_cobalt.equil differ diff --git a/examples/SPIN/run_spin_examples.sh b/examples/SPIN/run_spin_examples.sh index a71da82a04..432d70c2ba 100755 --- a/examples/SPIN/run_spin_examples.sh +++ b/examples/SPIN/run_spin_examples.sh @@ -1,6 +1,6 @@ #!/bin/bash -DATE=19Nov19 +DATE=14Apr20 # bfo cd bfo/ diff --git a/examples/SPIN/run_spin_examples_serial.sh b/examples/SPIN/run_spin_examples_serial.sh new file mode 100755 index 0000000000..f367dfd8f4 --- /dev/null +++ b/examples/SPIN/run_spin_examples_serial.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +DATE=21Fev20 + +# bfo +cd bfo/ +../../../src/lmp_serial -in in.spin.bfo +cp log.lammps log.${DATE}.spin.bfo.g++.1 +../../../src/lmp_serial -in in.spin.bfo +cp log.lammps log.${DATE}.spin.bfo.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# fcc cobalt +cd cobalt_fcc/ +../../../src/lmp_serial -in in.spin.cobalt_fcc +cp log.lammps log.${DATE}.spin.cobalt_fcc.g++.1 +../../../src/lmp_serial -in in.spin.cobalt_fcc +cp log.lammps log.${DATE}.spin.cobalt_fcc.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# hcp cobalt +cd cobalt_hcp/ +../../../src/lmp_serial -in in.spin.cobalt_hcp +cp log.lammps log.${DATE}.spin.cobalt_hcp.g++.1 +../../../src/lmp_serial -in in.spin.cobalt_hcp +cp log.lammps log.${DATE}.spin.cobalt_hcp.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# dipole spin +cd dipole_spin/ +../../../src/lmp_serial -in in.spin.iron_dipole_cut +cp log.lammps log.${DATE}.spin.iron_dipole_cut.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_cut +cp log.lammps log.${DATE}.spin.iron_dipole_cut.g++.4 +../../../src/lmp_serial -in in.spin.iron_dipole_ewald +cp log.lammps log.${DATE}.spin.iron_dipole_ewald.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_ewald +cp log.lammps log.${DATE}.spin.iron_dipole_ewald.g++.4 +../../../src/lmp_serial -in in.spin.iron_dipole_pppm +cp log.lammps log.${DATE}.spin.iron_dipole_pppm.g++.1 +../../../src/lmp_serial -in in.spin.iron_dipole_pppm +cp log.lammps log.${DATE}.spin.iron_dipole_pppm.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# bcc iron +cd iron/ +../../../src/lmp_serial -in in.spin.iron +cp log.lammps log.${DATE}.spin.iron.g++.1 +../../../src/lmp_serial -in in.spin.iron +cp log.lammps log.${DATE}.spin.iron.g++.4 +../../../src/lmp_serial -in in.spin.iron_cubic +cp log.lammps log.${DATE}.spin.iron_cubic.g++.1 +../../../src/lmp_serial -in in.spin.iron_cubic +cp log.lammps log.${DATE}.spin.iron_cubic.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# fcc nickel +cd nickel/ +../../../src/lmp_serial -in in.spin.nickel +cp log.lammps log.${DATE}.spin.nickel.g++.1 +../../../src/lmp_serial -in in.spin.nickel +cp log.lammps log.${DATE}.spin.nickel.g++.4 +../../../src/lmp_serial -in in.spin.nickel_cubic +cp log.lammps log.${DATE}.spin.nickel_cubic.g++.1 +../../../src/lmp_serial -in in.spin.nickel_cubic +cp log.lammps log.${DATE}.spin.nickel_cubic.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# read restart +cd read_restart/ +../../../src/lmp_serial -in in.spin.write_restart +cp log.lammps log.${DATE}.spin.write_restart.g++.1 +../../../src/lmp_serial -in in.spin.write_restart +cp log.lammps log.${DATE}.spin.write_restart.g++.4 +../../../src/lmp_serial -in in.spin.restart +cp log.lammps log.${DATE}.spin.restart.g++.1 +../../../src/lmp_serial -in in.spin.restart +cp log.lammps log.${DATE}.spin.restart.g++.4 +../../../src/lmp_serial -in in.spin.read_data +cp log.lammps log.${DATE}.spin.read_data.g++.1 +../../../src/lmp_serial -in in.spin.read_data +cp log.lammps log.${DATE}.spin.read_data.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# setforce +cd setforce_spin/ +../../../src/lmp_serial -in in.spin.setforce +cp log.lammps log.${DATE}.spin.setforce.g++.1 +../../../src/lmp_serial -in in.spin.setforce +cp log.lammps log.${DATE}.spin.setforce.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. + +# spin minimizers +cd spinmin/ +../../../src/lmp_serial -in in.spin.bfo_min +cp log.lammps log.${DATE}.spin.bfo_min.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min +cp log.lammps log.${DATE}.spin.bfo_min.g++.4 +../../../src/lmp_serial -in in.spin.bfo_min_cg +cp log.lammps log.${DATE}.spin.bfo_min_cg.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min_cg +cp log.lammps log.${DATE}.spin.bfo_min_cg.g++.4 +../../../src/lmp_serial -in in.spin.bfo_min_lbfgs +cp log.lammps log.${DATE}.spin.bfo_min_lbfgs.g++.1 +../../../src/lmp_serial -in in.spin.bfo_min_lbfgs +cp log.lammps log.${DATE}.spin.bfo_min_lbfgs.g++.4 +../../../src/lmp_serial -in in.spin.iron_min +cp log.lammps log.${DATE}.spin.iron_min.g++.1 +../../../src/lmp_serial -in in.spin.iron_min +cp log.lammps log.${DATE}.spin.iron_min.g++.4 +rm log.lammps log.cite dump*.lammpstrj +cd .. diff --git a/examples/SPIN/setforce_spin/in.spin.setforce b/examples/SPIN/setforce_spin/in.spin.setforce index 0d65955a29..4edf70eb52 100644 --- a/examples/SPIN/setforce_spin/in.spin.setforce +++ b/examples/SPIN/setforce_spin/in.spin.setforce @@ -48,7 +48,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz diff --git a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 similarity index 79% rename from examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 rename to examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 index ed5c037feb..ef5e86b0b3 100644 --- a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.1 +++ b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 @@ -18,10 +20,10 @@ region reg1 block 0.0 10.0 0.0 5.0 0.0 1.0 region reg2 block 0.0 10.0 6.0 10.0 0.0 1.0 create_atoms 1 region reg1 Created 120 atoms - create_atoms CPU = 0.000998974 secs + create_atoms CPU = 0.000992775 secs create_atoms 2 region reg2 Created 80 atoms - create_atoms CPU = 4.1008e-05 secs + create_atoms CPU = 0.000215054 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,7 +61,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -68,7 +70,7 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 20.0 minimize 1.0e-16 1.0e-16 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,21 +89,21 @@ Neighbor list info ... stencil: none bin: none Per MPI rank memory allocation (min/avg/max) = 7.215 | 7.215 | 7.215 Mbytes -Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 - 100 0.01 0.000616167502203097 0.594467364025194 0.594498630048783 0.00188964439583802 0.0371335982020527 - 200 0.02 0.000498981016106215 0.595175581059792 0.595218717456538 0.000158614984300385 0.036877233648055 - 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182375 - 400 0.04 7.98967577397158e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548781 - 500 0.05 2.9121648914103e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239539 - 600 0.06 1.04772320898497e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.0368524982492743 - 700 0.07 3.74634771616422e-06 0.595404627382825 0.595449536940641 2.63852407890463e-09 0.036852494912626 - 800 0.08 1.33525617457914e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963445 - 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440918 - 1000 0.1 1.68843135202601e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.036852494437518 -Loop time of 0.0966749 on 1 procs for 1000 steps with 200 atoms +Step Time v_magx v_magz v_magnorm v_tmag PotEng v_emag TotEng + 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 0.121881906963737 0.121881906963737 + 100 0.01 0.000616167502203097 0.594467364025194 0.594498630048783 0.00188964439583802 0.0371335982020527 0.0371335982020522 0.0371335982020527 + 200 0.02 0.000498981016106215 0.595175581059792 0.595218717456538 0.000158614984300385 0.036877233648055 0.0368772336480544 0.036877233648055 + 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182375 0.0368548794182369 0.0368548794182375 + 400 0.04 7.98967577397158e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548781 0.0368527556548775 0.0368527556548781 + 500 0.05 2.9121648914103e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239539 0.0368525254239533 0.0368525254239539 + 600 0.06 1.04772320898497e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.0368524982492743 0.0368524982492735 0.0368524982492743 + 700 0.07 3.74634771616422e-06 0.595404627382825 0.595449536940641 2.63852407890463e-09 0.036852494912626 0.0368524949126256 0.036852494912626 + 800 0.08 1.33525617457914e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963445 0.0368524944963439 0.0368524944963445 + 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440918 0.0368524944440912 0.0368524944440918 + 1000 0.1 1.68843135202601e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.036852494437518 0.0368524944375173 0.036852494437518 +Loop time of 0.093256 on 1 procs for 1000 steps with 200 atoms -100.0% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.08679 | 0.08679 | 0.08679 | 0.0 | 89.77 +Pair | 0.08325 | 0.08325 | 0.08325 | 0.0 | 89.27 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 7.2956e-05 | 7.2956e-05 | 7.2956e-05 | 0.0 | 0.08 -Output | 0.0033231 | 0.0033231 | 0.0033231 | 0.0 | 3.44 -Modify | 0.0022919 | 0.0022919 | 0.0022919 | 0.0 | 2.37 -Other | | 0.004197 | | | 4.34 +Comm | 6.032e-05 | 6.032e-05 | 6.032e-05 | 0.0 | 0.06 +Output | 0.0033138 | 0.0033138 | 0.0033138 | 0.0 | 3.55 +Modify | 0.0023961 | 0.0023961 | 0.0023961 | 0.0 | 2.57 +Other | | 0.004236 | | | 4.54 Nlocal: 200 ave 200 max 200 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 similarity index 57% rename from examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 rename to examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 index 4ce1497957..c99bad0b40 100644 --- a/examples/SPIN/setforce_spin/log.19Nov19.spin.setforce.g++.4 +++ b/examples/SPIN/setforce_spin/log.14Apr20.spin.setforce.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task units metal dimension 3 @@ -18,10 +20,10 @@ region reg1 block 0.0 10.0 0.0 5.0 0.0 1.0 region reg2 block 0.0 10.0 6.0 10.0 0.0 1.0 create_atoms 1 region reg1 Created 120 atoms - create_atoms CPU = 0.000770092 secs + create_atoms CPU = 0.000747204 secs create_atoms 2 region reg2 Created 80 atoms - create_atoms CPU = 7.9155e-05 secs + create_atoms CPU = 0.000154018 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,7 +61,7 @@ variable emag equal c_out_mag[5] variable tmag equal c_out_mag[6] thermo 100 -thermo_style custom step time v_magx v_magz v_magnorm v_tmag etotal +thermo_style custom step time v_magx v_magz v_magnorm v_tmag pe v_emag etotal thermo_modify format float %20.15g compute outsp all property/atom spx spy spz sp fmx fmy fmz @@ -68,7 +70,7 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 20.0 minimize 1.0e-16 1.0e-16 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,26 +89,26 @@ Neighbor list info ... stencil: none bin: none Per MPI rank memory allocation (min/avg/max) = 7.2 | 7.2 | 7.2 Mbytes -Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 21.735436446264 0.251043691626527 - 100 0.01 -0.00116085697754605 0.590264559350799 0.590315072966953 0.00283413081085227 0.0846048989956832 - 200 0.02 -0.000147570523236238 0.5896197627388 0.589686497206689 0.000451051163122381 0.0839054390713705 - 300 0.03 2.64982966536902e-05 0.59002969475615 0.590102003120244 5.22539631503911e-05 0.0838351677819014 - 400 0.04 1.77805448780033e-05 0.590195117338991 0.590268726215095 4.46490059775722e-06 0.0838382933245032 - 500 0.05 6.71566571038784e-06 0.590243842081075 0.590317756995865 3.63227563542099e-07 0.0838411433937997 - 600 0.06 2.2410340743112e-06 0.590257551861528 0.590331542128336 2.99360370345601e-08 0.0838420708305252 - 700 0.07 7.12179152897591e-07 0.5902614042958 0.590335413637884 2.51559188415894e-09 0.0838423375091767 - 800 0.08 2.20574733079126e-07 0.590262494529884 0.590336508799302 2.14455748236281e-10 0.0838424126463497 - 900 0.09 6.72564339365689e-08 0.590262805532644 0.590336821097688 1.84495767133404e-11 0.0838424338620728 - 1000 0.1 2.03001940390912e-08 0.590262894882646 0.590336910794094 1.5958531383517e-12 0.0838424398944951 -Loop time of 0.0617704 on 4 procs for 1000 steps with 200 atoms +Step Time v_magx v_magz v_magnorm v_tmag PotEng v_emag TotEng + 0 0 0.000143282585570239 0.615515043943419 0.615726279597251 24.9364200982478 0.121881906963737 0.121881906963737 0.121881906963737 + 100 0.01 0.000616167502203098 0.594467364025194 0.594498630048784 0.00188964439583802 0.0371335982020525 0.037133598202052 0.0371335982020525 + 200 0.02 0.000498981016106215 0.595175581059791 0.595218717456538 0.000158614984300385 0.0368772336480548 0.0368772336480543 0.0368772336480548 + 300 0.03 0.000211899815837572 0.595357874794342 0.595402442288391 1.44454891242177e-05 0.0368548794182373 0.0368548794182367 0.0368548794182373 + 400 0.04 7.9896757739716e-05 0.595395828381057 0.595440657806237 1.50721782707597e-06 0.0368527556548779 0.0368527556548773 0.0368527556548779 + 500 0.05 2.91216489141033e-05 0.595403174462525 0.595448064489507 1.74330474543395e-07 0.0368525254239537 0.0368525254239531 0.0368525254239537 + 600 0.06 1.04772320898493e-05 0.595404457003426 0.595449362424563 2.12204214498221e-08 0.036852498249274 0.0368524982492735 0.036852498249274 + 700 0.07 3.74634771616408e-06 0.595404627382825 0.59544953694064 2.63852407890464e-09 0.0368524949126259 0.0368524949126254 0.0368524949126259 + 800 0.08 1.33525617457997e-06 0.595404626884198 0.595449537611055 3.30772506699851e-10 0.0368524944963442 0.0368524944963437 0.0368524944963442 + 900 0.09 4.75054785504803e-07 0.595404613763238 0.595449524836571 4.15940445257144e-11 0.0368524944440915 0.036852494444091 0.0368524944440915 + 1000 0.1 1.68843135202462e-07 0.59540460640039 0.595449517580793 5.23632581178917e-12 0.0368524944375178 0.0368524944375172 0.0368524944375178 +Loop time of 0.041718 on 4 procs for 1000 steps with 200 atoms -98.7% CPU use with 4 MPI tasks x no OpenMP threads +97.4% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - 0.251043691627 0.0838424398641 0.0838424398945 + 0.121881906964 0.0368524944375 0.0368524944375 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.023753 | 0.029762 | 0.035936 | 3.3 | 48.18 +Pair | 0.020462 | 0.021665 | 0.024967 | 1.3 | 51.93 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.011783 | 0.019131 | 0.025404 | 4.3 | 30.97 -Output | 0.0019517 | 0.0019774 | 0.0020368 | 0.1 | 3.20 -Modify | 0.0006361 | 0.00087249 | 0.0011525 | 0.0 | 1.41 -Other | | 0.01003 | | | 16.23 +Comm | 0.0074174 | 0.010855 | 0.012527 | 2.0 | 26.02 +Output | 0.0012774 | 0.0012916 | 0.0013213 | 0.0 | 3.10 +Modify | 0.00056291 | 0.00068498 | 0.00096655 | 0.0 | 1.64 +Other | | 0.007222 | | | 17.31 Nlocal: 50 ave 50 max 50 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 similarity index 54% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 index 61b6ad8700..49a5f4dd3f 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00107217 secs + create_atoms CPU = 0.00207901 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,10 +62,10 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 0.0 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,30 +90,30 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 8.331 | 8.331 | 8.331 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.163834417271778 14831.3069413956 0 -0.0819172086358848 - 50 0.005 0.000106105812337003 -128.307447484203 0.104264818055985 0 -64.1537237421011 - 100 0.01 7.95347901119144e-06 -131.449389798071 0.0221943604064967 0 -65.7246948990356 - 150 0.015 5.63006161138875e-07 -132.296453030419 0.0085472877724348 0 -66.1482265152089 - 200 0.02 5.07390677383517e-07 -132.622857703805 0.00361380451198708 0 -66.3114288519012 - 250 0.025 3.28458336892231e-07 -132.774411992703 0.00187753161968493 0 -66.3872059963511 - 300 0.03 1.93294839202864e-07 -132.861283726084 0.00121374398924599 0 -66.4306418630428 - 350 0.035 1.13872157437693e-07 -132.923137019136 0.000954736871701507 0 -66.4615685095675 - 400 0.04 6.42075545620808e-08 -132.975239148591 0.000854064736183609 0 -66.4876195742954 - 450 0.045 3.44210513403008e-08 -133.023523287306 0.000812909459005007 0 -66.5117616436536 - 500 0.05 1.80394981485933e-08 -133.070071976252 0.000789742875305133 0 -66.5350359881254 - 550 0.055 9.54697157105863e-09 -133.11541233939 0.000769860218895372 0 -66.5577061696963 - 600 0.06 5.22455110720346e-09 -133.159676447906 0.000752941158466282 0 -66.5798382239526 - 650 0.065 2.95172977724016e-09 -133.203196195612 0.000745065216626277 0 -66.6015980978057 - 700 0.07 1.6727567441294e-09 -133.246696814329 0.000752898926000619 0 -66.6233484071653 - 750 0.075 9.17127001723567e-10 -133.291227007554 0.000780491405791262 0 -66.6456135037769 - 800 0.08 4.72669535949609e-10 -133.337962593396 0.000827942834401386 0 -66.6689812966976 - 850 0.085 2.25696738407094e-10 -133.387945245851 0.000890246383931885 0 -66.6939726229243 - 900 0.09 1.0030717061716e-10 -133.441737087546 0.000955403731484674 0 -66.720868543773 - 950 0.095 4.19867626359036e-11 -133.498969798312 0.00100352240545389 0 -66.7494848991554 - 1000 0.1 1.64283478182092e-11 -133.557979904763 0.00101162410316333 0 -66.7789899523816 -Loop time of 9.23057 on 1 procs for 1000 steps with 5780 atoms + 0 0 0.0100717228668283 -0.0819172086358897 14831.3069413956 0 -0.0819172086358848 + 50 0.005 0.000106105812337003 -64.1537237421016 0.104264818055985 0 -64.1537237421011 + 100 0.01 7.95347901119144e-06 -65.7246948990358 0.0221943604064967 0 -65.7246948990356 + 150 0.015 5.63006161138875e-07 -66.1482265152101 0.0085472877724348 0 -66.1482265152089 + 200 0.02 5.07390677383517e-07 -66.3114288519027 0.00361380451198708 0 -66.3114288519012 + 250 0.025 3.28458336892231e-07 -66.3872059963515 0.00187753161968493 0 -66.3872059963511 + 300 0.03 1.93294839202864e-07 -66.430641863042 0.00121374398924599 0 -66.4306418630428 + 350 0.035 1.13872157437693e-07 -66.461568509568 0.000954736871701507 0 -66.4615685095675 + 400 0.04 6.42075545620808e-08 -66.4876195742958 0.000854064736183609 0 -66.4876195742954 + 450 0.045 3.44210513403008e-08 -66.5117616436528 0.000812909459005007 0 -66.5117616436536 + 500 0.05 1.80394981485933e-08 -66.535035988126 0.000789742875305133 0 -66.5350359881254 + 550 0.055 9.54697157105863e-09 -66.5577061696949 0.000769860218895372 0 -66.5577061696963 + 600 0.06 5.22455110720346e-09 -66.5798382239523 0.000752941158466282 0 -66.5798382239526 + 650 0.065 2.95172977724016e-09 -66.6015980978057 0.000745065216626277 0 -66.6015980978057 + 700 0.07 1.6727567441294e-09 -66.6233484071647 0.000752898926000619 0 -66.6233484071653 + 750 0.075 9.17127001723567e-10 -66.6456135037774 0.000780491405791262 0 -66.6456135037769 + 800 0.08 4.72669535949609e-10 -66.6689812966982 0.000827942834401386 0 -66.6689812966976 + 850 0.085 2.25696738407094e-10 -66.6939726229253 0.000890246383931885 0 -66.6939726229243 + 900 0.09 1.0030717061716e-10 -66.7208685437728 0.000955403731484674 0 -66.720868543773 + 950 0.095 4.19867626359036e-11 -66.7494848991559 0.00100352240545389 0 -66.7494848991554 + 1000 0.1 1.64283478182092e-11 -66.778989952382 0.00101162410316333 0 -66.7789899523816 +Loop time of 8.81793 on 1 procs for 1000 steps with 5780 atoms -99.9% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -125,12 +127,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 8.7576 | 8.7576 | 8.7576 | 0.0 | 94.88 +Pair | 8.3388 | 8.3388 | 8.3388 | 0.0 | 94.57 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.023815 | 0.023815 | 0.023815 | 0.0 | 0.26 -Output | 0.31665 | 0.31665 | 0.31665 | 0.0 | 3.43 -Modify | 0.029446 | 0.029446 | 0.029446 | 0.0 | 0.32 -Other | | 0.1031 | | | 1.12 +Comm | 0.023951 | 0.023951 | 0.023951 | 0.0 | 0.27 +Output | 0.3172 | 0.3172 | 0.3172 | 0.0 | 3.60 +Modify | 0.034405 | 0.034405 | 0.034405 | 0.0 | 0.39 +Other | | 0.1036 | | | 1.17 Nlocal: 5780 ave 5780 max 5780 min Histogram: 1 0 0 0 0 0 0 0 0 0 @@ -145,4 +147,4 @@ Total # of neighbors = 92480 Ave neighs/atom = 16 Neighbor list builds = 0 Dangerous builds = 0 -Total wall time: 0:00:09 +Total wall time: 0:00:08 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 similarity index 51% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 index efc4628b9e..6ffa5eb861 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00102711 secs + create_atoms CPU = 0.00117207 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,10 +62,10 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 0.0 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,35 +90,35 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.788 | 7.788 | 7.788 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.162177519662199 14970.709092345 0 -0.157514482753586 - 50 0.005 0.000134061770654521 -128.226118665465 0.102634444037434 0 -128.286182424672 - 100 0.01 7.67769618983817e-06 -131.374599259781 0.0222596977749883 0 -131.428418504308 - 150 0.015 6.02904602224806e-07 -132.224372015825 0.00974271828169071 0 -132.273190134602 - 200 0.02 6.50197247050491e-07 -132.573383315469 0.00374227079785921 0 -132.617565541034 - 250 0.025 4.4053438575152e-07 -132.729743470508 0.0019334097282578 0 -132.770567114743 - 300 0.03 2.78356316513274e-07 -132.819255077939 0.00124938353773497 0 -132.857574876413 - 350 0.035 1.79684785125388e-07 -132.882714312877 0.000973166792896165 0 -132.919261229742 - 400 0.04 1.10949878459078e-07 -132.935357748213 0.000852955460997588 0 -132.970786605995 - 450 0.045 6.49064465617817e-08 -132.982991683198 0.000790741148426224 0 -133.017887798927 - 500 0.05 3.70514666559952e-08 -133.027689959766 0.00074794913288275 0 -133.06256199189 - 550 0.055 2.12433814830885e-08 -133.070148920145 0.00071263732127117 0 -133.105417593745 - 600 0.06 1.24676590171361e-08 -133.110772798503 0.000685051841817325 0 -133.14676746928 - 650 0.065 7.53611859129351e-09 -133.150126417754 0.000669443562813208 0 -133.187094895708 - 700 0.07 4.63539338651321e-09 -133.189024073453 0.000669619853917953 0 -133.227152349439 - 750 0.075 2.82145833974835e-09 -133.22844627026 0.000687338035086961 0 -133.267881315198 - 800 0.08 1.64378151566173e-09 -133.269413776733 0.000722197692175127 0 -133.310284062463 - 850 0.085 8.883310104497e-10 -133.312863108454 0.000771645398804486 0 -133.355293578462 - 900 0.09 4.33874801863461e-10 -133.359507749172 0.000830255722998156 0 -133.403626236688 - 950 0.095 1.8812784924272e-10 -133.409630495316 0.000888348219681112 0 -133.455560507802 - 1000 0.1 7.17748875671948e-11 -133.462806227865 0.000931427722404681 0 -133.510640942679 -Loop time of 2.53419 on 4 procs for 1000 steps with 5780 atoms + 0 0 0.0100717228668283 -0.0819172086358889 14831.3069413956 0 -0.0819172086358937 + 50 0.005 0.000106105812337003 -64.1537237421014 0.104264818055985 0 -64.1537237421015 + 100 0.01 7.95347901119157e-06 -65.7246948990355 0.0221943604064966 0 -65.7246948990356 + 150 0.015 5.63006161138918e-07 -66.1482265152097 0.0085472877724348 0 -66.1482265152095 + 200 0.02 5.07390677383159e-07 -66.3114288519023 0.00361380451198709 0 -66.3114288519024 + 250 0.025 3.28458336892466e-07 -66.3872059963513 0.00187753161968492 0 -66.3872059963514 + 300 0.03 1.93294839202718e-07 -66.4306418630421 0.00121374398924599 0 -66.4306418630421 + 350 0.035 1.1387215743759e-07 -66.461568509568 0.00095473687170151 0 -66.4615685095675 + 400 0.04 6.42075545625297e-08 -66.4876195742954 0.000854064736183607 0 -66.4876195742956 + 450 0.045 3.44210513402635e-08 -66.5117616436528 0.000812909459005008 0 -66.5117616436531 + 500 0.05 1.80394981487191e-08 -66.5350359881262 0.000789742875305131 0 -66.5350359881262 + 550 0.055 9.54697157097866e-09 -66.5577061696951 0.000769860218895372 0 -66.5577061696951 + 600 0.06 5.22455110708009e-09 -66.5798382239529 0.000752941158466282 0 -66.5798382239528 + 650 0.065 2.95172977729162e-09 -66.601598097806 0.000745065216626281 0 -66.6015980978062 + 700 0.07 1.67275674393186e-09 -66.6233484071645 0.000752898926000615 0 -66.6233484071643 + 750 0.075 9.17127001716818e-10 -66.6456135037772 0.000780491405791264 0 -66.6456135037769 + 800 0.08 4.72669535990986e-10 -66.6689812966981 0.000827942834401387 0 -66.668981296698 + 850 0.085 2.25696738737671e-10 -66.6939726229253 0.000890246383931887 0 -66.6939726229253 + 900 0.09 1.00307170296094e-10 -66.7208685437728 0.000955403731484673 0 -66.7208685437728 + 950 0.095 4.19867626693269e-11 -66.7494848991562 0.00100352240545389 0 -66.7494848991562 + 1000 0.1 1.64283481770759e-11 -66.7789899523816 0.00101162410316333 0 -66.7789899523818 +Loop time of 2.42884 on 4 procs for 1000 steps with 5780 atoms -99.9% CPU use with 4 MPI tasks x no OpenMP threads +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.157514482754 -133.509516066 -133.510640943 + -0.0819172086359 -66.778399627 -66.7789899524 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -125,12 +127,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 2.2697 | 2.2995 | 2.3564 | 2.2 | 90.74 +Pair | 2.1221 | 2.164 | 2.2349 | 2.9 | 89.10 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.05711 | 0.11414 | 0.14368 | 10.1 | 4.50 -Output | 0.084883 | 0.084915 | 0.084985 | 0.0 | 3.35 -Modify | 0.0071826 | 0.0072024 | 0.0072234 | 0.0 | 0.28 -Other | | 0.02847 | | | 1.12 +Comm | 0.065814 | 0.13626 | 0.17868 | 11.7 | 5.61 +Output | 0.090046 | 0.090083 | 0.090171 | 0.0 | 3.71 +Modify | 0.0081615 | 0.0083774 | 0.008673 | 0.2 | 0.34 +Other | | 0.03009 | | | 1.24 Nlocal: 1445 ave 1445 max 1445 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 similarity index 65% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 index ffd244b146..cef54797eb 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00135589 secs + create_atoms CPU = 0.00267482 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,11 +61,11 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin/cg # min_modify line spin_none discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_cg.cpp:105) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,20 +90,20 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 8.331 | 8.331 | 8.331 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.163834417271778 14831.3069413956 0 -0.0819172086358848 - 100 0.01 8.80197005314557e-06 -132.80634639767 0.00226660536216922 0 -66.4031731988347 - 200 0.02 6.70903250218956e-06 -133.127078243354 0.00103783628361662 0 -66.563539121677 - 300 0.03 4.5381603452565e-06 -133.471076972345 0.00144833375067451 0 -66.7355384861738 - 400 0.04 9.04820921016732e-07 -133.767843456662 0.000682239601485924 0 -66.8839217283314 - 500 0.05 1.6866160174916e-06 -133.893922160731 0.00032462625992713 0 -66.946961080365 - 600 0.06 1.78038217785001e-06 -133.957222680701 0.000160730704849448 0 -66.9786113403509 - 700 0.07 1.49199057723078e-06 -133.987255887786 7.39864656758093e-05 0 -66.9936279438931 - 800 0.08 1.15173756711067e-06 -134.000921126053 3.33959465206462e-05 0 -67.0004605630278 - 900 0.09 8.48526364752965e-07 -134.007049858868 1.49345737358251e-05 0 -67.0035249294347 - 1000 0.1 6.10346492876059e-07 -134.009791515671 6.71648807105468e-06 0 -67.0048957578347 -Loop time of 9.4449 on 1 procs for 1000 steps with 5780 atoms + 0 0 0.0100717228668283 -0.0819172086358897 14831.3069413956 0 -0.0819172086358848 + 100 0.01 8.80197005314557e-06 -66.4031731988352 0.00226660536216922 0 -66.4031731988347 + 200 0.02 6.70903250218956e-06 -66.5635391216766 0.00103783628361662 0 -66.563539121677 + 300 0.03 4.5381603452565e-06 -66.7355384861724 0.00144833375067451 0 -66.7355384861738 + 400 0.04 9.04820921016732e-07 -66.8839217283316 0.000682239601485924 0 -66.8839217283314 + 500 0.05 1.6866160174916e-06 -66.9469610803664 0.00032462625992713 0 -66.946961080365 + 600 0.06 1.78038217785001e-06 -66.9786113403508 0.000160730704849448 0 -66.9786113403509 + 700 0.07 1.49199057723078e-06 -66.9936279438935 7.39864656758093e-05 0 -66.9936279438931 + 800 0.08 1.15173756711067e-06 -67.0004605630263 3.33959465206462e-05 0 -67.0004605630278 + 900 0.09 8.48526364752965e-07 -67.0035249294341 1.49345737358251e-05 0 -67.0035249294347 + 1000 0.1 6.10346492876059e-07 -67.004895757836 6.71648807105468e-06 0 -67.0048957578347 +Loop time of 9.0061 on 1 procs for 1000 steps with 5780 atoms -100.0% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 8.7686 | 8.7686 | 8.7686 | 0.0 | 92.84 +Pair | 8.3237 | 8.3237 | 8.3237 | 0.0 | 92.42 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.02386 | 0.02386 | 0.02386 | 0.0 | 0.25 -Output | 0.31808 | 0.31808 | 0.31808 | 0.0 | 3.37 -Modify | 0.029416 | 0.029416 | 0.029416 | 0.0 | 0.31 -Other | | 0.305 | | | 3.23 +Comm | 0.023927 | 0.023927 | 0.023927 | 0.0 | 0.27 +Output | 0.31783 | 0.31783 | 0.31783 | 0.0 | 3.53 +Modify | 0.034633 | 0.034633 | 0.034633 | 0.0 | 0.38 +Other | | 0.306 | | | 3.40 Nlocal: 5780 ave 5780 max 5780 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 similarity index 62% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 index 730bf477d4..26dd99046b 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_cg.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_cg.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 19.8) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 5780 atoms - create_atoms CPU = 0.00138712 secs + create_atoms CPU = 0.000908136 secs # setting mass, mag. moments, and interactions for bcc iron @@ -59,11 +61,11 @@ dump 1 all custom 50 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[3 min_style spin/cg # min_modify line spin_none discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_cg.cpp:105) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -88,25 +90,25 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.788 | 7.788 | 7.788 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0100717228668283 -0.162177519662199 14970.709092345 0 -0.157514482753586 - 100 0.01 8.97646420936397e-06 -132.756468673032 0.00226858475243123 0 -132.79881239587 - 200 0.02 5.7049674439631e-06 -133.065966570145 0.000924384747875186 0 -133.105411060402 - 300 0.03 7.08166486348038e-06 -133.359072681024 0.00128114254070688 0 -133.406669528642 - 400 0.04 4.60224970353229e-06 -133.668643035703 0.000822334798448062 0 -133.725353643022 - 500 0.05 3.13737045264193e-06 -133.819548711647 0.000369678417461456 0 -133.878037514585 - 600 0.06 2.55239214469856e-06 -133.889302880669 0.0001696142482835 0 -133.948327309746 - 700 0.07 1.92236411979341e-06 -133.920147501261 7.31985644003847e-05 0 -133.979597440788 - 800 0.08 1.40879742055238e-06 -133.933445418833 3.19349095035109e-05 0 -133.993344750158 - 900 0.09 1.02629246257047e-06 -133.939321574068 1.44399877051467e-05 0 -133.999611147322 - 1000 0.1 7.52253147824893e-07 -133.942032102451 6.85789018963965e-06 0 -134.002604512509 -Loop time of 2.49676 on 4 procs for 1000 steps with 5780 atoms + 0 0 0.0100717228668283 -0.0819172086358889 14831.3069413956 0 -0.0819172086358937 + 100 0.01 8.80197005315463e-06 -66.403173198835 0.00226660536216922 0 -66.4031731988351 + 200 0.02 6.7090325022051e-06 -66.563539121677 0.00103783628361663 0 -66.5635391216769 + 300 0.03 4.53816034526952e-06 -66.7355384861727 0.00144833375067451 0 -66.7355384861724 + 400 0.04 9.04820921046463e-07 -66.8839217283311 0.000682239601485917 0 -66.883921728331 + 500 0.05 1.68661601751279e-06 -66.9469610803655 0.00032462625992713 0 -66.9469610803655 + 600 0.06 1.78038217786708e-06 -66.9786113403506 0.000160730704849447 0 -66.9786113403508 + 700 0.07 1.4919905772389e-06 -66.9936279438931 7.39864656758075e-05 0 -66.9936279438932 + 800 0.08 1.15173756711682e-06 -67.0004605630262 3.33959465206457e-05 0 -67.0004605630265 + 900 0.09 8.48526364761828e-07 -67.0035249294341 1.49345737358251e-05 0 -67.0035249294341 + 1000 0.1 6.10346492874848e-07 -67.0048957578355 6.71648807105468e-06 0 -67.0048957578356 +Loop time of 2.40179 on 4 procs for 1000 steps with 5780 atoms -100.0% CPU use with 4 MPI tasks x no OpenMP threads +100.0% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.157514482754 -134.00257032 -134.002604513 + -0.0819172086359 -67.0048809251 -67.0048957578 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -115,12 +117,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 2.2509 | 2.2589 | 2.2629 | 0.3 | 90.47 +Pair | 2.1174 | 2.1533 | 2.179 | 1.6 | 89.66 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.06036 | 0.064254 | 0.072356 | 1.9 | 2.57 -Output | 0.084002 | 0.085009 | 0.085985 | 0.3 | 3.40 -Modify | 0.0072496 | 0.0072694 | 0.0073116 | 0.0 | 0.29 -Other | | 0.08134 | | | 3.26 +Comm | 0.049849 | 0.075596 | 0.11152 | 8.4 | 3.15 +Output | 0.082974 | 0.084081 | 0.085145 | 0.3 | 3.50 +Modify | 0.0082345 | 0.008339 | 0.008424 | 0.1 | 0.35 +Other | | 0.08043 | | | 3.35 Nlocal: 1445 ave 1445 max 1445 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 similarity index 69% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 index a8685eb2cf..2d8b319b3c 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 3.96) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 1156 atoms - create_atoms CPU = 0.00136805 secs + create_atoms CPU = 0.00159788 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,11 +62,11 @@ min_style spin/lbfgs # min_modify line spin_cubic discrete_factor 10.0 min_modify norm max minimize 1.0e-15 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_lbfgs.cpp:109) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -89,17 +91,17 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.748 | 7.748 | 7.748 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0205636053306396 -0.218504643888467 1537.40479337332 0 -0.109252321944233 - 50 0.005 0.000800557938107919 -19.2937186217138 0.293526226015746 0 -9.65918446070018 - 100 0.01 0.000434178067296136 -19.6225314972776 0.136842093090897 0 -9.81803976806459 - 150 0.015 9.48307628510239e-06 -19.7062424007137 0.000835148627123792 0 -9.85315267460932 - 200 0.02 9.40072944704056e-06 -19.7072931204571 7.72334770010361e-06 0 -9.85364693487844 - 250 0.025 5.05117500164935e-07 -19.7072952885901 5.72437821949831e-08 0 -9.85364764712939 - 300 0.03 2.15063977474981e-09 -19.707295295749 2.09970244523395e-12 0 -9.8536476478746 - 303 0.0303 1.43831710574092e-09 -19.7072952957498 1.70336397715489e-13 0 -9.85364764787493 -Loop time of 0.335897 on 1 procs for 303 steps with 1156 atoms + 0 0 0.0205636053306396 -0.109252321944233 1537.40479337332 0 -0.109252321944233 + 50 0.005 0.000800557938107919 -9.65918446070017 0.293526226015746 0 -9.65918446070018 + 100 0.01 0.000434178067296136 -9.81803976806455 0.136842093090897 0 -9.81803976806459 + 150 0.015 9.48307628510239e-06 -9.85315267460926 0.000835148627123792 0 -9.85315267460932 + 200 0.02 9.40072944704056e-06 -9.85364693487847 7.72334770010361e-06 0 -9.85364693487844 + 250 0.025 5.05117500164935e-07 -9.85364764712935 5.72437821949831e-08 0 -9.85364764712939 + 300 0.03 2.15063977474981e-09 -9.85364764787459 2.09970244523395e-12 0 -9.8536476478746 + 303 0.0303 1.43831710574092e-09 -9.85364764787493 1.70336397715489e-13 0 -9.85364764787493 +Loop time of 0.328641 on 1 procs for 303 steps with 1156 atoms -100.0% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = energy tolerance @@ -113,12 +115,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.28129 | 0.28129 | 0.28129 | 0.0 | 83.74 +Pair | 0.27332 | 0.27332 | 0.27332 | 0.0 | 83.17 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0010796 | 0.0010796 | 0.0010796 | 0.0 | 0.32 -Output | 0.017942 | 0.017942 | 0.017942 | 0.0 | 5.34 -Modify | 0.001771 | 0.001771 | 0.001771 | 0.0 | 0.53 -Other | | 0.03382 | | | 10.07 +Comm | 0.001117 | 0.001117 | 0.001117 | 0.0 | 0.34 +Output | 0.018038 | 0.018038 | 0.018038 | 0.0 | 5.49 +Modify | 0.0020421 | 0.0020421 | 0.0020421 | 0.0 | 0.62 +Other | | 0.03412 | | | 10.38 Nlocal: 1156 ave 1156 max 1156 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 similarity index 66% rename from examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 index f2756d0f5e..2c26f64b56 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.bfo_min_lbfgs.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.bfo_min_lbfgs.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bfo in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (134.64 134.64 3.96) 2 by 2 by 1 MPI processor grid create_atoms 1 box Created 1156 atoms - create_atoms CPU = 0.000981808 secs + create_atoms CPU = 0.00098896 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,11 +62,11 @@ min_style spin/lbfgs # min_modify line spin_cubic discrete_factor 10.0 min_modify norm max minimize 1.0e-15 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) WARNING: Line search incompatible gneb (../min_spin_lbfgs.cpp:109) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -89,36 +91,36 @@ Neighbor list info ... bin: none Per MPI rank memory allocation (min/avg/max) = 7.619 | 7.619 | 7.619 Mbytes Step Time v_magnorm v_emag v_tmag Temp TotEng - 0 0 0.0205636053306396 -0.217760509274282 1541.29975585882 0 -0.217230771393012 - 50 0.005 0.000966655616837406 -19.2878369426356 0.312860071233841 0 -19.3229939390148 - 100 0.01 0.00154452800191107 -19.5948898197917 0.365367666925029 0 -19.6389064900413 - 150 0.015 4.89955946750017e-05 -19.6962580067431 0.000385536538802408 0 -19.7047140195852 - 200 0.02 5.66300530875654e-05 -19.6975252647309 9.8679922880911e-05 0 -19.7059140354146 - 250 0.025 5.21141123128679e-08 -19.6975359469038 2.52554968535685e-09 0 -19.7059189333986 - 300 0.03 2.9845103782958e-09 -19.6975359475094 2.31782597655471e-11 0 -19.7059191124033 - 342 0.0342 1.0526549233076e-10 -19.6975359475123 3.65641352240487e-16 0 -19.7059191178145 -Loop time of 0.117672 on 4 procs for 342 steps with 1156 atoms + 0 0 0.0205636053306396 -0.109252321944234 1537.40479337332 0 -0.109252321944234 + 50 0.005 0.00080055793810568 -9.65918446070018 0.293526226015742 0 -9.65918446070018 + 100 0.01 0.000434178089299568 -9.81803976800935 0.136842091062742 0 -9.81803976800936 + 150 0.015 9.48305696976556e-06 -9.85315267463944 0.000835124161214539 0 -9.85315267463943 + 200 0.02 9.48404729540188e-06 -9.85364693589303 7.83955637621785e-06 0 -9.85364693589302 + 250 0.025 2.53129638591035e-07 -9.85364764689316 5.38044599466511e-08 0 -9.85364764689315 + 300 0.03 1.70353573452093e-08 -9.85364764787448 5.65371929906417e-11 0 -9.85364764787448 + 318 0.0318 1.78606698438076e-09 -9.85364764787558 5.01580212144594e-14 0 -9.85364764787559 +Loop time of 0.107177 on 4 procs for 318 steps with 1156 atoms -99.4% CPU use with 4 MPI tasks x no OpenMP threads +99.6% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: - Stopping criterion = force tolerance + Stopping criterion = energy tolerance Energy initial, next-to-last, final = - -0.217230771393 -19.7059191178 -19.7059191178 + -0.109252321944 -9.85364764788 -9.85364764788 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 - Iterations, force evaluations = 342 342 + Iterations, force evaluations = 318 318 MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.084558 | 0.086668 | 0.091471 | 1.0 | 73.65 +Pair | 0.074155 | 0.077992 | 0.081876 | 1.2 | 72.77 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0052197 | 0.010042 | 0.012191 | 2.8 | 8.53 -Output | 0.0050647 | 0.0050726 | 0.0050921 | 0.0 | 4.31 -Modify | 0.00052595 | 0.00053537 | 0.00055242 | 0.0 | 0.45 -Other | | 0.01535 | | | 13.05 +Comm | 0.005573 | 0.0094561 | 0.013285 | 3.4 | 8.82 +Output | 0.005054 | 0.0050632 | 0.0050869 | 0.0 | 4.72 +Modify | 0.00055051 | 0.00058657 | 0.00066018 | 0.0 | 0.55 +Other | | 0.01408 | | | 13.14 Nlocal: 289 ave 289 max 289 min Histogram: 4 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 similarity index 87% rename from examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 rename to examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 index 6df5959c0b..7fe5920c1c 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.1 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.1 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 1 by 1 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000965834 secs + create_atoms CPU = 0.00106812 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,8 +62,8 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -87,9 +89,9 @@ Step Time v_magx v_magz v_magnorm v_tmag TotEng 800 0.08 -0.584017381477007 -0.0766850043611195 0.999999999999999 6.70634191991825e-06 -25.289508192325 900 0.09 -0.583812040722351 -0.0811143180675364 0.999999999999999 7.49814943594148e-06 -25.2895256658925 1000 0.1 -0.583582389243979 -0.0857958823565731 0.999999999999998 8.38204259112222e-06 -25.2895452009133 -Loop time of 0.195254 on 1 procs for 1000 steps with 250 atoms +Loop time of 0.199799 on 1 procs for 1000 steps with 250 atoms -100.0% CPU use with 1 MPI tasks x no OpenMP threads +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations @@ -103,12 +105,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.17668 | 0.17668 | 0.17668 | 0.0 | 90.49 +Pair | 0.1795 | 0.1795 | 0.1795 | 0.0 | 89.84 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.0052176 | 0.0052176 | 0.0052176 | 0.0 | 2.67 -Output | 0.0067751 | 0.0067751 | 0.0067751 | 0.0 | 3.47 -Modify | 0.0013788 | 0.0013788 | 0.0013788 | 0.0 | 0.71 -Other | | 0.005203 | | | 2.66 +Comm | 0.0056038 | 0.0056038 | 0.0056038 | 0.0 | 2.80 +Output | 0.0074773 | 0.0074773 | 0.0074773 | 0.0 | 3.74 +Modify | 0.0016823 | 0.0016823 | 0.0016823 | 0.0 | 0.84 +Other | | 0.005533 | | | 2.77 Nlocal: 250 ave 250 max 250 min Histogram: 1 0 0 0 0 0 0 0 0 0 diff --git a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 similarity index 72% rename from examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 rename to examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 index b0cdd4f94a..0eb7bd1783 100644 --- a/examples/SPIN/spinmin/log.19Nov19.spin.iron_min.g++.4 +++ b/examples/SPIN/spinmin/log.14Apr20.spin.iron_min.g++.4 @@ -1,4 +1,6 @@ -LAMMPS (30 Oct 2019) +LAMMPS (19 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (../comm.cpp:94) + using 1 OpenMP thread(s) per MPI task # bcc iron in a 3d periodic box units metal @@ -17,7 +19,7 @@ Created orthogonal box = (0 0 0) to (14.3325 14.3325 14.3325) 1 by 2 by 2 MPI processor grid create_atoms 1 box Created 250 atoms - create_atoms CPU = 0.000759125 secs + create_atoms CPU = 0.000731945 secs # setting mass, mag. moments, and interactions for bcc iron @@ -60,8 +62,8 @@ dump 1 all custom 100 dump.lammpstrj type x y z c_outsp[1] c_outsp[2] c_outsp[ min_style spin min_modify alpha_damp 1.0 discrete_factor 10.0 minimize 1.0e-10 1.0e-10 1000 100 -WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:177) -WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:87) +WARNING: Using 'neigh_modify every 1 delay 0 check yes' setting during minimization (../min.cpp:190) +WARNING: Using spin pair style without nve/spin or neb/spin (../pair_spin.cpp:88) Neighbor list info ... update every 1 steps, delay 0 steps, check yes max neighbors/atom: 2000, page size: 100000 @@ -76,25 +78,25 @@ Neighbor list info ... bin: standard Per MPI rank memory allocation (min/avg/max) = 6.828 | 6.829 | 6.829 Mbytes Step Time v_magx v_magz v_magnorm v_tmag TotEng - 0 0 -0.0285071136621457 -0.00948990815281275 0.0764569750905723 5048.5607623768 -0.701465876910695 - 100 0.01 -0.584953861980204 -0.0517163256267969 0.999992350892306 6.25556948778489e-05 -50.5787443620229 - 200 0.02 -0.584864756506845 -0.0547143484057154 0.999999990495506 3.49782260454051e-06 -50.5787971409246 - 300 0.03 -0.5847600493607 -0.0578846348986585 0.999999999988173 3.83095226804998e-06 -50.5788061208592 - 400 0.04 -0.584642875238891 -0.06123730753627 0.999999999999984 4.28575832708228e-06 -50.5788161053499 - 500 0.05 -0.584511765589526 -0.0647826190376232 0.999999999999999 4.79421486949061e-06 -50.5788272748473 - 600 0.06 -0.584365074206158 -0.0685313536438759 0.999999999999999 5.36242072641826e-06 -50.5788397688148 - 700 0.07 -0.584200963215272 -0.0724948469588718 1 5.99725249459218e-06 -50.5788537427249 - 800 0.08 -0.584017381477007 -0.0766850043611196 1 6.7063419199184e-06 -50.5788693699014 - 900 0.09 -0.583812040722352 -0.0811143180675365 0.999999999999998 7.49814943594153e-06 -50.5788868434688 - 1000 0.1 -0.583582389243979 -0.0857958823565732 0.999999999999999 8.38204259112203e-06 -50.5789063784897 -Loop time of 0.0845464 on 4 procs for 1000 steps with 250 atoms + 0 0 -0.0285071136621457 -0.00948990815281275 0.0764569750905723 5048.5607623768 -0.354774619362399 + 100 0.01 -0.584953861980204 -0.0517163256267969 0.999992350892306 6.25556948778489e-05 -25.2894057771132 + 200 0.02 -0.584864756506845 -0.0547143484057154 0.999999990495506 3.49782260454051e-06 -25.2894359914181 + 300 0.03 -0.5847600493607 -0.0578846348986585 0.999999999988173 3.83095226804998e-06 -25.2894449433168 + 400 0.04 -0.584642875238891 -0.06123730753627 0.999999999999984 4.28575832708228e-06 -25.2894549277729 + 500 0.05 -0.584511765589526 -0.0647826190376232 0.999999999999999 4.79421486949061e-06 -25.2894660972702 + 600 0.06 -0.584365074206158 -0.0685313536438759 0.999999999999999 5.36242072641826e-06 -25.2894785912378 + 700 0.07 -0.584200963215272 -0.0724948469588718 1 5.99725249459218e-06 -25.2894925651479 + 800 0.08 -0.584017381477007 -0.0766850043611196 1 6.7063419199184e-06 -25.2895081923244 + 900 0.09 -0.583812040722352 -0.0811143180675365 0.999999999999998 7.49814943594153e-06 -25.2895256658919 + 1000 0.1 -0.583582389243979 -0.0857958823565732 0.999999999999999 8.38204259112203e-06 -25.2895452009127 +Loop time of 0.086317 on 4 procs for 1000 steps with 250 atoms -99.8% CPU use with 4 MPI tasks x no OpenMP threads +99.9% CPU use with 4 MPI tasks x 1 OpenMP threads Minimization stats: Stopping criterion = max iterations Energy initial, next-to-last, final = - -0.701465876911 -50.5789061722 -50.5789063785 + -0.354774619362 -25.2895449946 -25.2895452009 Force two-norm initial, final = 0 0 Force max component initial, final = 0 0 Final line search alpha, max atom move = 0 0 @@ -103,12 +105,12 @@ Minimization stats: MPI task timing breakdown: Section | min time | avg time | max time |%varavg| %total --------------------------------------------------------------- -Pair | 0.043007 | 0.045307 | 0.04776 | 0.8 | 53.59 +Pair | 0.042744 | 0.045964 | 0.049416 | 1.2 | 53.25 Neigh | 0 | 0 | 0 | 0.0 | 0.00 -Comm | 0.026827 | 0.029139 | 0.031438 | 1.0 | 34.47 -Output | 0.0023198 | 0.0023302 | 0.0023572 | 0.0 | 2.76 -Modify | 0.00041103 | 0.0004673 | 0.00054026 | 0.0 | 0.55 -Other | | 0.007303 | | | 8.64 +Comm | 0.026905 | 0.030113 | 0.033464 | 1.5 | 34.89 +Output | 0.0023746 | 0.0023909 | 0.0024335 | 0.1 | 2.77 +Modify | 0.00052047 | 0.00055218 | 0.00058579 | 0.0 | 0.64 +Other | | 0.007297 | | | 8.45 Nlocal: 62.5 ave 65 max 60 min Histogram: 2 0 0 0 0 0 0 0 0 2 diff --git a/examples/SPIN/test_problems/README b/examples/SPIN/test_problems/README index 5557e3d42b..0a1362ec9c 100644 --- a/examples/SPIN/test_problems/README +++ b/examples/SPIN/test_problems/README @@ -1,4 +1,4 @@ -** The objective of the benchmark examples in this directory +** The objective of the test problems in this directory is the following twofold: - verify the implementation of the LAMMPS' SPIN package by comparing its results to well-known analytic results, or @@ -6,39 +6,39 @@ - provide users with simple comparisons, allowing them to better understand what is implemented in the code. -The LAMMPS input file (bench-*) can be modified, as well as the +The LAMMPS input file (test-*) can be modified, as well as the associated python script, in order to try different comparisons. All scripts can be run by executing the shell script from its directory. Example: -./run-bench-exchange.sh from the benchmarck_damped_exchange/ +./run-test-exchange.sh from the validation_damped_exchange/ directory. -** Below a brief description of the different benchmark +** Below a brief description of the different validation problems: -- benchmarck_damped_precession: +- validation_damped_precession: simulates the damped precession of a single spin in a magnetic field. - Run as: ./run-bench-prec.sh + Run as: ./run-test-prec.sh Output: x, y and z components of the magnetization, and magnetic energy. -- benchmarck_damped_exchange: +- validation_damped_exchange: simulates two spins interacting through the exchange interaction. The parameters in the LAMMPS input script - (bench-spin-precession.in) are calibrated to match the + (test-spin-precession.in) are calibrated to match the exchange definition in the python script (llg_exchange.py). - Run as: ./run-bench-exchange.sh + Run as: ./run-test-exchange.sh Output: average magnetization resulting from the damped precession of the two interacting spins. Also plots the evolution of the magnetic energy. -- benchmarck_langevin_precession: +- validation_langevin_precession: simulates a single spin in a magnetic field and in contact with a thermal bath, and compares the statistical averages of the output to the analytical result of the Langevin function. - Run as: ./run-bench-prec.sh + Run as: ./run-test-prec.sh Output: statistical average of the z-component of the magnetization (along the applied field) and of the magnetic energy versus temperature. Comparison to the Langevin function diff --git a/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py b/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py index dd1c543bb3..49eecb5b44 100755 --- a/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py +++ b/examples/SPIN/test_problems/validation_damped_exchange/llg_exchange.py @@ -65,6 +65,6 @@ for t in range (0,N): # calc. average magnetization Sm = (S1+S2)*0.5 # calc. energy - en = -J0*(np.dot(S1,S2)) + en = -2.0*J0*(np.dot(S1,S2)) # print res. in ps for comparison with LAMMPS print(t*dt/1000.0,Sm[0],Sm[1],Sm[2],en) diff --git a/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh b/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh index 15de8d350e..599730fe7b 100755 --- a/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh +++ b/examples/SPIN/test_problems/validation_damped_exchange/run-test-exchange.sh @@ -5,7 +5,7 @@ rm res_*.dat # compute Lammps ./../../../../src/lmp_serial \ - -in bench-spin-precession.in + -in test-spin-precession.in in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" in="$(echo "$in+1" | bc -l)" diff --git a/examples/SPIN/test_problems/validation_damped_exchange/bench-spin-precession.in b/examples/SPIN/test_problems/validation_damped_exchange/test-spin-precession.in similarity index 100% rename from examples/SPIN/test_problems/validation_damped_exchange/bench-spin-precession.in rename to examples/SPIN/test_problems/validation_damped_exchange/test-spin-precession.in diff --git a/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data b/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data new file mode 100644 index 0000000000..013f813751 --- /dev/null +++ b/examples/SPIN/test_problems/validation_damped_exchange/two_spins.data @@ -0,0 +1,22 @@ +LAMMPS data file via write_data, version 19 Sep 2019, timestep = 0 + +2 atoms +1 atom types + +0.0 6.0 xlo xhi +0.0 3.0 ylo yhi +0.0 3.0 zlo zhi + +Masses + +1 1 + +Atoms # spin + +1 1 2.0 0.0 0.0 0.0 1.0 0.0 0.0 0 0 0 +2 1 2.0 3.0 0.0 0.0 0.0 1.0 0.0 0 0 0 + +Velocities + +1 0.0 0.0 0.0 +2 0.0 0.0 0.0 diff --git a/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh b/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh index 49ebc2ac4e..8609179376 100755 --- a/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh +++ b/examples/SPIN/test_problems/validation_damped_precession/run-test-prec.sh @@ -5,7 +5,7 @@ rm res_*.dat # compute Lammps ./../../../../src/lmp_serial \ - -in bench-spin-precession.in + -in test-spin-precession.in in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" in="$(echo "$in+1" | bc -l)" @@ -13,7 +13,7 @@ en="$(echo "$en-$in" | bc -l)" tail -n +$in log.lammps | head -n $en > res_lammps.dat # compute Langevin -python3 -m llg_precession.py > res_llg.dat +python3 llg_precession.py > res_llg.dat # plot results -python3 -m plot_precession.py res_lammps.dat res_llg.dat +python3 plot_precession.py res_lammps.dat res_llg.dat diff --git a/examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in b/examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in similarity index 90% rename from examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in rename to examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in index ed8a5caeaf..b8e943e579 100644 --- a/examples/SPIN/test_problems/validation_damped_precession/bench-spin-precession.in +++ b/examples/SPIN/test_problems/validation_damped_precession/test-spin-precession.in @@ -3,7 +3,7 @@ units metal atom_style spin atom_modify map array -boundary p p p +boundary f f f # read_data singlespin.data @@ -26,6 +26,7 @@ variable Temperature equal 0.0 variable Nsteps equal 500000 fix 1 all nve/spin lattice no +# fix 2 all precession/spin zeeman ${H} 0.0 0.0 1.0 anisotropy ${Kan} 0.0 0.0 1.0 fix 2 all precession/spin zeeman ${H} 0.0 0.0 1.0 anisotropy ${Kan} 0.0 0.0 1.0 fix_modify 2 energy yes fix 3 all langevin/spin ${Temperature} 0.01 12345 diff --git a/examples/SPIN/test_problems/validation_langevin_precession/langevin.py b/examples/SPIN/test_problems/validation_langevin_precession/langevin.py index 7acb780143..0e9b634eeb 100755 --- a/examples/SPIN/test_problems/validation_langevin_precession/langevin.py +++ b/examples/SPIN/test_problems/validation_langevin_precession/langevin.py @@ -1,12 +1,8 @@ #!/usr/bin/env python3 -#Program fitting the exchange interaction -#Model curve: Bethe-Slater function import numpy as np, pylab, tkinter import matplotlib.pyplot as plt import mpmath as mp -# from scipy.optimize import curve_fit -# from decimal import * mub=5.78901e-5 # Bohr magneton (eV/T) kb=8.617333262145e-5 # Boltzman constant (eV/K) @@ -24,4 +20,3 @@ tf=20.0 for i in range (0,npoints): temp=ti+i*(tf-ti)/npoints print('%lf %lf %lf' % (temp,func(temp),-g*mub*Hz*func(temp))) - diff --git a/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh b/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh index 98fceeca95..bcdd337903 100755 --- a/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh +++ b/examples/SPIN/test_problems/validation_langevin_precession/run-test-prec.sh @@ -10,10 +10,9 @@ N=20 for (( i=0; i<$N; i++ )) do temp="$(echo "$tempi+$i*($tempf-$tempi)/$N" | bc -l)" - sed s/temperature/${temp}/g bench-prec-spin.template > \ - bench-prec-spin.in - ./../../../../src/lmp_serial \ - -in bench-prec-spin.in + sed s/temperature/${temp}/g test-prec-spin.template > \ + test-prec-spin.in + ./../../../../src/lmp_serial -in test-prec-spin.in Hz="$(tail -n 1 average_spin | awk -F " " '{print $3}')" sz="$(tail -n 1 average_spin | awk -F " " '{print $5}')" en="$(tail -n 1 average_spin | awk -F " " '{print $6}')" @@ -21,7 +20,7 @@ do done # compute Langevin -python3 -m langevin.py > res_langevin.dat +python3 langevin.py > res_langevin.dat # plot results -python3 -m plot_precession.py res_lammps.dat res_langevin.dat +python3 plot_precession.py res_lammps.dat res_langevin.dat diff --git a/examples/SPIN/test_problems/validation_langevin_precession/bench-prec-spin.template b/examples/SPIN/test_problems/validation_langevin_precession/test-prec-spin.template similarity index 100% rename from examples/SPIN/test_problems/validation_langevin_precession/bench-prec-spin.template rename to examples/SPIN/test_problems/validation_langevin_precession/test-prec-spin.template diff --git a/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy b/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy new file mode 100644 index 0000000000..69231bb7ee --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/Fe_Mishin2006.eam.alloy @@ -0,0 +1,15009 @@ +comment 1 +comment 2 +Converted by Ganga P Purja Pun using C++ code on Mon Nov 3 10:48:17 2014 +1 Fe +5001 2.400000000000000e-03 5001 1.134673400048920e-03 5.673367000244601e+00 +26 5.584700000000000e+01 2.855300000000000e+00 BCC + 0.000000000000000e+00 + -2.338738741480766e-02 + -4.628214468925276e-02 + -6.912258036387915e-02 + -9.175603963501618e-02 + -1.141497214000000e-01 + -1.363388222824136e-01 + -1.583226111166723e-01 + -1.800965752913192e-01 + -2.016645989796093e-01 + -2.230289901000000e-01 + -2.441907391820846e-01 + -2.651515164004249e-01 + -2.859130554412839e-01 + -3.064769176965011e-01 + -3.268446844000000e-01 + -3.470179531091855e-01 + -3.669982968636285e-01 + -3.867872784635140e-01 + -4.063864535839295e-01 + -4.257973671999999e-01 + -4.450215551034667e-01 + -4.640605423684923e-01 + -4.829158454463851e-01 + -5.015889707095635e-01 + -5.200814146000000e-01 + -5.383946650297390e-01 + -5.565301991603658e-01 + -5.744894857268537e-01 + -5.922739837155686e-01 + -6.098851423000000e-01 + -6.273244022645037e-01 + -6.445931939756846e-01 + -6.616929394780281e-01 + -6.786250511352716e-01 + -6.953909318999999e-01 + -7.119919765952718e-01 + -7.284295696432279e-01 + -7.447050873484842e-01 + -7.608198966551055e-01 + -7.767753551999997e-01 + -7.925728126189590e-01 + -8.082136084644670e-01 + -8.236990743647939e-01 + -8.390305327768360e-01 + -8.542092970000001e-01 + -8.692366724924486e-01 + -8.841139549244775e-01 + -8.988424321942350e-01 + -9.134233831702263e-01 + -9.278580778000000e-01 + -9.421477783833053e-01 + -9.562937376266863e-01 + -9.702972006149804e-01 + -9.841594035897939e-01 + -9.978815741999999e-01 + -1.011464932626877e+00 + -1.024910689374227e+00 + -1.038220047492379e+00 + -1.051394201530088e+00 + -1.064434338000000e+00 + -1.077341636148692e+00 + -1.090117264995124e+00 + -1.102762386491716e+00 + -1.115278154872840e+00 + -1.127665716000000e+00 + -1.139926208397942e+00 + -1.152060761127338e+00 + -1.164070496370367e+00 + -1.175956528594255e+00 + -1.187719964000000e+00 + -1.199361901614307e+00 + -1.210883431375302e+00 + -1.222285636497208e+00 + -1.233569592577916e+00 + -1.244736367000000e+00 + -1.255787020138112e+00 + -1.266722603828963e+00 + -1.277544163022812e+00 + -1.288252734706093e+00 + -1.298849349000000e+00 + -1.309335029515858e+00 + -1.319710789766606e+00 + -1.329977637188551e+00 + -1.340136572985217e+00 + -1.350188590000000e+00 + -1.360134674013246e+00 + -1.369975802739515e+00 + -1.379712947700534e+00 + -1.389347073262624e+00 + -1.398879136000000e+00 + -1.408310086019347e+00 + -1.417640866058629e+00 + -1.426872412303706e+00 + -1.436005653062347e+00 + -1.445041510000000e+00 + -1.453980898788701e+00 + -1.462824726507764e+00 + -1.471573894410746e+00 + -1.480229297407323e+00 + -1.488791823000000e+00 + -1.497262352264216e+00 + -1.505641758273629e+00 + -1.513930908641532e+00 + -1.522130664940848e+00 + -1.530241881000000e+00 + -1.538265404312065e+00 + -1.546202075663511e+00 + -1.554052729949587e+00 + -1.561818194964572e+00 + -1.569499292000000e+00 + -1.577096836740392e+00 + -1.584611637631811e+00 + -1.592044497451960e+00 + -1.599396212413425e+00 + -1.606667572000000e+00 + -1.613859360067553e+00 + -1.620972353850301e+00 + -1.628007324820493e+00 + -1.634965037585305e+00 + -1.641846251000000e+00 + -1.648651718746855e+00 + -1.655382187120556e+00 + -1.662038397074428e+00 + -1.668621083574689e+00 + -1.675130975000000e+00 + -1.681568794320005e+00 + -1.687935258528783e+00 + -1.694231079461267e+00 + -1.700456962682860e+00 + -1.706613607000000e+00 + -1.712701705964896e+00 + -1.718721948142722e+00 + -1.724675016499694e+00 + -1.730561586828580e+00 + -1.736382330000000e+00 + -1.742137912457411e+00 + -1.747828994017656e+00 + -1.753456229026646e+00 + -1.759020265526974e+00 + -1.764521747000000e+00 + -1.769961312537754e+00 + -1.775339594027171e+00 + -1.780657218386706e+00 + -1.785914807192779e+00 + -1.791112977000000e+00 + -1.796252339747382e+00 + -1.801333500673453e+00 + -1.806357060360666e+00 + -1.811323614328498e+00 + -1.816233753000000e+00 + -1.821088062150410e+00 + -1.825887120985276e+00 + -1.830631504346673e+00 + -1.835321782397329e+00 + -1.839958520000000e+00 + -1.844542277375300e+00 + -1.849073608812209e+00 + -1.853553064572349e+00 + -1.857981190434394e+00 + -1.862358526000000e+00 + -1.866685606181999e+00 + -1.870962962067843e+00 + -1.875191119963197e+00 + -1.879370599902195e+00 + -1.883501918000000e+00 + -1.887585586736770e+00 + -1.891622112755978e+00 + -1.895611998485404e+00 + -1.899555741651499e+00 + -1.903453835000000e+00 + -1.907306767129976e+00 + -1.911115021853308e+00 + -1.914879078883785e+00 + -1.918599413067872e+00 + -1.922276495000000e+00 + -1.925910791456493e+00 + -1.929502763824010e+00 + -1.933052869645757e+00 + -1.936561562187136e+00 + -1.940029290000000e+00 + -1.943456497684402e+00 + -1.946843625242175e+00 + -1.950191109032846e+00 + -1.953499381145042e+00 + -1.956768869000000e+00 + -1.959999996258605e+00 + -1.963193182622893e+00 + -1.966348843806269e+00 + -1.969467390558660e+00 + -1.972549230000000e+00 + -1.975594766012145e+00 + -1.978604397775412e+00 + -1.981578520793180e+00 + -1.984517526364655e+00 + -1.987421802000000e+00 + -1.990291731861874e+00 + -1.993127695559656e+00 + -1.995930069364364e+00 + -1.998699225767272e+00 + -2.001435533000000e+00 + -2.004139355858378e+00 + -2.006811055538481e+00 + -2.009450989763890e+00 + -2.012059511961222e+00 + -2.014636972000000e+00 + -2.017183716742461e+00 + -2.019700089231340e+00 + -2.022186429001382e+00 + -2.024643071393187e+00 + -2.027070349000000e+00 + -2.029468591629637e+00 + -2.031838124095820e+00 + -2.034179268100215e+00 + -2.036492342201070e+00 + -2.038777662000000e+00 + -2.041035540156417e+00 + -2.043266284490031e+00 + -2.045470200083453e+00 + -2.047647589335177e+00 + -2.049798751000000e+00 + -2.051923980684690e+00 + -2.054023570214561e+00 + -2.056097808968247e+00 + -2.058146983649728e+00 + -2.060171377000000e+00 + -2.062171268551674e+00 + -2.064146934640815e+00 + -2.066098649159696e+00 + -2.068026683134947e+00 + -2.069931304000000e+00 + -2.071812776276246e+00 + -2.073671361471072e+00 + -2.075507318519566e+00 + -2.077320903301032e+00 + -2.079112369000000e+00 + -2.080881966139512e+00 + -2.082629940879466e+00 + -2.084356537486872e+00 + -2.086061998635378e+00 + -2.087746563000000e+00 + -2.089410466229110e+00 + -2.091053941828707e+00 + -2.092677220771137e+00 + -2.094280530723314e+00 + -2.095864097000000e+00 + -2.097428142794787e+00 + -2.098972888139794e+00 + -2.100498550462023e+00 + -2.102005344216306e+00 + -2.103493482000000e+00 + -2.104963174393948e+00 + -2.106414628045970e+00 + -2.107848047779690e+00 + -2.109263636820768e+00 + -2.110661595000000e+00 + -2.112042119471926e+00 + -2.113405405101743e+00 + -2.114751645122076e+00 + -2.116081030815763e+00 + -2.117393750000000e+00 + -2.118689987937912e+00 + -2.119969928191390e+00 + -2.121233752504371e+00 + -2.122481640223532e+00 + -2.123713768000000e+00 + -2.124930310308645e+00 + -2.126131439345980e+00 + -2.127317325650607e+00 + -2.128488137848567e+00 + -2.129644042000000e+00 + -2.130785202020548e+00 + -2.131911779449632e+00 + -2.133023934312029e+00 + -2.134121824962966e+00 + -2.135205607000000e+00 + -2.136275433910872e+00 + -2.137331457489147e+00 + -2.138373827747866e+00 + -2.139402692469194e+00 + -2.140418197999999e+00 + -2.141420489142081e+00 + -2.142409707722756e+00 + -2.143385994030927e+00 + -2.144349486926974e+00 + -2.145300323000000e+00 + -2.146238637076898e+00 + -2.147164562479522e+00 + -2.148078231051947e+00 + -2.148979772720135e+00 + -2.149869315000000e+00 + -2.150746983835218e+00 + -2.151612904662093e+00 + -2.152467200947815e+00 + -2.153309993218032e+00 + -2.154141401000000e+00 + -2.154961542883784e+00 + -2.155770535619624e+00 + -2.156568494253713e+00 + -2.157355531798206e+00 + -2.158131760000000e+00 + -2.158897289576013e+00 + -2.159652229917454e+00 + -2.160396688379106e+00 + -2.161130769707277e+00 + -2.161854579000000e+00 + -2.162568220747540e+00 + -2.163271795520776e+00 + -2.163965403104795e+00 + -2.164649143391419e+00 + -2.165323114000000e+00 + -2.165987410839904e+00 + -2.166642128874188e+00 + -2.167287362019562e+00 + -2.167923202786971e+00 + -2.168549742000000e+00 + -2.169167069289456e+00 + -2.169775273557456e+00 + -2.170374442741976e+00 + -2.170964663390078e+00 + -2.171546020000000e+00 + -2.172118595836383e+00 + -2.172682474202094e+00 + -2.173237737093693e+00 + -2.173784464448991e+00 + -2.174322736000000e+00 + -2.174852630794634e+00 + -2.175374225221640e+00 + -2.175887595177988e+00 + -2.176392816678372e+00 + -2.176889964000000e+00 + -2.177379110090651e+00 + -2.177860327096639e+00 + -2.178333686466943e+00 + -2.178799258747679e+00 + -2.179257113000000e+00 + -2.179707317299531e+00 + -2.180149939409128e+00 + -2.180585046165010e+00 + -2.181012703042297e+00 + -2.181432975000000e+00 + -2.181845926509731e+00 + -2.182251621108717e+00 + -2.182650121032542e+00 + -2.183041486916871e+00 + -2.183425780000000e+00 + -2.183803061402426e+00 + -2.184173389857596e+00 + -2.184536823542827e+00 + -2.184893420600431e+00 + -2.185243238000000e+00 + -2.185586332004080e+00 + -2.185922759056265e+00 + -2.186252574500474e+00 + -2.186575831916728e+00 + -2.186892585000000e+00 + -2.187202887393400e+00 + -2.187506791832633e+00 + -2.187804350256584e+00 + -2.188095613705712e+00 + -2.188380633000000e+00 + -2.188659458643119e+00 + -2.188932140238827e+00 + -2.189198726845117e+00 + -2.189459267002233e+00 + -2.189713809000000e+00 + -2.189962400732430e+00 + -2.190205089044370e+00 + -2.190441920581333e+00 + -2.190672942002169e+00 + -2.190898199000000e+00 + -2.191117736662499e+00 + -2.191331600149922e+00 + -2.191539834109547e+00 + -2.191742482380937e+00 + -2.191939589000000e+00 + -2.192131197889608e+00 + -2.192317351831708e+00 + -2.192498093290244e+00 + -2.192673464653123e+00 + -2.192843508000000e+00 + -2.193008265149897e+00 + -2.193167777657277e+00 + -2.193322086821774e+00 + -2.193471233640668e+00 + -2.193615259000000e+00 + -2.193754203483761e+00 + -2.193888106693908e+00 + -2.194017008313546e+00 + -2.194140948496115e+00 + -2.194259967000000e+00 + -2.194374103230534e+00 + -2.194483396307522e+00 + -2.194587885225862e+00 + -2.194687608881331e+00 + -2.194782606000000e+00 + -2.194872915200472e+00 + -2.194958575082935e+00 + -2.195039623893670e+00 + -2.195116099394696e+00 + -2.195188040000000e+00 + -2.195255484352944e+00 + -2.195318470219860e+00 + -2.195377035223315e+00 + -2.195431217130485e+00 + -2.195481054000000e+00 + -2.195526583926446e+00 + -2.195567844428364e+00 + -2.195604873029958e+00 + -2.195637707516086e+00 + -2.195666386000000e+00 + -2.195690946583878e+00 + -2.195711426545218e+00 + -2.195727863658885e+00 + -2.195740296673043e+00 + -2.195748763000000e+00 + -2.195753299607926e+00 + -2.195753945356280e+00 + -2.195750739132009e+00 + -2.195743719093693e+00 + -2.195732923000000e+00 + -2.195718388771941e+00 + -2.195700155840199e+00 + -2.195678263244660e+00 + -2.195652748934943e+00 + -2.195623652000000e+00 + -2.195591012099144e+00 + -2.195554867993407e+00 + -2.195515258985693e+00 + -2.195472225489285e+00 + -2.195425807000000e+00 + -2.195376042760528e+00 + -2.195322973614639e+00 + -2.195266640267298e+00 + -2.195207082686241e+00 + -2.195144342000000e+00 + -2.195078459809379e+00 + -2.195009476453377e+00 + -2.194937433066797e+00 + -2.194862372466152e+00 + -2.194784337000000e+00 + -2.194703368615792e+00 + -2.194619509257747e+00 + -2.194532801771278e+00 + -2.194443290305841e+00 + -2.194351018000000e+00 + -2.194256027737437e+00 + -2.194158364279782e+00 + -2.194058072250755e+00 + -2.193955195446114e+00 + -2.193849779000000e+00 + -2.193741868885436e+00 + -2.193631510785602e+00 + -2.193518750557477e+00 + -2.193403634464252e+00 + -2.193286209000000e+00 + -2.193166521091908e+00 + -2.193044618884907e+00 + -2.192920550384463e+00 + -2.192794362988794e+00 + -2.192666105000000e+00 + -2.192535825071375e+00 + -2.192403570783236e+00 + -2.192269389306454e+00 + -2.192133327558461e+00 + -2.191995432000000e+00 + -2.191855748732582e+00 + -2.191714323538803e+00 + -2.191571201647935e+00 + -2.191426427571225e+00 + -2.191280046000000e+00 + -2.191132101498374e+00 + -2.190982637495000e+00 + -2.190831697104933e+00 + -2.190679323369808e+00 + -2.190525559000000e+00 + -2.190370446239990e+00 + -2.190214026313205e+00 + -2.190056340263544e+00 + -2.189897429222788e+00 + -2.189737334000000e+00 + -2.189576094894156e+00 + -2.189413751026334e+00 + -2.189250341662785e+00 + -2.189085906605225e+00 + -2.188920484000000e+00 + -2.188754111167896e+00 + -2.188586826517446e+00 + -2.188418667831656e+00 + -2.188249671505532e+00 + -2.188079874000000e+00 + -2.187909311655257e+00 + -2.187738019850499e+00 + -2.187566033827133e+00 + -2.187393388939138e+00 + -2.187220120000000e+00 + -2.187046261198758e+00 + -2.186871845656608e+00 + -2.186696906557450e+00 + -2.186521477499284e+00 + -2.186345591000000e+00 + -2.186169278929348e+00 + -2.185992573469870e+00 + -2.185815507881449e+00 + -2.185638116930091e+00 + -2.185460435000000e+00 + -2.185282496723147e+00 + -2.185104339046128e+00 + -2.184925999471230e+00 + -2.184747515559985e+00 + -2.184568926000000e+00 + -2.184390270419004e+00 + -2.184211589307456e+00 + -2.184032923638580e+00 + -2.183854314918697e+00 + -2.183675806000000e+00 + -2.183497440664645e+00 + -2.183319263062145e+00 + -2.183141318226586e+00 + -2.182963652427783e+00 + -2.182786312000000e+00 + -2.182609343803448e+00 + -2.182432796909005e+00 + -2.182256720850984e+00 + -2.182081165100584e+00 + -2.181906180000000e+00 + -2.181731816772078e+00 + -2.181558127901617e+00 + -2.181385166690271e+00 + -2.181212987245827e+00 + -2.181041644000000e+00 + -2.180871191888493e+00 + -2.180701687214938e+00 + -2.180533187286951e+00 + -2.180365750436408e+00 + -2.180199435000000e+00 + -2.180034299706266e+00 + -2.179870405073628e+00 + -2.179707812476292e+00 + -2.179546583956951e+00 + -2.179386782000000e+00 + -2.179228469569652e+00 + -2.179071710572415e+00 + -2.178916570209873e+00 + -2.178763115274171e+00 + -2.178611412000000e+00 + -2.178461526719817e+00 + -2.178313527960817e+00 + -2.178167485170301e+00 + -2.178023468394170e+00 + -2.177881548000000e+00 + -2.177741794911888e+00 + -2.177604281585152e+00 + -2.177469081134519e+00 + -2.177336267165003e+00 + -2.177205914000000e+00 + -2.177078096781969e+00 + -2.176952892179310e+00 + -2.176830377266735e+00 + -2.176710629289710e+00 + -2.176593727000000e+00 + -2.176479750089725e+00 + -2.176368778170715e+00 + -2.176260892035270e+00 + -2.176156174290115e+00 + -2.176054707000000e+00 + -2.175956572480315e+00 + -2.175861855847520e+00 + -2.175770642395214e+00 + -2.175683016800304e+00 + -2.175599066000000e+00 + -2.175518878125867e+00 + -2.175442540126260e+00 + -2.175370140365612e+00 + -2.175301769822078e+00 + -2.175237519000000e+00 + -2.175177478422601e+00 + -2.175121740498959e+00 + -2.175070398200953e+00 + -2.175023544771637e+00 + -2.174981275000000e+00 + -2.174943684570890e+00 + -2.174910868831887e+00 + -2.174882924320826e+00 + -2.174859949548785e+00 + -2.174842043000000e+00 + -2.174829303331104e+00 + -2.174821830410927e+00 + -2.174819725228089e+00 + -2.174823090027552e+00 + -2.174832027000000e+00 + -2.174846638699704e+00 + -2.174867029559900e+00 + -2.174893304738549e+00 + -2.174925569834485e+00 + -2.174963931000000e+00 + -2.175008495084293e+00 + -2.175059370394289e+00 + -2.175116665887380e+00 + -2.175180491027200e+00 + -2.175250956000000e+00 + -2.175328171805043e+00 + -2.175412250937077e+00 + -2.175503306357684e+00 + -2.175601451271103e+00 + -2.175706800000000e+00 + -2.175819467765500e+00 + -2.175939570549688e+00 + -2.176067225041086e+00 + -2.176202548787411e+00 + -2.176345660000000e+00 + -2.176496677644949e+00 + -2.176655722183164e+00 + -2.176822914601843e+00 + -2.176998376191533e+00 + -2.177182229000000e+00 + -2.177374595921169e+00 + -2.177575601341799e+00 + -2.177785370087074e+00 + -2.178004027196875e+00 + -2.178231699000000e+00 + -2.178468512862275e+00 + -2.178714597011325e+00 + -2.178970080062299e+00 + -2.179235091002944e+00 + -2.179509760000000e+00 + -2.179794218272507e+00 + -2.180088598277346e+00 + -2.180393032665419e+00 + -2.180707654074028e+00 + -2.181032597000000e+00 + -2.181367997183408e+00 + -2.181713990526695e+00 + -2.182070713375897e+00 + -2.182438302803596e+00 + -2.182816897000000e+00 + -2.183206635150288e+00 + -2.183607657686531e+00 + -2.184020105275964e+00 + -2.184444118610974e+00 + -2.184879840000000e+00 + -2.185327412889597e+00 + -2.185786981109685e+00 + -2.186258689137973e+00 + -2.186742682377725e+00 + -2.187239107000000e+00 + -2.187748109862624e+00 + -2.188269838801850e+00 + -2.188804442677439e+00 + -2.189352071523296e+00 + -2.189912875000000e+00 + -2.190487003093955e+00 + -2.191074608322116e+00 + -2.191675843787203e+00 + -2.192290862610885e+00 + -2.192919819000000e+00 + -2.193562867915730e+00 + -2.194220164552147e+00 + -2.194891865404003e+00 + -2.195578128862665e+00 + -2.196279113000000e+00 + -2.196994976008069e+00 + -2.197725877814248e+00 + -2.198471979076637e+00 + -2.199233441010543e+00 + -2.200010426000000e+00 + -2.200803097224683e+00 + -2.201611618154960e+00 + -2.202436152922896e+00 + -2.203276866629103e+00 + -2.204133924999999e+00 + -2.205007494076377e+00 + -2.205897739759803e+00 + -2.206804828380775e+00 + -2.207728927012060e+00 + -2.208670203000000e+00 + -2.209628824019662e+00 + -2.210604958519273e+00 + -2.211598775212799e+00 + -2.212610442959744e+00 + -2.213640131000000e+00 + -2.214688008997179e+00 + -2.215754247358006e+00 + -2.216839016713686e+00 + -2.217942487809252e+00 + -2.219064832000000e+00 + -2.220206221175054e+00 + -2.221366827800435e+00 + -2.222546824259503e+00 + -2.223746382727115e+00 + -2.224965677000000e+00 + -2.226204881636015e+00 + -2.227464170011377e+00 + -2.228743716360184e+00 + -2.230043696636770e+00 + -2.231364286000000e+00 + -2.232705659360148e+00 + -2.234067993079961e+00 + -2.235451464138931e+00 + -2.236856249856844e+00 + -2.238282527000000e+00 + -2.239730472524265e+00 + -2.241200265638654e+00 + -2.242692085287406e+00 + -2.244206109271373e+00 + -2.245742517000000e+00 + -2.247301488882056e+00 + -2.248883204910052e+00 + -2.250487845315945e+00 + -2.252115590911750e+00 + -2.253766623000000e+00 + -2.255441123360801e+00 + -2.257139274542631e+00 + -2.258861259219423e+00 + -2.260607260005651e+00 + -2.262377460000000e+00 + -2.264172042724291e+00 + -2.265991192137124e+00 + -2.267835092970031e+00 + -2.269703930922391e+00 + -2.271597891000000e+00 + -2.273517158096056e+00 + -2.275461918662951e+00 + -2.277432359442962e+00 + -2.279428667085979e+00 + -2.281451029000000e+00 + -2.283499633104788e+00 + -2.285574667389452e+00 + -2.287676320195930e+00 + -2.289804780398682e+00 + -2.291960237000000e+00 + -2.294142879454985e+00 + -2.296352898805551e+00 + -2.298590485768695e+00 + -2.300855830078550e+00 + -2.303149122999999e+00 + -2.305470556795997e+00 + -2.307820323545887e+00 + -2.310198615647936e+00 + -2.312605626084681e+00 + -2.315041548000000e+00 + -2.317506574745174e+00 + -2.320000900224320e+00 + -2.322524719122656e+00 + -2.325078227061991e+00 + -2.327661619000000e+00 + -2.330275089774933e+00 + -2.332918835677479e+00 + -2.335593053485966e+00 + -2.338297940187764e+00 + -2.341033693000000e+00 + -2.343800509406319e+00 + -2.346598587410773e+00 + -2.349428125303093e+00 + -2.352289321686241e+00 + -2.355182376000000e+00 + -2.358107488272472e+00 + -2.361064858786305e+00 + -2.364054687769198e+00 + -2.367077175375671e+00 + -2.370132523000000e+00 + -2.373220932737141e+00 + -2.376342606269360e+00 + -2.379497745810629e+00 + -2.382686554578887e+00 + -2.385909236000000e+00 + -2.389165993580208e+00 + -2.392457030846979e+00 + -2.395782552198058e+00 + -2.399142763298191e+00 + -2.402537869000000e+00 + -2.405968074028050e+00 + -2.409433584975734e+00 + -2.412934608682329e+00 + -2.416471351683178e+00 + -2.420044020999999e+00 + -2.423652824137149e+00 + -2.427297969206864e+00 + -2.430979664728804e+00 + -2.434698119632543e+00 + -2.438453543000000e+00 + -2.442246144168148e+00 + -2.446076133195298e+00 + -2.449943720571852e+00 + -2.453849117184546e+00 + -2.457792534000000e+00 + -2.461774182279110e+00 + -2.465794274341647e+00 + -2.469853022790889e+00 + -2.473950640299041e+00 + -2.478087340000000e+00 + -2.482263335430129e+00 + -2.486478840559030e+00 + -2.490734069661931e+00 + -2.495029237377275e+00 + -2.499364559000000e+00 + -2.503740250274207e+00 + -2.508156527124766e+00 + -2.512613605692700e+00 + -2.517111702474631e+00 + -2.521651035000000e+00 + -2.526231821279088e+00 + -2.530854278665131e+00 + -2.535518625205626e+00 + -2.540225080261870e+00 + -2.544973863000000e+00 + -2.549765192573217e+00 + -2.554599288888442e+00 + -2.559476372494764e+00 + -2.564396664604125e+00 + -2.569360386000000e+00 + -2.574367757513646e+00 + -2.579419001423863e+00 + -2.584514340338642e+00 + -2.589653996851589e+00 + -2.594838193999999e+00 + -2.600067155253186e+00 + -2.605341104662645e+00 + -2.610660266783872e+00 + -2.616024866689215e+00 + -2.621435129000000e+00 + -2.626891278432673e+00 + -2.632393541366548e+00 + -2.637942144571998e+00 + -2.643537314797249e+00 + -2.649179279000000e+00 + -2.654868264471787e+00 + -2.660604499292826e+00 + -2.666388211776204e+00 + -2.672219630341834e+00 + -2.678098984000000e+00 + -2.684026502306392e+00 + -2.690002415494505e+00 + -2.696026954017429e+00 + -2.702100348331385e+00 + -2.708222828000000e+00 + -2.714394620651795e+00 + -2.720615948464916e+00 + -2.726887030965477e+00 + -2.733208085290139e+00 + -2.739579324000000e+00 + -2.746000955816228e+00 + -2.752473185385865e+00 + -2.758996214641690e+00 + -2.765570242469367e+00 + -2.772195463000000e+00 + -2.778872066776275e+00 + -2.785600241574056e+00 + -2.792380171842594e+00 + -2.799212037606767e+00 + -2.806096015000000e+00 + -2.813032277129447e+00 + -2.820020994035244e+00 + -2.827062331911353e+00 + -2.834156451970738e+00 + -2.841303512999999e+00 + -2.848503671252426e+00 + -2.855757077985244e+00 + -2.863063881095023e+00 + -2.870424224890850e+00 + -2.877838249999999e+00 + -2.885306093928394e+00 + -2.892827890388094e+00 + -2.900403769651381e+00 + -2.908033857857018e+00 + -2.915718277999999e+00 + -2.923457150039019e+00 + -2.931250588977881e+00 + -2.939098706889091e+00 + -2.947001612872150e+00 + -2.954959411999998e+00 + -2.962972205810575e+00 + -2.971040091434060e+00 + -2.979163163083816e+00 + -2.987341511831155e+00 + -2.995575224999998e+00 + -3.003864386499012e+00 + -3.012209075546996e+00 + -3.020609368336417e+00 + -3.029065337835770e+00 + -3.037577052999999e+00 + -3.046144579520789e+00 + -3.054767979666142e+00 + -3.063447312238444e+00 + -3.072182631692820e+00 + -3.080973988999999e+00 + -3.089821432099915e+00 + -3.098725004829577e+00 + -3.107684748048404e+00 + -3.116700699185534e+00 + -3.125772890999999e+00 + -3.134901352715559e+00 + -3.144086110699870e+00 + -3.153327188100508e+00 + -3.162624603843408e+00 + -3.171978372999998e+00 + -3.181388507546933e+00 + -3.190855016073102e+00 + -3.200377903469057e+00 + -3.209957169965227e+00 + -3.219592812999998e+00 + -3.229284827363896e+00 + -3.239033203402719e+00 + -3.248837927724427e+00 + -3.258698982652799e+00 + -3.268616347999998e+00 + -3.278590000914418e+00 + -3.288619913401403e+00 + -3.298706054004673e+00 + -3.308848387625167e+00 + -3.319046875999998e+00 + -3.329301477934322e+00 + -3.339612147844228e+00 + -3.349978836592516e+00 + -3.360401490990458e+00 + -3.370880054999998e+00 + -3.381414469596773e+00 + -3.392004670266620e+00 + -3.402650589976440e+00 + -3.413352159446950e+00 + -3.424109303999998e+00 + -3.434921945057392e+00 + -3.445790001867386e+00 + -3.456713390096515e+00 + -3.467692020464062e+00 + -3.478725800999998e+00 + -3.489814636890736e+00 + -3.500958427789171e+00 + -3.512157070764700e+00 + -3.523410460589800e+00 + -3.534718486999998e+00 + -3.546081035880271e+00 + -3.557497990199610e+00 + -3.568969229634993e+00 + -3.580494629586285e+00 + -3.592074061999998e+00 + -3.603707395759104e+00 + -3.615394495446970e+00 + -3.627135222559569e+00 + -3.638929435115233e+00 + -3.650776987000000e+00 + -3.662677728742245e+00 + -3.674631507325805e+00 + -3.686638166579447e+00 + -3.698697546469576e+00 + -3.710809483000000e+00 + -3.722973808828209e+00 + -3.735190352595157e+00 + -3.747458940125227e+00 + -3.759779394010754e+00 + -3.772151532000000e+00 + -3.784575168342916e+00 + -3.797050115062026e+00 + -3.809576180292756e+00 + -3.822153166803174e+00 + -3.834780875000000e+00 + -3.847459102820189e+00 + -3.860187643203489e+00 + -3.872966285667494e+00 + -3.885794816080159e+00 + -3.898673017000000e+00 + -3.911600667911320e+00 + -3.924577543724543e+00 + -3.937603416269590e+00 + -3.950678053996523e+00 + -3.963801221000000e+00 + -3.976972678103186e+00 + -3.990192183512261e+00 + -4.003459491521890e+00 + -4.016774351164608e+00 + -4.030136509000000e+00 + -4.043545709147031e+00 + -4.057001691149858e+00 + -4.070504190840202e+00 + -4.084052939835804e+00 + -4.097647667000000e+00 + -4.111288098419088e+00 + -4.124973955231095e+00 + -4.138704955371648e+00 + -4.152480813397504e+00 + -4.166301240000000e+00 + -4.180165942572286e+00 + -4.194074624537170e+00 + -4.208026986370140e+00 + -4.222022725101974e+00 + -4.236061533000000e+00 + -4.250143098846027e+00 + -4.264267108996074e+00 + -4.278433246261039e+00 + -4.292641188627767e+00 + -4.306890611000000e+00 + -4.321181185386233e+00 + -4.335512579090955e+00 + -4.349884456391911e+00 + -4.364296478350024e+00 + -4.378748302000000e+00 + -4.393239580901771e+00 + -4.407769964394226e+00 + -4.422339098999587e+00 + -4.436946628138934e+00 + -4.451592191000000e+00 + -4.466275423153566e+00 + -4.480995955945297e+00 + -4.495753418053649e+00 + -4.510547435209278e+00 + -4.525377628000000e+00 + -4.540243613376725e+00 + -4.555145006346341e+00 + -4.570081418154680e+00 + -4.585052454714511e+00 + -4.600057719000000e+00 + -4.615096811360685e+00 + -4.630169328028963e+00 + -4.645274861878902e+00 + -4.660413001844071e+00 + -4.675583333000000e+00 + -4.690785437259527e+00 + -4.706018892955582e+00 + -4.721283275134317e+00 + -4.736578154793736e+00 + -4.751903099000000e+00 + -4.767257671719769e+00 + -4.782641433752779e+00 + -4.798053942304498e+00 + -4.813494749953581e+00 + -4.828963406000000e+00 + -4.844459456896403e+00 + -4.859982445123970e+00 + -4.875531909767719e+00 + -4.891107385884028e+00 + -4.906708405000000e+00 + -4.922334495555203e+00 + -4.937985181840751e+00 + -4.953659984814029e+00 + -4.969358421588884e+00 + -4.985080006000000e+00 + -5.000824248840914e+00 + -5.016590656340991e+00 + -5.032378731598668e+00 + -5.048187974277996e+00 + -5.064017880000000e+00 + -5.079867940973151e+00 + -5.095737645459726e+00 + -5.111626480823102e+00 + -5.127533934923204e+00 + -5.143459499999997e+00 + -5.159402672483111e+00 + -5.175362954369220e+00 + -5.191339851303958e+00 + -5.207332872891304e+00 + -5.223341533999998e+00 + -5.239365353688981e+00 + -5.255403855076862e+00 + -5.271456565155025e+00 + -5.287523015792912e+00 + -5.303602743999997e+00 + -5.319695290809813e+00 + -5.335800201177900e+00 + -5.351917023850607e+00 + -5.368045312413205e+00 + -5.384184625999997e+00 + -5.400334527792659e+00 + -5.416494583936292e+00 + -5.432664365069262e+00 + -5.448843447960337e+00 + -5.465031412999998e+00 + -5.481227843963791e+00 + -5.497432330198948e+00 + -5.513644465003656e+00 + -5.529863846001168e+00 + -5.546090074999998e+00 + -5.562322757794160e+00 + -5.578561506278419e+00 + -5.594805935740466e+00 + -5.611055664858841e+00 + -5.627310317999997e+00 + -5.643569523867812e+00 + -5.659832914603254e+00 + -5.676100126691991e+00 + -5.692370802379017e+00 + -5.708644587999997e+00 + -5.724921133351554e+00 + -5.741200092816319e+00 + -5.757481124931912e+00 + -5.773763893288205e+00 + -5.790048065999996e+00 + -5.806333315035348e+00 + -5.822619317227983e+00 + -5.838905753168464e+00 + -5.855192307810226e+00 + -5.871478670999998e+00 + -5.887764536642293e+00 + -5.904049603224381e+00 + -5.920333573291820e+00 + -5.936616154283948e+00 + -5.952897057999997e+00 + -5.969175999997380e+00 + -5.985452700712029e+00 + -6.001726884408804e+00 + -6.017998279848357e+00 + -6.034266620999998e+00 + -6.050531645917840e+00 + -6.066793096565948e+00 + -6.083050719154321e+00 + -6.099304265265417e+00 + -6.115553489999997e+00 + -6.131798152082848e+00 + -6.148038016850530e+00 + -6.164272852974367e+00 + -6.180502432221914e+00 + -6.196726531999996e+00 + -6.212944934149927e+00 + -6.229157424398349e+00 + -6.245363792906498e+00 + -6.261563835474968e+00 + -6.277757350999996e+00 + -6.293944141712862e+00 + -6.310124016459128e+00 + -6.326296787703612e+00 + -6.342462271423720e+00 + -6.358620288999997e+00 + -6.374770666045148e+00 + -6.390913231950829e+00 + -6.407047820441565e+00 + -6.423174270811367e+00 + -6.439292425999997e+00 + -6.455402132459349e+00 + -6.471503242529590e+00 + -6.487595612302881e+00 + -6.503679101824962e+00 + -6.519753575999998e+00 + -6.535818903908587e+00 + -6.551874959678351e+00 + -6.567921621040533e+00 + -6.583958769810407e+00 + -6.599986292999997e+00 + -6.616004081814032e+00 + -6.632012031732002e+00 + -6.648010042213294e+00 + -6.663998017633650e+00 + -6.679975866999997e+00 + -6.695943503153600e+00 + -6.711900843425355e+00 + -6.727847809100131e+00 + -6.743784326279565e+00 + -6.759710325999998e+00 + -6.775625743222601e+00 + -6.791530516957345e+00 + -6.807424590371784e+00 + -6.823307911868753e+00 + -6.839180434000000e+00 + -6.855042113095231e+00 + -6.870892911056689e+00 + -6.886732793343159e+00 + -6.902561729209182e+00 + -6.918379693000000e+00 + -6.934186663377454e+00 + -6.949982623974822e+00 + -6.965767562024019e+00 + -6.981541468861679e+00 + -6.997304341000000e+00 + -7.013056179149240e+00 + -7.028796988372566e+00 + -7.044526777567921e+00 + -7.060245560311449e+00 + -7.075953355000000e+00 + -7.091650184045992e+00 + -7.107336074531737e+00 + -7.123011057484204e+00 + -7.138675168614964e+00 + -7.154328448000000e+00 + -7.169970939523060e+00 + -7.185602692089082e+00 + -7.201223758621499e+00 + -7.216834196750369e+00 + -7.232434069000000e+00 + -7.248023441650831e+00 + -7.263602384451917e+00 + -7.279170972021925e+00 + -7.294729285389790e+00 + -7.310277408000000e+00 + -7.325815426345584e+00 + -7.341343434315181e+00 + -7.356861529294433e+00 + -7.372369811668458e+00 + -7.387868387000000e+00 + -7.403357365322682e+00 + -7.418836861854650e+00 + -7.434306995371617e+00 + -7.449767888578398e+00 + -7.465219669000000e+00 + -7.480662468362842e+00 + -7.496096423635060e+00 + -7.511521675509539e+00 + -7.526938368822738e+00 + -7.542346653000000e+00 + -7.557746681513402e+00 + -7.573138613106206e+00 + -7.588522610348394e+00 + -7.603898840111169e+00 + -7.619267474000000e+00 + -7.634628687556104e+00 + -7.649982660818226e+00 + -7.665329578157865e+00 + -7.680669629212444e+00 + -7.696003007000000e+00 + -7.711329908041110e+00 + -7.726650535322031e+00 + -7.741965095546557e+00 + -7.757273799081691e+00 + -7.772576861000000e+00 + -7.787874500482072e+00 + -7.803166941820082e+00 + -7.818454413357075e+00 + -7.833737148083599e+00 + -7.849015383000000e+00 + -7.864289358883056e+00 + -7.879559322305754e+00 + -7.894825523493435e+00 + -7.910088216491122e+00 + -7.925347660000000e+00 + -7.940604117030949e+00 + -7.955857856656654e+00 + -7.971109151088187e+00 + -7.986358275553978e+00 + -8.001605510999999e+00 + -8.016851143003514e+00 + -8.032095461580266e+00 + -8.047338760488806e+00 + -8.062581337981985e+00 + -8.077823497000001e+00 + -8.093065544394179e+00 + -8.108307791577566e+00 + -8.123550554205780e+00 + -8.138794153086785e+00 + -8.154038912999997e+00 + -8.169285162395173e+00 + -8.184533235306290e+00 + -8.199783469410583e+00 + -8.215036206329255e+00 + -8.230291792999997e+00 + -8.245550580528674e+00 + -8.260812923863309e+00 + -8.276079182235739e+00 + -8.291349720375887e+00 + -8.306624906999998e+00 + -8.321905114401627e+00 + -8.337190720129563e+00 + -8.352482105562956e+00 + -8.367779656388556e+00 + -8.383083762999997e+00 + -8.398394819954635e+00 + -8.413713227271362e+00 + -8.429039388338369e+00 + -8.444373710123081e+00 + -8.459716604999997e+00 + -8.475068489773328e+00 + -8.490429785807564e+00 + -8.505800918193746e+00 + -8.521182316459129e+00 + -8.536574414999997e+00 + -8.551977652264261e+00 + -8.567392471335332e+00 + -8.582819319266729e+00 + -8.598258647834163e+00 + -8.613710912999998e+00 + -8.629176574613023e+00 + -8.644656098315155e+00 + -8.660149953296175e+00 + -8.675658612460374e+00 + -8.691182553999997e+00 + -8.706722260346437e+00 + -8.722278218076596e+00 + -8.737850917931995e+00 + -8.753440855866357e+00 + -8.769048531999996e+00 + -8.784674450012169e+00 + -8.800319118245726e+00 + -8.815983049456063e+00 + -8.831666761752983e+00 + -8.847370776999997e+00 + -8.863095620572640e+00 + -8.878841823409489e+00 + -8.894609920288579e+00 + -8.910400450202564e+00 + -8.926213956999996e+00 + -8.942050988642462e+00 + -8.957912097948702e+00 + -8.973797841249739e+00 + -8.989708778929735e+00 + -9.005645476999996e+00 + -9.021608505891232e+00 + -9.037598440089864e+00 + -9.053615857732957e+00 + -9.069661341483132e+00 + -9.085735478999997e+00 + -9.101838862125330e+00 + -9.117972087538210e+00 + -9.134135755396374e+00 + -9.150330469883286e+00 + -9.166556840999997e+00 + -9.182815483079517e+00 + -9.199107013675079e+00 + -9.215432054583626e+00 + -9.231791233265957e+00 + -9.248185180999997e+00 + -9.264614532683632e+00 + -9.281079929109534e+00 + -9.297582014702897e+00 + -9.314121437654787e+00 + -9.330698850999996e+00 + -9.347314912003528e+00 + -9.363970283188971e+00 + -9.380665630745685e+00 + -9.397401624940878e+00 + -9.414178940999998e+00 + -9.430998258244742e+00 + -9.447860260503859e+00 + -9.464765635761319e+00 + -9.481715077019253e+00 + -9.498709280999996e+00 + -9.515748948037771e+00 + -9.532834784442173e+00 + -9.549967500513571e+00 + -9.567147810785697e+00 + -9.584376433999996e+00 + -9.601654092679016e+00 + -9.618981514566920e+00 + -9.636359431588877e+00 + -9.653788580476306e+00 + -9.671269701999998e+00 + -9.688803540653204e+00 + -9.706390846469759e+00 + -9.724032373152683e+00 + -9.741728878390186e+00 + -9.759481124999997e+00 + -9.777289880050315e+00 + -9.795155915261526e+00 + -9.813080006095444e+00 + -9.831062932433847e+00 + -9.849105478999997e+00 + -9.867208434586416e+00 + -9.885372592754736e+00 + -9.903598750780114e+00 + -9.921887710289466e+00 + -9.940240277999996e+00 + -9.958657264807959e+00 + -9.977139486156000e+00 + -9.995687760994512e+00 + -1.001430291248964e+01 + -1.003298577000000e+01 + -1.005173716741557e+01 + -1.007055794170109e+01 + -1.008944893313717e+01 + -1.010841098632109e+01 + -1.012744495000000e+01 + -1.014655167843811e+01 + -1.016573203809090e+01 + -1.018498689280856e+01 + -1.020431709913334e+01 + -1.022372353000000e+01 + -1.024320706932260e+01 + -1.026276860085199e+01 + -1.028240900753776e+01 + -1.030212917226508e+01 + -1.032192999000000e+01 + -1.034181236270672e+01 + -1.036177718920571e+01 + -1.038182537682789e+01 + -1.040195784652257e+01 + -1.042217551000000e+01 + -1.044247927800797e+01 + -1.046287008429113e+01 + -1.048334886197221e+01 + -1.050391653528098e+01 + -1.052457404000000e+01 + -1.054532232135862e+01 + -1.056616233045517e+01 + -1.058709501821704e+01 + -1.060812133414642e+01 + -1.062924224000000e+01 + -1.065045870478386e+01 + -1.067177169452023e+01 + -1.069318217826154e+01 + -1.071469113174534e+01 + -1.073629954000000e+01 + -1.075800839260036e+01 + -1.077981867489276e+01 + -1.080173138018267e+01 + -1.082374751517595e+01 + -1.084586808000000e+01 + -1.086809407471729e+01 + -1.089042651905600e+01 + -1.091286643223940e+01 + -1.093541482646511e+01 + -1.095807273000000e+01 + -1.098084118002027e+01 + -1.100372120627546e+01 + -1.102671384288809e+01 + -1.104982013415814e+01 + -1.107304113000000e+01 + -1.109637788366843e+01 + -1.111983144958023e+01 + -1.114340288661973e+01 + -1.116709326043168e+01 + -1.119090364000000e+01 + -1.121483509746141e+01 + -1.123888871051257e+01 + -1.126306556352589e+01 + -1.128736674843474e+01 + -1.131179335000000e+01 + -1.133634645465832e+01 + -1.136102717560643e+01 + -1.138583662099362e+01 + -1.141077588304154e+01 + -1.143584608000000e+01 + -1.146104834278586e+01 + -1.148638378227322e+01 + -1.151185351768562e+01 + -1.153745868860526e+01 + -1.156320043000000e+01 + -1.158907987623694e+01 + -1.161509817570602e+01 + -1.164125647703402e+01 + -1.166755592508495e+01 + -1.169399768000000e+01 + -1.172058291041322e+01 + -1.174731277861368e+01 + -1.177418844974534e+01 + -1.180121109657743e+01 + -1.182838189999999e+01 + -1.185570204579555e+01 + -1.188317272006268e+01 + -1.191079511332836e+01 + -1.193857042312863e+01 + -1.196649984999999e+01 + -1.199458459773775e+01 + -1.202282587700695e+01 + -1.205122490269986e+01 + -1.207978289369385e+01 + -1.210850107000000e+01 + -1.213738065440104e+01 + -1.216642287877588e+01 + -1.219562898067898e+01 + -1.222500020303083e+01 + -1.225453779000000e+01 + -1.228424298812801e+01 + -1.231411705144751e+01 + -1.234416123799292e+01 + -1.237437680934280e+01 + -1.240476502999999e+01 + -1.243532716862228e+01 + -1.246606450371358e+01 + -1.249697831672881e+01 + -1.252806989018327e+01 + -1.255934050999999e+01 + -1.259079146663310e+01 + -1.262242406008652e+01 + -1.265423959346900e+01 + -1.268623937120698e+01 + -1.271842469999999e+01 + -1.275079689157634e+01 + -1.278335727210730e+01 + -1.281610716656313e+01 + -1.284904789393346e+01 + -1.288218078999999e+01 + -1.291550719883721e+01 + -1.294902845305343e+01 + -1.298274589252790e+01 + -1.301666087289315e+01 + -1.305077474999999e+01 + -1.308508888000826e+01 + -1.311960462335401e+01 + -1.315432334734565e+01 + -1.318924642810263e+01 + -1.322437523999999e+01 + -1.325971115825294e+01 + -1.329525556797775e+01 + -1.333100986045822e+01 + -1.336697543256989e+01 + -1.340315367999999e+01 + -1.343954599979837e+01 + -1.347615379884262e+01 + -1.351297848978655e+01 + -1.355002149050285e+01 + -1.358728421999999e+01 + -1.362476809975143e+01 + -1.366247455932581e+01 + -1.370040503112116e+01 + -1.373856094968169e+01 + -1.377694375999999e+01 + -1.381555491216690e+01 + -1.385439585021465e+01 + -1.389346802391279e+01 + -1.393277289476898e+01 + -1.397231192999999e+01 + -1.401208659972945e+01 + -1.405209837359219e+01 + -1.409234872413346e+01 + -1.413283912961124e+01 + -1.417357107999999e+01 + -1.421454607102127e+01 + -1.425576559243092e+01 + -1.429723113890712e+01 + -1.433894421536596e+01 + -1.438090632999999e+01 + -1.442311899332400e+01 + -1.446558371903957e+01 + -1.450830202765583e+01 + -1.455127544879836e+01 + -1.459450550999999e+01 + -1.463799373973040e+01 + -1.468174167767078e+01 + -1.472575086722947e+01 + -1.477002285383887e+01 + -1.481455918999999e+01 + -1.485936143285468e+01 + -1.490443114031971e+01 + -1.494976987417127e+01 + -1.499537920245961e+01 + -1.504126069999999e+01 + -1.508741594534576e+01 + -1.513384651602358e+01 + -1.518055399754435e+01 + -1.522753998778076e+01 + -1.527480607999999e+01 + -1.532235386668537e+01 + -1.537018495189360e+01 + -1.541830094448208e+01 + -1.546670345694024e+01 + -1.551539410999999e+01 + -1.556437452805486e+01 + -1.561364632974880e+01 + -1.566321114034704e+01 + -1.571307059764629e+01 + -1.576322633999999e+01 + -1.581368000886133e+01 + -1.586443325932826e+01 + -1.591548774371123e+01 + -1.596684510633198e+01 + -1.601850700999999e+01 + -1.607047512763592e+01 + -1.612275112310131e+01 + -1.617533666634843e+01 + -1.622823344044972e+01 + -1.628144312999999e+01 + -1.633496741991010e+01 + -1.638880799548294e+01 + -1.644296655089035e+01 + -1.649744479372194e+01 + -1.655224442999999e+01 + -1.660736716535794e+01 + -1.666281471098534e+01 + -1.671858878335293e+01 + -1.677469110504862e+01 + -1.683112339999999e+01 + -1.688788739643611e+01 + -1.694498483759857e+01 + -1.700241746523662e+01 + -1.706018701398879e+01 + -1.711829522999999e+01 + -1.717674386844329e+01 + -1.723553468924849e+01 + -1.729466945458232e+01 + -1.735414992865347e+01 + -1.741397788000000e+01 + -1.747415508099178e+01 + -1.753468330867943e+01 + -1.759556434768274e+01 + -1.765679999203036e+01 + -1.771839203000000e+01 + -1.778034225038387e+01 + -1.784265246094084e+01 + -1.790532446959567e+01 + -1.796836007816754e+01 + -1.803176110000000e+01 + -1.809552935720682e+01 + -1.815966667575656e+01 + -1.822417488078262e+01 + -1.828905579597735e+01 + -1.835431126000000e+01 + -1.841994311958100e+01 + -1.848595321475806e+01 + -1.855234338946741e+01 + -1.861911549709020e+01 + -1.868627140000000e+01 + -1.875381296458075e+01 + -1.882174205218422e+01 + -1.889006052915166e+01 + -1.895877027181235e+01 + -1.902787316000000e+01 + -1.909737107797965e+01 + -1.916726592078253e+01 + -1.923755957952593e+01 + -1.930825393714747e+01 + -1.937935090000000e+01 + -1.945085238647674e+01 + -1.952276030080435e+01 + -1.959507655061933e+01 + -1.966780305499929e+01 + -1.974094174000000e+01 + -1.981449453618971e+01 + -1.988846337667131e+01 + -1.996285019538828e+01 + -2.003765692760933e+01 + -2.011288552000000e+01 + -2.018853792621815e+01 + -2.026461609876727e+01 + -2.034112199510926e+01 + -2.041805758075517e+01 + -2.049542482000000e+01 + -2.057322568006621e+01 + -2.065146214471047e+01 + -2.073013619484967e+01 + -2.080924980240138e+01 + -2.088880495999999e+01 + -2.096880367169853e+01 + -2.104924793164823e+01 + -2.113013973534757e+01 + -2.121148108557274e+01 + -2.129327399999999e+01 + -2.137552050224973e+01 + -2.145822260294134e+01 + -2.154138232011573e+01 + -2.162500168821614e+01 + -2.170908273999999e+01 + -2.179362750931509e+01 + -2.187863804236205e+01 + -2.196411638687395e+01 + -2.205006458870765e+01 + -2.213648469999998e+01 + -2.222337877888525e+01 + -2.231074889087196e+01 + -2.239859710699802e+01 + -2.248692550346576e+01 + -2.257573614999999e+01 + -2.266503111825121e+01 + -2.275481250533621e+01 + -2.284508240538831e+01 + -2.293584289979252e+01 + -2.302709608999998e+01 + -2.311884408797131e+01 + -2.321108899308483e+01 + -2.330383291176923e+01 + -2.339707796636268e+01 + -2.349082627999998e+01 + -2.358507997580010e+01 + -2.367984117841246e+01 + -2.377511201965127e+01 + -2.387089464198777e+01 + -2.396719118999998e+01 + -2.406400380890314e+01 + -2.416133464354705e+01 + -2.425918584868402e+01 + -2.435755959379285e+01 + -2.445645803999998e+01 + -2.455588334675619e+01 + -2.465583769169985e+01 + -2.475632325621871e+01 + -2.485734222078360e+01 + -2.495889676999998e+01 + -2.506098909313465e+01 + -2.516362138717989e+01 + -2.526679585042222e+01 + -2.537051468121954e+01 + -2.547478008999998e+01 + -2.557959429416147e+01 + -2.568495950782481e+01 + -2.579087794894506e+01 + -2.589735184307986e+01 + -2.600438342000000e+01 + -2.611197491414786e+01 + -2.622012856937007e+01 + -2.632884662779481e+01 + -2.643813132670684e+01 + -2.654798492000000e+01 + -2.665840967182968e+01 + -2.676940784333127e+01 + -2.688098169572292e+01 + -2.699313349264397e+01 + -2.710586551000000e+01 + -2.721918002998715e+01 + -2.733307932892397e+01 + -2.744756569229334e+01 + -2.756264142140602e+01 + -2.767830881000000e+01 + -2.779457014947893e+01 + -2.791142774510221e+01 + -2.802888390934165e+01 + -2.814694096000261e+01 + -2.826560121000000e+01 + -2.838486697334714e+01 + -2.850474058224100e+01 + -2.862522436986088e+01 + -2.874632066501767e+01 + -2.886803181000000e+01 + -2.899036015446983e+01 + -2.911330804180568e+01 + -2.923687781925054e+01 + -2.936107184331833e+01 + -2.948589248000000e+01 + -2.961134209910011e+01 + -2.973742306338333e+01 + -2.986413774439124e+01 + -2.999148852946843e+01 + -3.011947780000000e+01 + -3.024810793640462e+01 + -3.037738133418102e+01 + -3.050730039248281e+01 + -3.063786751048302e+01 + -3.076908508999995e+01 + -3.090095553801825e+01 + -3.103348127528379e+01 + -3.116666472242296e+01 + -3.130050829541340e+01 + -3.143501442000000e+01 + -3.157018552904646e+01 + -3.170602405762165e+01 + -3.184253244456109e+01 + -3.197971313442012e+01 + -3.211756857999995e+01 + -3.225610123724459e+01 + -3.239531355479052e+01 + -3.253520799120049e+01 + -3.267578702273956e+01 + -3.281705312000000e+01 + -3.295900875199519e+01 + -3.310165640007095e+01 + -3.324499854981137e+01 + -3.338903768896250e+01 + -3.353377630999994e+01 + -3.367921690900354e+01 + -3.382536198480008e+01 + -3.397221404252836e+01 + -3.411977559608952e+01 + -3.426804916000000e+01 + -3.441703725062087e+01 + -3.456674239208418e+01 + -3.471716711255165e+01 + -3.486831394366235e+01 + -3.502018541999994e+01 + -3.517278408030963e+01 + -3.532611247314257e+01 + -3.548017315006599e+01 + -3.563496866378715e+01 + -3.579050157000000e+01 + -3.594677442911821e+01 + -3.610378981290209e+01 + -3.626155029281482e+01 + -3.642005843692588e+01 + -3.657931682999995e+01 + -3.673932806450152e+01 + -3.690009471994529e+01 + -3.706161938636306e+01 + -3.722390467398576e+01 + -3.738695318000000e+01 + -3.755076749872251e+01 + -3.771535025114206e+01 + -3.788070405790332e+01 + -3.804683152981476e+01 + -3.821373528999994e+01 + -3.838141797088579e+01 + -3.854988220802454e+01 + -3.871913063690455e+01 + -3.888916589260551e+01 + -3.905999062000000e+01 + -3.923160747217621e+01 + -3.940401910958111e+01 + -3.957722819105772e+01 + -3.975123737141870e+01 + -3.992604931999993e+01 + -4.010166671506960e+01 + -4.027809223215413e+01 + -4.045532854769397e+01 + -4.063337834194039e+01 + -4.081224431000000e+01 + -4.099192915380458e+01 + -4.117243556512365e+01 + -4.135376624213356e+01 + -4.153592389680070e+01 + -4.171891123999993e+01 + -4.190273098404973e+01 + -4.208738585293229e+01 + -4.227287857304115e+01 + -4.245921187066173e+01 + -4.264638848000000e+01 + -4.283441114000928e+01 + -4.302328258823673e+01 + -4.321300556988346e+01 + -4.340358284230594e+01 + -4.359501715999993e+01 + -4.378731127802528e+01 + -4.398046796379431e+01 + -4.417448998677844e+01 + -4.436938011574802e+01 + -4.456514113000000e+01 + -4.476177581459606e+01 + -4.495928695037842e+01 + -4.515767732491430e+01 + -4.535694973791337e+01 + -4.555710698999992e+01 + -4.575815188265746e+01 + -4.596008722124420e+01 + -4.616291581815827e+01 + -4.636664049491562e+01 + -4.657126407000000e+01 + -4.677678936329625e+01 + -4.698321921012249e+01 + -4.719055644645270e+01 + -4.739880390447049e+01 + -4.760796442999992e+01 + -4.781804087601476e+01 + -4.802903608806237e+01 + -4.824095291782189e+01 + -4.845379422957746e+01 + -4.866756289000000e+01 + -4.888226176793614e+01 + -4.909789373759357e+01 + -4.931446167624577e+01 + -4.953196846445736e+01 + -4.975041698999991e+01 + -4.996981014490353e+01 + -5.019015082042799e+01 + -5.041144191483806e+01 + -5.063368633767777e+01 + -5.085688700000000e+01 + -5.108104681363623e+01 + -5.130616869222998e+01 + -5.153225555529192e+01 + -5.175931033093434e+01 + -5.198733594999992e+01 + -5.221633534632925e+01 + -5.244631146064819e+01 + -5.267726723814258e+01 + -5.290920562821528e+01 + -5.314212958000000e+01 + -5.337604204603855e+01 + -5.361094599406351e+01 + -5.384684439089011e+01 + -5.408374019738633e+01 + -5.432163638999991e+01 + -5.456053595406303e+01 + -5.480044186892415e+01 + -5.504135712069162e+01 + -5.528328470761400e+01 + -5.552622762000000e+01 + -5.577018884907901e+01 + -5.601517141284037e+01 + -5.626117832498171e+01 + -5.650821258404100e+01 + -5.675627720999991e+01 + -5.700537523542541e+01 + -5.725550968441112e+01 + -5.750668358361924e+01 + -5.775889996777224e+01 + -5.801216188000000e+01 + -5.826647236807500e+01 + -5.852183447842080e+01 + -5.877825126017432e+01 + -5.903572576827786e+01 + -5.929426106999990e+01 + -5.955386023803142e+01 + -5.981452633626228e+01 + -6.007626243395679e+01 + -6.033907161265666e+01 + -6.060295696000000e+01 + -6.086792156620618e+01 + -6.113396851889991e+01 + -6.140110091270849e+01 + -6.166932185413050e+01 + -6.193863444999990e+01 + -6.220904180836374e+01 + -6.248054704389698e+01 + -6.275315327303672e+01 + -6.302686361375484e+01 + -6.330168120000000e+01 + -6.357760917255472e+01 + -6.385465065869710e+01 + -6.413280879313753e+01 + -6.441208672717147e+01 + -6.469248761000000e+01 + -6.497401459207148e+01 + -6.525667083785252e+01 + -6.554045951020694e+01 + -6.582538376572784e+01 + -6.611144677999999e+01 + -6.639865173804421e+01 + -6.668700181225154e+01 + -6.697650018322166e+01 + -6.726715004865791e+01 + -6.755895460000001e+01 + -6.785191702754956e+01 + -6.814604053706709e+01 + -6.844132833746227e+01 + -6.873778363702108e+01 + -6.903540965000001e+01 + -6.933420959649558e+01 + -6.963418670433917e+01 + -6.993534420146086e+01 + -7.023768531400407e+01 + -7.054121327999999e+01 + -7.084593134500037e+01 + -7.115184275353549e+01 + -7.145895075432828e+01 + -7.176725860332733e+01 + -7.207676955999989e+01 + -7.238748688676215e+01 + -7.269941385037480e+01 + -7.301255372170849e+01 + -7.332690977697268e+01 + -7.364248530000000e+01 + -7.395928357816831e+01 + -7.427730789465394e+01 + -7.459656154154558e+01 + -7.491704782593285e+01 + -7.523877004999987e+01 + -7.556173151464178e+01 + -7.588593553169946e+01 + -7.621138541944168e+01 + -7.653808450163682e+01 + -7.686603610000000e+01 + -7.719524353805981e+01 + -7.752571015313293e+01 + -7.785743928572181e+01 + -7.819043427631229e+01 + -7.852469846999988e+01 + -7.886023521608357e+01 + -7.919704786882774e+01 + -7.953513978918475e+01 + -7.987451434652237e+01 + -8.021517491000000e+01 + -8.055712484895521e+01 + -8.090036753598372e+01 + -8.124490635250547e+01 + -8.159074469172391e+01 + -8.193788593999987e+01 + -8.228633348416081e+01 + -8.263609073313938e+01 + -8.298716109541151e+01 + -8.333954797103145e+01 + -8.369325477000000e+01 + -8.404828491028280e+01 + -8.440464181412719e+01 + -8.476232890793086e+01 + -8.512134962287996e+01 + -8.548170738999985e+01 + -8.584340564352934e+01 + -8.620644783199920e+01 + -8.657083740482393e+01 + -8.693657780824788e+01 + -8.730367250000000e+01 + -8.767212494371194e+01 + -8.804193859639375e+01 + -8.841311692593891e+01 + -8.878566341870939e+01 + -8.915958154999986e+01 + -8.953487479168578e+01 + -8.991154663471849e+01 + -9.028960057395298e+01 + -9.066904010318096e+01 + -9.104986872000001e+01 + -9.143208992686800e+01 + -9.181570723560402e+01 + -9.220072416114067e+01 + -9.258714421980083e+01 + -9.297497092999986e+01 + -9.336420781525349e+01 + -9.375485841426487e+01 + -9.414692626313253e+01 + -9.454041488965315e+01 + -9.493532784000000e+01 + -9.533166867059026e+01 + -9.572944092950100e+01 + -9.612864816948191e+01 + -9.652929395407540e+01 + -9.693138184999984e+01 + -9.733491542620831e+01 + -9.773989825487583e+01 + -9.814633391500892e+01 + -9.855422599474231e+01 + -9.896357808000000e+01 + -9.937439375553778e+01 + -9.978667660951464e+01 + -1.002004302498873e+02 + -1.006156583113964e+02 + -1.010323643999998e+02 + -1.014505521033422e+02 + -1.018702250167552e+02 + -1.022913867648504e+02 + -1.027140410123223e+02 + -1.031381914000000e+02 + -1.035638415489902e+02 + -1.039909950728156e+02 + -1.044196556182856e+02 + -1.048498268814360e+02 + -1.052815124999998e+02 + -1.057147160851634e+02 + -1.061494413040264e+02 + -1.065856918490972e+02 + -1.070234714266025e+02 + -1.074627837000000e+02 + -1.079036323073108e+02 + -1.083460208994117e+02 + -1.087899531790557e+02 + -1.092354329168266e+02 + -1.096824637999998e+02 + -1.101310494754678e+02 + -1.105811936593542e+02 + -1.110329000832090e+02 + -1.114861724712110e+02 + -1.119410145000000e+02 + -1.123974298499644e+02 + -1.128554223365672e+02 + -1.133149957276709e+02 + -1.137761536721920e+02 + -1.142388998999998e+02 + -1.147032381951767e+02 + -1.151691723223398e+02 + -1.156367060267053e+02 + -1.161058430340008e+02 + -1.165765871000000e+02 + -1.170489420136372e+02 + -1.175229116129598e+02 + -1.179984996644491e+02 + -1.184757098197581e+02 + -1.189545458999998e+02 + -1.194350118004852e+02 + -1.199171112501401e+02 + -1.204008480035419e+02 + -1.208862259161727e+02 + -1.213732488000000e+02 + -1.218619204419225e+02 + -1.223522446637547e+02 + -1.228442252993658e+02 + -1.233378661832514e+02 + -1.238331710999998e+02 + -1.243301438371392e+02 + -1.248287883218059e+02 + -1.253291084262349e+02 + -1.258311078947497e+02 + -1.263347906000000e+02 + -1.268401604692280e+02 + -1.273472212887372e+02 + -1.278559768940106e+02 + -1.283664312433523e+02 + -1.288785881999998e+02 + -1.293924515790011e+02 + -1.299080252758494e+02 + -1.304253131892150e+02 + -1.309443191941241e+02 + -1.314650472000000e+02 + -1.319875011267033e+02 + -1.325116848422910e+02 + -1.330376022476014e+02 + -1.335652573069860e+02 + -1.340946538999998e+02 + -1.346257958892211e+02 + -1.351586873010075e+02 + -1.356933320835031e+02 + -1.362297340185993e+02 + -1.367678971000000e+02 + -1.373078254089903e+02 + -1.378495227933144e+02 + -1.383929931342302e+02 + -1.389382404506380e+02 + -1.394852686999998e+02 + -1.400340818144768e+02 + -1.405846838138316e+02 + -1.411370786853171e+02 + -1.416912703410207e+02 + -1.422472628000000e+02 + -1.428050601118040e+02 + -1.433646661562381e+02 + -1.439260848884736e+02 + -1.444893204354731e+02 + -1.450543767999998e+02 + -1.456212579173106e+02 + -1.461899678121767e+02 + -1.467605105156440e+02 + -1.473328900351724e+02 + -1.479071104000000e+02 + -1.484831756560318e+02 + -1.490610898537698e+02 + -1.496408570188282e+02 + -1.502224811435522e+02 + -1.508059663000000e+02 + -1.513913165938295e+02 + -1.519785360517816e+02 + -1.525676287092765e+02 + -1.531585986433134e+02 + -1.537514498999998e+02 + -1.543461865304275e+02 + -1.549428126935839e+02 + -1.555413324987964e+02 + -1.561417499433451e+02 + -1.567440691000000e+02 + -1.573482940967384e+02 + -1.579544290618548e+02 + -1.585624781173004e+02 + -1.591724453723425e+02 + -1.597843348999998e+02 + -1.603981507727703e+02 + -1.610138971515657e+02 + -1.616315781879854e+02 + -1.622511979884414e+02 + -1.628727607000000e+02 + -1.634962704824068e+02 + -1.641217314319286e+02 + -1.647491476962519e+02 + -1.653785235159056e+02 + -1.660098629999998e+02 + -1.666431702132707e+02 + -1.672784494025457e+02 + -1.679157047805161e+02 + -1.685549404452619e+02 + -1.691961606000000e+02 + -1.698393694891200e+02 + -1.704845712252935e+02 + -1.711317699673001e+02 + -1.717809699904770e+02 + -1.724321754999998e+02 + -1.730853906636835e+02 + -1.737406197069920e+02 + -1.743978668390772e+02 + -1.750571362267288e+02 + -1.757184321000000e+02 + -1.763817587329360e+02 + -1.770471204015179e+02 + -1.777145213342737e+02 + -1.783839656916257e+02 + -1.790554576999997e+02 + -1.797290016348361e+02 + -1.804046017812607e+02 + -1.810822624074922e+02 + -1.817619877551305e+02 + -1.824437821000000e+02 + -1.831276497291853e+02 + -1.838135948809234e+02 + -1.845016218201234e+02 + -1.851917348679873e+02 + -1.858839382999998e+02 + -1.865782363787204e+02 + -1.872746334451072e+02 + -1.879731338116912e+02 + -1.886737417231232e+02 + -1.893764615000000e+02 + -1.900812975069574e+02 + -1.907882540726982e+02 + -1.914973354967214e+02 + -1.922085460544087e+02 + -1.929218900999998e+02 + -1.936373720194894e+02 + -1.943549961169706e+02 + -1.950747667123967e+02 + -1.957966881820183e+02 + -1.965207649000000e+02 + -1.972470012275908e+02 + -1.979754014928308e+02 + -1.987059700466389e+02 + -1.994387112870401e+02 + -2.001736295999997e+02 + -2.009107293657797e+02 + -2.016500149830146e+02 + -2.023914908231823e+02 + -2.031351612174878e+02 + -2.038810306000000e+02 + -2.046291034502658e+02 + -2.053793841544099e+02 + -2.061318770718700e+02 + -2.068865865659776e+02 + -2.076435170999997e+02 + -2.084026731731890e+02 + -2.091640591716557e+02 + -2.099276794995817e+02 + -2.106935386300690e+02 + -2.114616410000000e+02 + -2.122319910255027e+02 + -2.130045931490896e+02 + -2.137794518362718e+02 + -2.145565715736208e+02 + -2.153359567999997e+02 + -2.161176119510612e+02 + -2.169015415764713e+02 + -2.176877501734888e+02 + -2.184762421224458e+02 + -2.192670219000000e+02 + -2.200600940505141e+02 + -2.208554631128035e+02 + -2.216531335679009e+02 + -2.224531098207831e+02 + -2.232553963999997e+02 + -2.240599978997985e+02 + -2.248669188401317e+02 + -2.256761637092654e+02 + -2.264877369816284e+02 + -2.273016432000000e+02 + -2.281178869415660e+02 + -2.289364727411434e+02 + -2.297574051254879e+02 + -2.305806886273285e+02 + -2.314063277999997e+02 + -2.322343272067604e+02 + -2.330646913987978e+02 + -2.338974249389632e+02 + -2.347325324124834e+02 + -2.355700184000000e+02 + -2.364098874788788e+02 + -2.372521442301691e+02 + -2.380967932316097e+02 + -2.389438390563173e+02 + -2.397932862999997e+02 + -2.406451395699314e+02 + -2.414994034603918e+02 + -2.423560825773335e+02 + -2.432151815494090e+02 + -2.440767050000000e+02 + -2.449406575494712e+02 + -2.458070438246558e+02 + -2.466758684438366e+02 + -2.475471360143982e+02 + -2.484208511999996e+02 + -2.492970186823550e+02 + -2.501756430678903e+02 + -2.510567289807632e+02 + -2.519402811010791e+02 + -2.528263041000000e+02 + -2.537148026392507e+02 + -2.546057813786489e+02 + -2.554992449908618e+02 + -2.563951981686694e+02 + -2.572936455999996e+02 + -2.581945919701986e+02 + -2.590980419711499e+02 + -2.600040002856254e+02 + -2.609124715849065e+02 + -2.618234606000000e+02 + -2.627369720784054e+02 + -2.636530106775519e+02 + -2.645715811006028e+02 + -2.654926881465747e+02 + -2.664163364999997e+02 + -2.673425308135112e+02 + -2.682712759281322e+02 + -2.692025766270102e+02 + -2.701364375396030e+02 + -2.710728634000000e+02 + -2.720118590149941e+02 + -2.729534291763713e+02 + -2.738975786300543e+02 + -2.748443120676544e+02 + -2.757936342999997e+02 + -2.767455501787848e+02 + -2.777000643997414e+02 + -2.786571816989448e+02 + -2.796169069311608e+02 + -2.805792449000000e+02 + -2.815442003833780e+02 + -2.825117782146733e+02 + -2.834819832031981e+02 + -2.844548201023671e+02 + -2.854302936999997e+02 + -2.864084088239409e+02 + -2.873891703596309e+02 + -2.883725831483384e+02 + -2.893586519481404e+02 + -2.903473816000000e+02 + -2.913387769892773e+02 + -2.923328429461174e+02 + -2.933295843055886e+02 + -2.943290059327143e+02 + -2.953311126999996e+02 + -2.963359094688269e+02 + -2.973434010446480e+02 + -2.983535922946355e+02 + -2.993664881931061e+02 + -3.003820936000000e+02 + -3.014004133189056e+02 + -3.024214522481863e+02 + -3.034452153213495e+02 + -3.044717074825456e+02 + -3.055009335999997e+02 + -3.065328985120693e+02 + -3.075676071368449e+02 + -3.086050644279766e+02 + -3.096452753553112e+02 + -3.106882447999996e+02 + -3.117339776182681e+02 + -3.127824787990806e+02 + -3.138337533166528e+02 + -3.148878060732492e+02 + -3.159446419999996e+02 + -3.170042660531784e+02 + -3.180666831989414e+02 + -3.191318984100575e+02 + -3.201999166629245e+02 + -3.212707428999996e+02 + -3.223443820580361e+02 + -3.234208391391681e+02 + -3.245001191331626e+02 + -3.255822269890704e+02 + -3.266671677000000e+02 + -3.277549462905472e+02 + -3.288455677875165e+02 + -3.299390371822755e+02 + -3.310353594166140e+02 + -3.321345394999996e+02 + -3.332365824854958e+02 + -3.343414934139734e+02 + -3.354492773045120e+02 + -3.365599391517927e+02 + -3.376734840000000e+02 + -3.387899169180687e+02 + -3.399092429402671e+02 + -3.410314670886827e+02 + -3.421565943845668e+02 + -3.432846298999996e+02 + -3.444155787253185e+02 + -3.455494458929433e+02 + -3.466862364711880e+02 + -3.478259555971263e+02 + -3.489686083000000e+02 + -3.501141995805723e+02 + -3.512627346186799e+02 + -3.524142185495274e+02 + -3.535686563776662e+02 + -3.547260531999996e+02 + -3.558864141649047e+02 + -3.570497443588797e+02 + -3.582160488781844e+02 + -3.593853328562213e+02 + -3.605576014000000e+02 + -3.617328596203877e+02 + -3.629111127187760e+02 + -3.640923658296630e+02 + -3.652766239636252e+02 + -3.664638922999996e+02 + -3.676541760949210e+02 + -3.688474804479435e+02 + -3.700438104641370e+02 + -3.712431713192604e+02 + -3.724455682000000e+02 + -3.736510062917217e+02 + -3.748594907617601e+02 + -3.760710267615897e+02 + -3.772856194324323e+02 + -3.785032739999995e+02 + -3.797239957155368e+02 + -3.809477897125092e+02 + -3.821746611611994e+02 + -3.834046153278488e+02 + -3.846376574000000e+02 + -3.858737925371631e+02 + -3.871130260109560e+02 + -3.883553630688227e+02 + -3.896008088834330e+02 + -3.908493686999996e+02 + -3.921010477895665e+02 + -3.933558513233559e+02 + -3.946137845264227e+02 + -3.958748527376295e+02 + -3.971390612000000e+02 + -3.984064151109821e+02 + -3.996769197585321e+02 + -4.009505804131749e+02 + -4.022274022910074e+02 + -4.035073906999995e+02 + -4.047905509809469e+02 + -4.060768883569506e+02 + -4.073664080804837e+02 + -4.086591154923789e+02 + -4.099550159000000e+02 + -4.112541145837781e+02 + -4.125564168232350e+02 + -4.138619279264533e+02 + -4.151706532436620e+02 + -4.164825980999995e+02 + -4.177977678009782e+02 + -4.191161676484082e+02 + -4.204378029703449e+02 + -4.217626791355866e+02 + -4.230908015000000e+02 + -4.244221753941545e+02 + -4.257568060915354e+02 + -4.270946989435893e+02 + -4.284358594322071e+02 + -4.297802928999995e+02 + -4.311280046181774e+02 + -4.324789999621266e+02 + -4.338332843406102e+02 + -4.351908631680312e+02 + -4.365517418000000e+02 + -4.379159255766193e+02 + -4.392834199290754e+02 + -4.406542302894571e+02 + -4.420283620541414e+02 + -4.434058205999995e+02 + -4.447866113008026e+02 + -4.461707395611302e+02 + -4.475582108324218e+02 + -4.489490306166049e+02 + -4.503432043000000e+02 + -4.517407372272212e+02 + -4.531416348857516e+02 + -4.545459027469901e+02 + -4.559535462055746e+02 + -4.573645706999994e+02 + -4.587789817092769e+02 + -4.601967847429616e+02 + -4.616179852648949e+02 + -4.630425886631084e+02 + -4.644706004000000e+02 + -4.659020259847504e+02 + -4.673368709042424e+02 + -4.687751406384571e+02 + -4.702168406679589e+02 + -4.716619764999995e+02 + -4.731105536500806e+02 + -4.745625775980045e+02 + -4.760180538297375e+02 + -4.774769878558138e+02 + -4.789393852000000e+02 + -4.804052513950263e+02 + -4.818745919803221e+02 + -4.833474124743740e+02 + -4.848237183656923e+02 + -4.863035151999994e+02 + -4.877868085453354e+02 + -4.892736039052198e+02 + -4.907639068268139e+02 + -4.922577229425711e+02 + -4.937550578000000e+02 + -4.952559169027076e+02 + -4.967603058178435e+02 + -4.982682301416788e+02 + -4.997796954853789e+02 + -5.012947073999994e+02 + -5.028132714194502e+02 + -5.043353931676896e+02 + -5.058610782730644e+02 + -5.073903323321115e+02 + -5.089231609000000e+02 + -5.104595695386481e+02 + -5.119995639358939e+02 + -5.135431497453484e+02 + -5.150903325247945e+02 + -5.166411178999995e+02 + -5.181955115300345e+02 + -5.197535190105982e+02 + -5.213151459679927e+02 + -5.228803980937674e+02 + -5.244492810000000e+02 + -5.260218002787250e+02 + -5.275979616609960e+02 + -5.291777708470047e+02 + -5.307612334400695e+02 + -5.323483550999995e+02 + -5.339391415258532e+02 + -5.355335984055753e+02 + -5.371317314077364e+02 + -5.387335461786030e+02 + -5.403390484000000e+02 + -5.419482437848817e+02 + -5.435611380730882e+02 + -5.451777369746786e+02 + -5.467980461477442e+02 + -5.484220712999994e+02 + -5.500498181706811e+02 + -5.516812924850077e+02 + -5.533164999617454e+02 + -5.549554463178829e+02 + -5.565981372999994e+02 + -5.582445786620732e+02 + -5.598947761113077e+02 + -5.615487353817318e+02 + -5.632064622631164e+02 + -5.648679624999994e+02 + -5.665332418238921e+02 + -5.682023060429622e+02 + -5.698751609401672e+02 + -5.715518122344557e+02 + -5.732322656999993e+02 + -5.749165271427032e+02 + -5.766046023386582e+02 + -5.782964970855437e+02 + -5.799922172195480e+02 + -5.816917684999994e+02 + -5.833951566715182e+02 + -5.851023876259811e+02 + -5.868134672043320e+02 + -5.885284011210334e+02 + -5.902471951999993e+02 + -5.919698553287222e+02 + -5.936963873375214e+02 + -5.954267970459595e+02 + -5.971610902818754e+02 + -5.988992728999993e+02 + -6.006413507613062e+02 + -6.023873296844711e+02 + -6.041372155081436e+02 + -6.058910141182311e+02 + -6.076487313999993e+02 + -6.094103732273231e+02 + -6.111759454411804e+02 + -6.129454539033432e+02 + -6.147189045204622e+02 + -6.164963032000001e+02 + -6.182776558378988e+02 + -6.200629682925465e+02 + -6.218522464515877e+02 + -6.236454962577363e+02 + -6.254427235999993e+02 + -6.272439343587159e+02 + -6.290491345295636e+02 + -6.308583300588987e+02 + -6.326715267788347e+02 + -6.344887306000001e+02 + -6.363099474872770e+02 + -6.381351833941451e+02 + -6.399644442896572e+02 + -6.417977361628546e+02 + -6.436350648999993e+02 + -6.454764363684590e+02 + -6.473218566292518e+02 + -6.491713317125350e+02 + -6.510248675271151e+02 + -6.528824700000000e+02 + -6.547441451008573e+02 + -6.566098988943567e+02 + -6.584797373808955e+02 + -6.603536664383599e+02 + -6.622316920999993e+02 + -6.641138204605409e+02 + -6.660000574329696e+02 + -6.678904089961051e+02 + -6.697848812885560e+02 + -6.716834803000000e+02 + -6.735862119462199e+02 + -6.754930822688247e+02 + -6.774040973560088e+02 + -6.793192633079277e+02 + -6.812385860999992e+02 + -6.831620716800010e+02 + -6.850897262089175e+02 + -6.870215557906422e+02 + -6.889575663679858e+02 + -6.908977640000001e+02 + -6.928421548145254e+02 + -6.947907448778678e+02 + -6.967435402481692e+02 + -6.987005469959722e+02 + -7.006617711999993e+02 + -7.026272189534540e+02 + -7.045968963883171e+02 + -7.065708096021641e+02 + -7.085489646318440e+02 + -7.105313676000000e+02 + -7.125180246666655e+02 + -7.145089419064375e+02 + -7.165041254104643e+02 + -7.185035813280965e+02 + -7.205073157999993e+02 + -7.225153349567163e+02 + -7.245276449240899e+02 + -7.265442518464891e+02 + -7.285651618951177e+02 + -7.305903812000000e+02 + -7.326199158832851e+02 + -7.346537721472801e+02 + -7.366919561576724e+02 + -7.387344740033407e+02 + -7.407813318999993e+02 + -7.428325361060332e+02 + -7.448880927069067e+02 + -7.469480078569493e+02 + -7.490122878703525e+02 + -7.510809389000000e+02 + -7.531539670309830e+02 + -7.552313785295614e+02 + -7.573131796529314e+02 + -7.593993765758833e+02 + -7.614899754999992e+02 + -7.635849826556215e+02 + -7.656844042998581e+02 + -7.677882466626152e+02 + -7.698965159244972e+02 + -7.720092183000000e+02 + -7.741263600422619e+02 + -7.762479474594095e+02 + -7.783739868214470e+02 + -7.805044843222228e+02 + -7.826394461999992e+02 + -7.847788787331309e+02 + -7.869227882276110e+02 + -7.890711809493379e+02 + -7.912240631015031e+02 + -7.933814410000000e+02 + -7.955433210068886e+02 + -7.977097093640062e+02 + -7.998806123337483e+02 + -8.020560362523712e+02 + -8.042359873999992e+02 + -8.064204720446323e+02 + -8.086094965664213e+02 + -8.108030673053524e+02 + -8.130011905039750e+02 + -8.152038725000000e+02 + -8.174111196729889e+02 + -8.196229383028856e+02 + -8.218393347070877e+02 + -8.240603152936806e+02 + -8.262858863999992e+02 + -8.285160543327190e+02 + -8.307508254786889e+02 + -8.329902062108882e+02 + -8.352342028550189e+02 + -8.374828218000000e+02 + -8.397360694583805e+02 + -8.419939521636865e+02 + -8.442564762682848e+02 + -8.465236481828920e+02 + -8.487954742999991e+02 + -8.510719610066617e+02 + -8.533531147279726e+02 + -8.556389418758610e+02 + -8.579294488297842e+02 + -8.602246420000000e+02 + -8.625245278111681e+02 + -8.648291126593311e+02 + -8.671384029773127e+02 + -8.694524052578884e+02 + -8.717711258999991e+02 + -8.740945712693658e+02 + -8.764227478547659e+02 + -8.787556621558366e+02 + -8.810933206360036e+02 + -8.834357297000000e+02 + -8.857828957473700e+02 + -8.881348253000285e+02 + -8.904915248639498e+02 + -8.928530008754832e+02 + -8.952192597999992e+02 + -8.975903081273602e+02 + -8.999661523549036e+02 + -9.023467989915350e+02 + -9.047322545556517e+02 + -9.071225255000001e+02 + -9.095176182642568e+02 + -9.119175394073789e+02 + -9.143222954857910e+02 + -9.167318930073794e+02 + -9.191463384999990e+02 + -9.215656384936522e+02 + -9.239897994653498e+02 + -9.264188279352956e+02 + -9.288527305046009e+02 + -9.312915136999991e+02 + -9.337351840217814e+02 + -9.361837480744202e+02 + -9.386372124507991e+02 + -9.410955836845210e+02 + -9.435588682999991e+02 + -9.460270728408829e+02 + -9.485002039375468e+02 + -9.509782681943145e+02 + -9.534612721474879e+02 + -9.559492223999991e+02 + -9.584421255818636e+02 + -9.609399882443452e+02 + -9.634428169536077e+02 + -9.659506183300578e+02 + -9.684633989999990e+02 + -9.709811655949806e+02 + -9.735039247636544e+02 + -9.760316831188370e+02 + -9.785644472182881e+02 + -9.811022236999991e+02 + -9.836450192407020e+02 + -9.861928404528198e+02 + -9.887456939756197e+02 + -9.913035865101050e+02 + -9.938665246999991e+02 + -9.964345151588749e+02 + -9.990075645416573e+02 + -1.001585679508894e+03 + -1.004168866725833e+03 + -1.006757132999999e+03 + -1.009350485088950e+03 + -1.011948929193798e+03 + -1.014552472017065e+03 + -1.017161121149714e+03 + -1.019774882999999e+03 + -1.022393763477281e+03 + -1.025017769745607e+03 + -1.027646908706880e+03 + -1.030281186449000e+03 + -1.032920609999999e+03 + -1.035565186772393e+03 + -1.038214923106640e+03 + -1.040869825393126e+03 + -1.043529900494233e+03 + -1.046195154999999e+03 + -1.048865595518282e+03 + -1.051541229525064e+03 + -1.054222063914517e+03 + -1.056908104442841e+03 + -1.059599357999999e+03 + -1.062295832060889e+03 + -1.064997533253958e+03 + -1.067704468147946e+03 + -1.070416643533965e+03 + -1.073134066000000e+03 + -1.075856742179453e+03 + -1.078584679453085e+03 + -1.081317884808236e+03 + -1.084056364390976e+03 + -1.086800124999999e+03 + -1.089549173751501e+03 + -1.092303517161580e+03 + -1.095063162007773e+03 + -1.097828115663242e+03 + -1.100598385000000e+03 + -1.103373976558467e+03 + -1.106154896984741e+03 + -1.108941153067517e+03 + -1.111732751768433e+03 + -1.114529699999999e+03 + -1.117332004656191e+03 + -1.120139672719820e+03 + -1.122952711033347e+03 + -1.125771126202256e+03 + -1.128594925000000e+03 + -1.131424114391635e+03 + -1.134258701616623e+03 + -1.137098693726371e+03 + -1.139944097362279e+03 + -1.142794918999999e+03 + -1.145651165298781e+03 + -1.148512843975744e+03 + -1.151379962155369e+03 + -1.154252525751212e+03 + -1.157130542000000e+03 + -1.160014018647746e+03 + -1.162902961796701e+03 + -1.165797378002409e+03 + -1.168697275086654e+03 + -1.171602659999999e+03 + -1.174513539162380e+03 + -1.177429919411751e+03 + -1.180351807863387e+03 + -1.183279211838743e+03 + -1.186212138000000e+03 + -1.189150592852472e+03 + -1.192094584016226e+03 + -1.195044118757429e+03 + -1.197999203418656e+03 + -1.200959844999999e+03 + -1.203926050837031e+03 + -1.206897827704460e+03 + -1.209875182550288e+03 + -1.212858122751144e+03 + -1.215846655000000e+03 + -1.218840785789916e+03 + -1.221840522671824e+03 + -1.224845872987532e+03 + -1.227856843385819e+03 + -1.230873440999999e+03 + -1.233895673111333e+03 + -1.236923546192640e+03 + -1.239957067026041e+03 + -1.242996243170224e+03 + -1.246041082000000e+03 + -1.249091590572761e+03 + -1.252147775334870e+03 + -1.255209643154299e+03 + -1.258277201722480e+03 + -1.261350457999999e+03 + -1.264429418673061e+03 + -1.267514091394071e+03 + -1.270604483350415e+03 + -1.273700600740823e+03 + -1.276802451000000e+03 + -1.279910042060014e+03 + -1.283023380429426e+03 + -1.286142472688095e+03 + -1.289267326090673e+03 + -1.292397947999999e+03 + -1.295534345774196e+03 + -1.298676526619691e+03 + -1.301824497524618e+03 + -1.304978265240164e+03 + -1.308137837000000e+03 + -1.311303220289759e+03 + -1.314474422298625e+03 + -1.317651449999705e+03 + -1.320834310180745e+03 + -1.324023009999999e+03 + -1.327217556910311e+03 + -1.330417958531748e+03 + -1.333624221890711e+03 + -1.336836353161044e+03 + -1.340054360000000e+03 + -1.343278250622039e+03 + -1.346508031450468e+03 + -1.349743709083262e+03 + -1.352985291084887e+03 + -1.356232784999999e+03 + -1.359486198143600e+03 + -1.362745537187869e+03 + -1.366010809195683e+03 + -1.369282022032803e+03 + -1.372559183000000e+03 + -1.375842299046690e+03 + -1.379131377356324e+03 + -1.382426425045151e+03 + -1.385727449081319e+03 + -1.389034456999999e+03 + -1.392347456524483e+03 + -1.395666454635196e+03 + -1.398991458464414e+03 + -1.402322475610563e+03 + -1.405659513000000e+03 + -1.409002577344138e+03 + -1.412351676333446e+03 + -1.415706817605260e+03 + -1.419068008353351e+03 + -1.422435255999999e+03 + -1.425808567964313e+03 + -1.429187950969556e+03 + -1.432573411998037e+03 + -1.435964958719179e+03 + -1.439362599000000e+03 + -1.442766340439504e+03 + -1.446176189225631e+03 + -1.449592152360744e+03 + -1.453014238529275e+03 + -1.456442454999999e+03 + -1.459876808245435e+03 + -1.463317305593544e+03 + -1.466763954627232e+03 + -1.470216762966463e+03 + -1.473675738000000e+03 + -1.477140887005819e+03 + -1.480612217422037e+03 + -1.484089736511155e+03 + -1.487573451238554e+03 + -1.491063368999999e+03 + -1.494559497483912e+03 + -1.498061844364458e+03 + -1.501570417048641e+03 + -1.505085222548716e+03 + -1.508606267999999e+03 + -1.512133560790714e+03 + -1.515667108906144e+03 + -1.519206919999463e+03 + -1.522753000996140e+03 + -1.526305358999999e+03 + -1.529864001422905e+03 + -1.533428936276007e+03 + -1.537000171179230e+03 + -1.540577712963951e+03 + -1.544161568999999e+03 + -1.547751747018138e+03 + -1.551348254621497e+03 + -1.554951099280540e+03 + -1.558560288310355e+03 + -1.562175828999999e+03 + -1.565797728689906e+03 + -1.569425994958102e+03 + -1.573060635400619e+03 + -1.576701657553543e+03 + -1.580349068999999e+03 + -1.584002877314684e+03 + -1.587663089901193e+03 + -1.591329713983250e+03 + -1.595002756657312e+03 + -1.598682225999999e+03 + -1.602368130282464e+03 + -1.606060476003810e+03 + -1.609759270166310e+03 + -1.613464521194204e+03 + -1.617176236999999e+03 + -1.620894424965112e+03 + -1.624619091996676e+03 + -1.628350245392540e+03 + -1.632087893195996e+03 + -1.635832042999999e+03 + -1.639582702219653e+03 + -1.643339878880967e+03 + -1.647103580589531e+03 + -1.650873814125992e+03 + -1.654650586999999e+03 + -1.658433907212865e+03 + -1.662223782656940e+03 + -1.666020220927786e+03 + -1.669823229228366e+03 + -1.673632814999999e+03 + -1.677448985879331e+03 + -1.681271749582079e+03 + -1.685101113981527e+03 + -1.688937087086922e+03 + -1.692779675999999e+03 + -1.696628887589857e+03 + -1.700484730162859e+03 + -1.704347211664743e+03 + -1.708216339003609e+03 + -1.712092119999999e+03 + -1.715974562928047e+03 + -1.719863675285195e+03 + -1.723759464515762e+03 + -1.727661938269359e+03 + -1.731571103999999e+03 + -1.735486969207614e+03 + -1.739409542118944e+03 + -1.743338830603560e+03 + -1.747274841733617e+03 + -1.751217582999999e+03 + -1.755167062315964e+03 + -1.759123288009624e+03 + -1.763086267739076e+03 + -1.767056008086542e+03 + -1.771032516999999e+03 + -1.775015803098183e+03 + -1.779005873908525e+03 + -1.783002736756524e+03 + -1.787006399126180e+03 + -1.791016869000000e+03 + -1.795034154506543e+03 + -1.799058263107071e+03 + -1.803089202321619e+03 + -1.807126980015989e+03 + -1.811171603999998e+03 + -1.815223082025504e+03 + -1.819281421835726e+03 + -1.823346631247284e+03 + -1.827418718154363e+03 + -1.831497690000000e+03 + -1.835583554215247e+03 + -1.839676319355180e+03 + -1.843775993459518e+03 + -1.847882583413063e+03 + -1.851996096999998e+03 + -1.856116542493618e+03 + -1.860243927542824e+03 + -1.864378259929794e+03 + -1.868519547846306e+03 + -1.872667799000000e+03 + -1.876823020804590e+03 + -1.880985220849939e+03 + -1.885154407199114e+03 + -1.889330588466962e+03 + -1.893513771999998e+03 + -1.897703964771626e+03 + -1.901901175652285e+03 + -1.906105412964281e+03 + -1.910316683519642e+03 + -1.914534995000000e+03 + -1.918760355678869e+03 + -1.922992773607248e+03 + -1.927232256642219e+03 + -1.931478812455566e+03 + -1.935732448999998e+03 + -1.939993174351127e+03 + -1.944260996306766e+03 + -1.948535922544343e+03 + -1.952817960710870e+03 + -1.957107119000000e+03 + -1.961403405774432e+03 + -1.965706828656891e+03 + -1.970017395435863e+03 + -1.974335114379128e+03 + -1.978659992999998e+03 + -1.982992038634364e+03 + -1.987331259958815e+03 + -1.991677665406797e+03 + -1.996031262526477e+03 + -2.000392059000000e+03 + -2.004760062662544e+03 + -2.009135281420714e+03 + -2.013517723394651e+03 + -2.017907396963966e+03 + -2.022304309999998e+03 + -2.026708470129302e+03 + -2.031119885371980e+03 + -2.035538563783372e+03 + -2.039964513323241e+03 + -2.044397742000000e+03 + -2.048838257821666e+03 + -2.053286068652342e+03 + -2.057741182613919e+03 + -2.062203608203423e+03 + -2.066673352999998e+03 + -2.071150424296029e+03 + -2.075634830690161e+03 + -2.080126580708826e+03 + -2.084625682242698e+03 + -2.089132143000000e+03 + -2.093645970720743e+03 + -2.098167173603271e+03 + -2.102695759983049e+03 + -2.107231738184183e+03 + -2.111775115999998e+03 + -2.116325901057278e+03 + -2.120884101683680e+03 + -2.125449726108530e+03 + -2.130022782163747e+03 + -2.134603278000000e+03 + -2.139191221924691e+03 + -2.143786621957349e+03 + -2.148389485981318e+03 + -2.152999821836896e+03 + -2.157617637999998e+03 + -2.162242943074696e+03 + -2.166875744534069e+03 + -2.171516050262848e+03 + -2.176163869133622e+03 + -2.180819209000000e+03 + -2.185482077327040e+03 + -2.190152482871076e+03 + -2.194830433950524e+03 + -2.199515937814626e+03 + -2.204209002999998e+03 + -2.208909638535144e+03 + -2.213617851857877e+03 + -2.218333650781778e+03 + -2.223057044225780e+03 + -2.227788040000000e+03 + -2.232526645541351e+03 + -2.237272869895315e+03 + -2.242026721580949e+03 + -2.246788207788726e+03 + -2.251557336999999e+03 + -2.256334118250544e+03 + -2.261118559161676e+03 + -2.265910667352767e+03 + -2.270710451040453e+03 + -2.275517918999998e+03 + -2.280333080014267e+03 + -2.285155941593499e+03 + -2.289986511539142e+03 + -2.294824798563590e+03 + -2.299670810999998e+03 + -2.304524556927484e+03 + -2.309386044592275e+03 + -2.314255282195762e+03 + -2.319132277836808e+03 + -2.324017039999998e+03 + -2.328909577220459e+03 + -2.333809897224200e+03 + -2.338718008145149e+03 + -2.343633918994289e+03 + -2.348557637999998e+03 + -2.353489172912608e+03 + -2.358428531797252e+03 + -2.363375723128402e+03 + -2.368330755820120e+03 + -2.373293637999998e+03 + -2.378264377427878e+03 + -2.383242982526706e+03 + -2.388229461889857e+03 + -2.393223824063392e+03 + -2.398226076999998e+03 + -2.403236228525699e+03 + -2.408254287477974e+03 + -2.413280262494937e+03 + -2.418314161547432e+03 + -2.423355992999998e+03 + -2.428405765403690e+03 + -2.433463486890615e+03 + -2.438529165661329e+03 + -2.443602810190047e+03 + -2.448684428999998e+03 + -2.453774030522361e+03 + -2.458871622761939e+03 + -2.463977214083831e+03 + -2.469090813490053e+03 + -2.474212428999998e+03 + -2.479342068313477e+03 + -2.484479740528551e+03 + -2.489625454539379e+03 + -2.494779218375099e+03 + -2.499941039999998e+03 + -2.505110927624773e+03 + -2.510288890409285e+03 + -2.515474937052364e+03 + -2.520669075212765e+03 + -2.525871312999998e+03 + -2.531081658990217e+03 + -2.536300122220549e+03 + -2.541526711425658e+03 + -2.546761434689014e+03 + -2.552004299999998e+03 + -2.557255315555890e+03 + -2.562514490474060e+03 + -2.567781833501355e+03 + -2.573057352510476e+03 + -2.578341055999998e+03 + -2.583632952807378e+03 + -2.588933051312229e+03 + -2.594241359879810e+03 + -2.599557887014108e+03 + -2.604882640999998e+03 + -2.610215630196411e+03 + -2.615556863841929e+03 + -2.620906350516468e+03 + -2.626264097579904e+03 + -2.631630113999998e+03 + -2.637004409331177e+03 + -2.642386991059378e+03 + -2.647777867311749e+03 + -2.653177047878767e+03 + -2.658584540999998e+03 + -2.664000354190328e+03 + -2.669424496446296e+03 + -2.674856976816473e+03 + -2.680297803844182e+03 + -2.685746985999998e+03 + -2.691204531699670e+03 + -2.696670449221115e+03 + -2.702144747143579e+03 + -2.707627434527642e+03 + -2.713118519999998e+03 + -2.718618011850782e+03 + -2.724125918263655e+03 + -2.729642247949266e+03 + -2.735167010382318e+03 + -2.740700213999998e+03 + -2.746241866699575e+03 + -2.751791977076697e+03 + -2.757350554006517e+03 + -2.762917606479738e+03 + -2.768493143000000e+03 + -2.774077171851313e+03 + -2.779669701724049e+03 + -2.785270741295763e+03 + -2.790880299095185e+03 + -2.796498383999998e+03 + -2.802125004968091e+03 + -2.807760170344632e+03 + -2.813403888501788e+03 + -2.819056168106567e+03 + -2.824717018000000e+03 + -2.830386447079949e+03 + -2.836064464083062e+03 + -2.841751077483182e+03 + -2.847446295475182e+03 + -2.853150126999998e+03 + -2.858862581346520e+03 + -2.864583667200011e+03 + -2.870313392988722e+03 + -2.876051767015894e+03 + -2.881798798000000e+03 + -2.887554494917011e+03 + -2.893318866668325e+03 + -2.899091922031221e+03 + -2.904873669618204e+03 + -2.910664117999998e+03 + -2.916463275808826e+03 + -2.922271151990944e+03 + -2.928087755363771e+03 + -2.933913094445438e+03 + -2.939747178000000e+03 + -2.945590014959967e+03 + -2.951441614226611e+03 + -2.957301984588625e+03 + -2.963171134666888e+03 + -2.969049072999998e+03 + -2.974935808208847e+03 + -2.980831349417174e+03 + -2.986735705755709e+03 + -2.992648886132359e+03 + -2.998570899000000e+03 + -3.004501752659864e+03 + -3.010441455936329e+03 + -3.016390017858416e+03 + -3.022347447554316e+03 + -3.028313753999998e+03 + -3.034288946006182e+03 + -3.040273032141582e+03 + -3.046266021078663e+03 + -3.052267921739149e+03 + -3.058278743000000e+03 + -3.064298493698561e+03 + -3.070327182685398e+03 + -3.076364818827055e+03 + -3.082411411001809e+03 + -3.088466967999998e+03 + -3.094531498652569e+03 + -3.100605012177602e+03 + -3.106687517554494e+03 + -3.112779023262762e+03 + -3.118879538000000e+03 + -3.124989070695736e+03 + -3.131107630527278e+03 + -3.137235226491168e+03 + -3.143371867254447e+03 + -3.149517561999998e+03 + -3.155672319953097e+03 + -3.161836149115856e+03 + -3.168009058152651e+03 + -3.174191057129923e+03 + -3.180382155000000e+03 + -3.186582360055111e+03 + -3.192791681144167e+03 + -3.199010127274887e+03 + -3.205237707493432e+03 + -3.211474430999998e+03 + -3.217720306909038e+03 + -3.223975343611381e+03 + -3.230239549889151e+03 + -3.236512935355519e+03 + -3.242795509000000e+03 + -3.249087279482800e+03 + -3.255388255944051e+03 + -3.261698447489179e+03 + -3.268017862963467e+03 + -3.274346510999998e+03 + -3.280684400401313e+03 + -3.287031541121871e+03 + -3.293387942369432e+03 + -3.299753611888159e+03 + -3.306128559000000e+03 + -3.312512793688797e+03 + -3.318906324190636e+03 + -3.325309159228279e+03 + -3.331721308860119e+03 + -3.338142781999998e+03 + -3.344573586933190e+03 + -3.351013732692941e+03 + -3.357463228591956e+03 + -3.363922084041577e+03 + -3.370390307999997e+03 + -3.376867909193196e+03 + -3.383354896616435e+03 + -3.389851279509897e+03 + -3.396357067337695e+03 + -3.402872268999997e+03 + -3.409396893188545e+03 + -3.415930949268335e+03 + -3.422474446597307e+03 + -3.429027394238212e+03 + -3.435589800999997e+03 + -3.442161675674332e+03 + -3.448743027574159e+03 + -3.455333866086035e+03 + -3.461934200499129e+03 + -3.468544039999997e+03 + -3.475163393634110e+03 + -3.481792270131310e+03 + -3.488430678432703e+03 + -3.495078627927044e+03 + -3.501736127999997e+03 + -3.508403187927853e+03 + -3.515079816661691e+03 + -3.521766023349286e+03 + -3.528461817510494e+03 + -3.535167207999998e+03 + -3.541882203477436e+03 + -3.548606813620608e+03 + -3.555341047964333e+03 + -3.562084915431213e+03 + -3.568838424999997e+03 + -3.575601585773337e+03 + -3.582374407066678e+03 + -3.589156898185297e+03 + -3.595949068354316e+03 + -3.602750926999998e+03 + -3.609562483468940e+03 + -3.616383746264014e+03 + -3.623214724488114e+03 + -3.630055428392298e+03 + -3.636905866999997e+03 + -3.643766048793625e+03 + -3.650635983516456e+03 + -3.657515680680700e+03 + -3.664405149014039e+03 + -3.671304397999997e+03 + -3.678213437374473e+03 + -3.685132275760156e+03 + -3.692060922035369e+03 + -3.698999385901594e+03 + -3.705947676999997e+03 + -3.712905804797643e+03 + -3.719873778403308e+03 + -3.726851606752037e+03 + -3.733839298718909e+03 + -3.740836863999997e+03 + -3.747844312547997e+03 + -3.754861653211674e+03 + -3.761888895074585e+03 + -3.768926047955085e+03 + -3.775973120999997e+03 + -3.783030123074376e+03 + -3.790097063813473e+03 + -3.797173952745452e+03 + -3.804260798968512e+03 + -3.811357611999997e+03 + -3.818464401472386e+03 + -3.825581176303607e+03 + -3.832707945585785e+03 + -3.839844718954238e+03 + -3.846991505999998e+03 + -3.854148316201428e+03 + -3.861315158822798e+03 + -3.868492043129635e+03 + -3.875678978458987e+03 + -3.882875973999997e+03 + -3.890083038997269e+03 + -3.897300183310783e+03 + -3.904527416633095e+03 + -3.911764748154242e+03 + -3.919012186999997e+03 + -3.926269742452437e+03 + -3.933537424464352e+03 + -3.940815242621297e+03 + -3.948103205720294e+03 + -3.955401322999997e+03 + -3.962709604107895e+03 + -3.970028058998795e+03 + -3.977356697163350e+03 + -3.984695527326294e+03 + -3.992044558999997e+03 + -3.999403802144854e+03 + -4.006773266302151e+03 + -4.014152960820400e+03 + -4.021542894928240e+03 + -4.028943077999997e+03 + -4.036353519583599e+03 + -4.043774229513202e+03 + -4.051205217431756e+03 + -4.058646492564995e+03 + -4.066098063999997e+03 + -4.073559940995820e+03 + -4.081032133755226e+03 + -4.088514652104142e+03 + -4.096007504979870e+03 + -4.103510702000000e+03 + -4.111024253097677e+03 + -4.118548167510886e+03 + -4.126082454416708e+03 + -4.133627123225803e+03 + -4.141182184000000e+03 + -4.148747646853033e+03 + -4.156323520486682e+03 + -4.163909814098826e+03 + -4.171506538118098e+03 + -4.179113701999993e+03 + -4.186731314728609e+03 + -4.194359386189681e+03 + -4.201997926116393e+03 + -4.209646943720986e+03 + -4.217306449000000e+03 + -4.224976452122954e+03 + -4.232656961817923e+03 + -4.240347987240753e+03 + -4.248049538718602e+03 + -4.255761626000000e+03 + -4.263484258397591e+03 + -4.271217445246016e+03 + -4.278961196276451e+03 + -4.286715521743729e+03 + -4.294480431000000e+03 + -4.302255933066315e+03 + -4.310042038085233e+03 + -4.317838756007569e+03 + -4.325646096087124e+03 + -4.333464067999995e+03 + -4.341292681602012e+03 + -4.349131946225953e+03 + -4.356981871466000e+03 + -4.364842467485710e+03 + -4.372713744000000e+03 + -4.380595710372718e+03 + -4.388488375861229e+03 + -4.396391750264422e+03 + -4.404305844160006e+03 + -4.412230667000000e+03 + -4.420166227736474e+03 + -4.428112536405837e+03 + -4.436069603088062e+03 + -4.444037437481978e+03 + -4.452016049000000e+03 + -4.460005447074678e+03 + -4.468005641867199e+03 + -4.476016643204103e+03 + -4.484038460200805e+03 + -4.492071102999995e+03 + -4.500114582060231e+03 + -4.508168906299815e+03 + -4.516234085098625e+03 + -4.524310129075975e+03 + -4.532397048000000e+03 + -4.540494851104932e+03 + -4.548603547974418e+03 + -4.556723148595408e+03 + -4.564853663376500e+03 + -4.572995102000000e+03 + -4.581147473755137e+03 + -4.589310788334117e+03 + -4.597485055595826e+03 + -4.605670285497996e+03 + -4.613866488000000e+03 + -4.622073673035890e+03 + -4.630291850451470e+03 + -4.638521029781397e+03 + -4.646761220207367e+03 + -4.655012431999994e+03 + -4.663274675842531e+03 + -4.671547961173437e+03 + -4.679832297267551e+03 + -4.688127693710799e+03 + -4.696434161000000e+03 + -4.704751709752170e+03 + -4.713080348793824e+03 + -4.721420087561993e+03 + -4.729770937031085e+03 + -4.738132907000000e+03 + -4.746506006604411e+03 + -4.754890245715343e+03 + -4.763285634540516e+03 + -4.771692183448750e+03 + -4.780109901999994e+03 + -4.788538799466528e+03 + -4.796978886096308e+03 + -4.805430172162917e+03 + -4.813892667577567e+03 + -4.822366381999994e+03 + -4.830851325088147e+03 + -4.839347507060687e+03 + -4.847854938123849e+03 + -4.856373628221604e+03 + -4.864903587000000e+03 + -4.873444824101478e+03 + -4.881997349859994e+03 + -4.890561174364984e+03 + -4.899136307099045e+03 + -4.907722758000000e+03 + -4.916320537362784e+03 + -4.924929655587317e+03 + -4.933550122528648e+03 + -4.942181947260091e+03 + -4.950825139999994e+03 + -4.959479711457802e+03 + -4.968145671175524e+03 + -4.976823028809056e+03 + -4.985511794647833e+03 + -4.994211978999993e+03 + -5.002923592007461e+03 + -5.011646643245516e+03 + -5.020381142480443e+03 + -5.029127099988444e+03 + -5.037884526000000e+03 + -5.046653430641881e+03 + -5.055433823857634e+03 + -5.064225715527905e+03 + -5.073029115529123e+03 + -5.081844034000000e+03 + -5.090670481166524e+03 + -5.099508466925161e+03 + -5.108358001180135e+03 + -5.117219094017154e+03 + -5.126091755999993e+03 + -5.134975997626908e+03 + -5.143871827953957e+03 + -5.152779256704796e+03 + -5.161698295082459e+03 + -5.170628952999993e+03 + -5.179571239778229e+03 + -5.188525166029071e+03 + -5.197490742130520e+03 + -5.206467977662855e+03 + -5.215456883000000e+03 + -5.224457468752808e+03 + -5.233469744262066e+03 + -5.242493719420175e+03 + -5.251529405361101e+03 + -5.260576812000000e+03 + -5.269635948683524e+03 + -5.278706825911151e+03 + -5.287789454172008e+03 + -5.296883843495917e+03 + -5.305990003999993e+03 + -5.315107945871962e+03 + -5.324237679253967e+03 + -5.333379214299059e+03 + -5.342532561186867e+03 + -5.351697729999993e+03 + -5.360874730862462e+03 + -5.370063574314554e+03 + -5.379264270604972e+03 + -5.388476829418795e+03 + -5.397701261000000e+03 + -5.406937575887649e+03 + -5.416185784228891e+03 + -5.425445896082409e+03 + -5.434717921547790e+03 + -5.444001871000000e+03 + -5.453297754883592e+03 + -5.462605583222671e+03 + -5.471925366229407e+03 + -5.481257114511772e+03 + -5.490600837999993e+03 + -5.499956546429602e+03 + -5.509324250592133e+03 + -5.518703961068116e+03 + -5.528095687743725e+03 + -5.537499440999993e+03 + -5.546915231361128e+03 + -5.556343068498348e+03 + -5.565782962476614e+03 + -5.575234924238452e+03 + -5.584698964000000e+03 + -5.594175091687396e+03 + -5.603663318133420e+03 + -5.613163653827974e+03 + -5.622676108416792e+03 + -5.632200692000000e+03 + -5.641737415094019e+03 + -5.651286288507457e+03 + -5.660847322604374e+03 + -5.670420527008017e+03 + -5.680005911999993e+03 + -5.689603488367688e+03 + -5.699213267038800e+03 + -5.708835258184300e+03 + -5.718469470901945e+03 + -5.728115915999992e+03 + -5.737774605002216e+03 + -5.747445547634946e+03 + -5.757128753652110e+03 + -5.766824233562706e+03 + -5.776531998000000e+03 + -5.786252057564687e+03 + -5.795984422575620e+03 + -5.805729103393980e+03 + -5.815486110517342e+03 + -5.825255454000000e+03 + -5.835037143865786e+03 + -5.844831191184440e+03 + -5.854637606668529e+03 + -5.864456400127173e+03 + -5.874287581999993e+03 + -5.884131163070459e+03 + -5.893987153665426e+03 + -5.903855564090225e+03 + -5.913736404781277e+03 + -5.923629686000000e+03 + -5.933535418061928e+03 + -5.943453611973397e+03 + -5.953384278439786e+03 + -5.963327427438455e+03 + -5.973283069000000e+03 + -5.983251213525049e+03 + -5.993231872573540e+03 + -6.003225056744913e+03 + -6.013230774891098e+03 + -6.023249038000000e+03 + -6.033279857947543e+03 + -6.043323244258373e+03 + -6.053379206808452e+03 + -6.063447756872029e+03 + -6.073528904999992e+03 + -6.083622661322729e+03 + -6.093729036471069e+03 + -6.103848041095609e+03 + -6.113979685699232e+03 + -6.124123981000000e+03 + -6.134280937646101e+03 + -6.144450565430006e+03 + -6.154632874755873e+03 + -6.164827877191976e+03 + -6.175035583000000e+03 + -6.185256001935748e+03 + -6.195489145360168e+03 + -6.205735024246584e+03 + -6.215993648382001e+03 + -6.226265028000000e+03 + -6.236549173839082e+03 + -6.246846097225249e+03 + -6.257155808917487e+03 + -6.267478318647532e+03 + -6.277813636999993e+03 + -6.288161775039973e+03 + -6.298522743318929e+03 + -6.308896552332133e+03 + -6.319283212690151e+03 + -6.329682735000000e+03 + -6.340095129841887e+03 + -6.350520407726451e+03 + -6.360958579315570e+03 + -6.371409655512612e+03 + -6.381873647000000e+03 + -6.392350564293673e+03 + -6.402840417873253e+03 + -6.413343218449801e+03 + -6.423858977046060e+03 + -6.434387704000000e+03 + -6.444929409479182e+03 + -6.455484104800415e+03 + -6.466051800878468e+03 + -6.476632507643834e+03 + -6.487226235999992e+03 + -6.497832997260840e+03 + -6.508452801649206e+03 + -6.519085659445890e+03 + -6.529731581482401e+03 + -6.540390579000000e+03 + -6.551062663116751e+03 + -6.561747843536938e+03 + -6.572446130639311e+03 + -6.583157536279329e+03 + -6.593882071000000e+03 + -6.604619744763902e+03 + -6.615370568925604e+03 + -6.626134554413510e+03 + -6.636911711064461e+03 + -6.647702049999992e+03 + -6.658505582840396e+03 + -6.669322319643610e+03 + -6.680152270785791e+03 + -6.690995447680221e+03 + -6.701851860999993e+03 + -6.712721521095499e+03 + -6.723604439166686e+03 + -6.734500626205399e+03 + -6.745410092579864e+03 + -6.756332849000000e+03 + -6.767268906377159e+03 + -6.778218275408767e+03 + -6.789180966973428e+03 + -6.800156992270870e+03 + -6.811146362000000e+03 + -6.822149086596000e+03 + -6.833165176809022e+03 + -6.844194643583943e+03 + -6.855237498022965e+03 + -6.866293750999992e+03 + -6.877363413246921e+03 + -6.888446495554016e+03 + -6.899543008749155e+03 + -6.910652963678518e+03 + -6.921776370999992e+03 + -6.932913241460104e+03 + -6.944063586638155e+03 + -6.955227417524035e+03 + -6.966404743973952e+03 + -6.977595577000000e+03 + -6.988799928187199e+03 + -7.000017808173458e+03 + -7.011249227729020e+03 + -7.022494198140058e+03 + -7.033752730000000e+03 + -7.045024833685246e+03 + -7.056310520605544e+03 + -7.067609802022496e+03 + -7.078922688575014e+03 + -7.090249190999992e+03 + -7.101589320140114e+03 + -7.112943086874141e+03 + -7.124310502404821e+03 + -7.135691578318693e+03 + -7.147086324999992e+03 + -7.158494752468790e+03 + -7.169916872475182e+03 + -7.181352696510882e+03 + -7.192802235036809e+03 + -7.204265499000000e+03 + -7.215742499578333e+03 + -7.227233247391960e+03 + -7.238737753333357e+03 + -7.250256028887258e+03 + -7.261788085000000e+03 + -7.273333932305611e+03 + -7.284893581718426e+03 + -7.296467044458082e+03 + -7.308054332025791e+03 + -7.319655454999992e+03 + -7.331270423697963e+03 + -7.342899249833889e+03 + -7.354541944862027e+03 + -7.366198519315069e+03 + -7.377868983999992e+03 + -7.389553350032372e+03 + -7.401251628839381e+03 + -7.412963831476617e+03 + -7.424689968390403e+03 + -7.436430051000000e+03 + -7.448184091042194e+03 + -7.459952098877861e+03 + -7.471734085033887e+03 + -7.483530060856024e+03 + -7.495340038000000e+03 + -7.507164027981260e+03 + -7.519002041143363e+03 + -7.530854088088799e+03 + -7.542720180263871e+03 + -7.554600328999991e+03 + -7.566494545476364e+03 + -7.578402840747203e+03 + -7.590325225748156e+03 + -7.602261711334166e+03 + -7.614212308999990e+03 + -7.626177030375796e+03 + -7.638155885970103e+03 + -7.650148886679111e+03 + -7.662156044359298e+03 + -7.674177370000000e+03 + -7.686212874133715e+03 + -7.698262567927072e+03 + -7.710326462858042e+03 + -7.722404570574414e+03 + -7.734496902000000e+03 + -7.746603467726753e+03 + -7.758724278933637e+03 + -7.770859347070250e+03 + -7.783008683730720e+03 + -7.795172299999991e+03 + -7.807350206747154e+03 + -7.819542415323729e+03 + -7.831748936928582e+03 + -7.843969782379609e+03 + -7.856204962999991e+03 + -7.868454490389281e+03 + -7.880718375856472e+03 + -7.892996630459527e+03 + -7.905289265033489e+03 + -7.917596291000000e+03 + -7.929917719977670e+03 + -7.942253562811284e+03 + -7.954603830544003e+03 + -7.966968534806996e+03 + -7.979347687000000e+03 + -7.991741298275563e+03 + -8.004149379532312e+03 + -8.016571941875890e+03 + -8.029008996834256e+03 + -8.041460555999991e+03 + -8.053926630829063e+03 + -8.066407232164392e+03 + -8.078902371093302e+03 + -8.091412059283156e+03 + -8.103936307999991e+03 + -8.116475128348727e+03 + -8.129028531951320e+03 + -8.141596530160446e+03 + -8.154179133753471e+03 + -8.166776354000000e+03 + -8.179388202500601e+03 + -8.192014690774517e+03 + -8.204655830155478e+03 + -8.217311631752342e+03 + -8.229982107000000e+03 + -8.242667267443709e+03 + -8.255367124181290e+03 + -8.268081688506129e+03 + -8.280810972124258e+03 + -8.293554985999990e+03 + -8.306313740930424e+03 + -8.319087249046417e+03 + -8.331875522237864e+03 + -8.344678571514016e+03 + -8.357496408000001e+03 + -8.370329042981793e+03 + -8.383176487893126e+03 + -8.396038754236946e+03 + -8.408915853538949e+03 + -8.421807796999999e+03 + -8.434714595760075e+03 + -8.447636261558551e+03 + -8.460572806112319e+03 + -8.473524240852536e+03 + -8.486490577000000e+03 + -8.499471825734363e+03 + -8.512467998550057e+03 + -8.525479106914012e+03 + -8.538505162160651e+03 + -8.551546175999989e+03 + -8.564602160205039e+03 + -8.577673125806519e+03 + -8.590759084111069e+03 + -8.603860047081474e+03 + -8.616976026000000e+03 + -8.630107031891077e+03 + -8.643253076647045e+03 + -8.656414171854482e+03 + -8.669590328370747e+03 + -8.682781558000001e+03 + -8.695987872906602e+03 + -8.709209284090668e+03 + -8.722445802791402e+03 + -8.735697441007769e+03 + -8.748964210000000e+03 + -8.762246120773598e+03 + -8.775543185390141e+03 + -8.788855415760128e+03 + -8.802182823160678e+03 + -8.815525418999991e+03 + -8.828883214778334e+03 + -8.842256221886048e+03 + -8.855644451817980e+03 + -8.869047916276779e+03 + -8.882466627000000e+03 + -8.895900595656447e+03 + -8.909349833599377e+03 + -8.922814352320849e+03 + -8.936294163622882e+03 + -8.949789279000001e+03 + -8.963299709806102e+03 + -8.976825467697834e+03 + -8.990366564425331e+03 + -9.003923011733321e+03 + -9.017494820999989e+03 + -9.031082003570244e+03 + -9.044684571593196e+03 + -9.058302536801553e+03 + -9.071935910043938e+03 + -9.085584702999991e+03 + -9.099248927805194e+03 + -9.112928596071970e+03 + -9.126623719398038e+03 + -9.140334309541779e+03 + -9.154060378000000e+03 + -9.167801936143134e+03 + -9.181558995552323e+03 + -9.195331568052554e+03 + -9.209119665711642e+03 + -9.222923300000000e+03 + -9.236742482191501e+03 + -9.250577224366738e+03 + -9.264427538348678e+03 + -9.278293435300837e+03 + -9.292174926999989e+03 + -9.306072025514850e+03 + -9.319984742353185e+03 + -9.333913089239737e+03 + -9.347857078372264e+03 + -9.361816720999988e+03 + -9.375792028076301e+03 + -9.389783011923670e+03 + -9.403789684689073e+03 + -9.417812057757259e+03 + -9.431850143000000e+03 + -9.445903952452922e+03 + -9.459973497383473e+03 + -9.474058789280491e+03 + -9.488159840242311e+03 + -9.502276662000000e+03 + -9.516409266099370e+03 + -9.530557664431180e+03 + -9.544721868948835e+03 + -9.558901891554557e+03 + -9.573097743999990e+03 + -9.587309437907974e+03 + -9.601536984771086e+03 + -9.615780396443668e+03 + -9.630039685320193e+03 + -9.644314862999989e+03 + -9.658605940694017e+03 + -9.672912930240365e+03 + -9.687235843732506e+03 + -9.701574693379191e+03 + -9.715929491000001e+03 + -9.730300248146514e+03 + -9.744686976337332e+03 + -9.759089687418043e+03 + -9.773508393705906e+03 + -9.787943106999999e+03 + -9.802393838806964e+03 + -9.816860600913731e+03 + -9.831343405392694e+03 + -9.845842264577648e+03 + -9.860357189999990e+03 + -9.874888192875391e+03 + -9.889435285292540e+03 + -9.903998479577405e+03 + -9.918577788007758e+03 + -9.933173221999989e+03 + -9.947784792760667e+03 + -9.962412512866247e+03 + -9.977056394630243e+03 + -9.991716449514770e+03 + -1.000639269000000e+04 + -1.002108512827329e+04 + -1.003579377260607e+04 + -1.005051863510592e+04 + -1.006525973450424e+04 + -1.008001708000000e+04 + -1.009479067738785e+04 + -1.010958054488143e+04 + -1.012438669671707e+04 + -1.013920913701616e+04 + -1.015404787999999e+04 + -1.016890294387830e+04 + -1.018377433451942e+04 + -1.019866206104612e+04 + -1.021356614193473e+04 + -1.022848658999999e+04 + -1.024342341422066e+04 + -1.025837662498870e+04 + -1.027334623400460e+04 + -1.028833225445739e+04 + -1.030333470000000e+04 + -1.031835358368700e+04 + -1.033338891534757e+04 + -1.034844070623467e+04 + -1.036350897076085e+04 + -1.037859372000000e+04 + -1.039369496373519e+04 + -1.040881271589677e+04 + -1.042394698923176e+04 + -1.043909779325773e+04 + -1.045426513999999e+04 + -1.046944904320307e+04 + -1.048464951626011e+04 + -1.049986657147346e+04 + -1.051510021951631e+04 + -1.053035046999999e+04 + -1.054561733346490e+04 + -1.056090082649770e+04 + -1.057620096382862e+04 + -1.059151775501402e+04 + -1.060685121000000e+04 + -1.062220133982168e+04 + -1.063756815769902e+04 + -1.065295167679853e+04 + -1.066835190951487e+04 + -1.068376887000000e+04 + -1.069920257172756e+04 + -1.071465302083269e+04 + -1.073012022736731e+04 + -1.074560420973240e+04 + -1.076110497999999e+04 + -1.077662254698522e+04 + -1.079215692468130e+04 + -1.080770812602064e+04 + -1.082327616047952e+04 + -1.083886103999999e+04 + -1.085446277832030e+04 + -1.087008138917964e+04 + -1.088571688451933e+04 + -1.090136927381244e+04 + -1.091703857000000e+04 + -1.093272478707855e+04 + -1.094842793381667e+04 + -1.096414802233435e+04 + -1.097988507119364e+04 + -1.099563909000000e+04 + -1.101141008471468e+04 + -1.102719807131307e+04 + -1.104300306633410e+04 + -1.105882508291791e+04 + -1.107466412999999e+04 + -1.109052021511841e+04 + -1.110639335038349e+04 + -1.112228355101184e+04 + -1.113819083451417e+04 + -1.115411520999999e+04 + -1.117005668368749e+04 + -1.118601527242516e+04 + -1.120199099177015e+04 + -1.121798385114509e+04 + -1.123399386000000e+04 + -1.125002102929127e+04 + -1.126606537454743e+04 + -1.128212691149971e+04 + -1.129820565394142e+04 + -1.131430161000000e+04 + -1.133041478676655e+04 + -1.134654520134832e+04 + -1.136269286880464e+04 + -1.137885779750048e+04 + -1.139503999999999e+04 + -1.141123949060806e+04 + -1.142745627830015e+04 + -1.144369037489935e+04 + -1.145994179819328e+04 + -1.147621056000000e+04 + -1.149249666929363e+04 + -1.150880014040390e+04 + -1.152512098586926e+04 + -1.154145921396878e+04 + -1.155781484000000e+04 + -1.157418788116290e+04 + -1.159057834325663e+04 + -1.160698623573862e+04 + -1.162341157750990e+04 + -1.163985437999999e+04 + -1.165631465167572e+04 + -1.167279241049137e+04 + -1.168928767028212e+04 + -1.170580043559586e+04 + -1.172233071999999e+04 + -1.173887854137991e+04 + -1.175544390951204e+04 + -1.177202683447122e+04 + -1.178862732998528e+04 + -1.180524541000000e+04 + -1.182188108744826e+04 + -1.183853437152247e+04 + -1.185520527417456e+04 + -1.187189381268968e+04 + -1.188860000000000e+04 + -1.190532384555031e+04 + -1.192206535739879e+04 + -1.193882454957090e+04 + -1.195560144455150e+04 + -1.197239604999999e+04 + -1.198920836801576e+04 + -1.200603841862060e+04 + -1.202288622005816e+04 + -1.203975178087160e+04 + -1.205663510999999e+04 + -1.207353621858988e+04 + -1.209045512337030e+04 + -1.210739183876488e+04 + -1.212434637372514e+04 + -1.214131874000000e+04 + -1.215830895146994e+04 + -1.217531702204386e+04 + -1.219234296344827e+04 + -1.220938678454126e+04 + -1.222644850000000e+04 + -1.224352812665298e+04 + -1.226062567426862e+04 + -1.227774115320455e+04 + -1.229487457740941e+04 + -1.231202595999999e+04 + -1.232919531361497e+04 + -1.234638265190763e+04 + -1.236358798719477e+04 + -1.238081132944854e+04 + -1.239805268999999e+04 + -1.241531208224867e+04 + -1.243258952373459e+04 + -1.244988502746729e+04 + -1.246719859854622e+04 + -1.248453025000000e+04 + -1.250187999937352e+04 + -1.251924785997314e+04 + -1.253663384322633e+04 + -1.255403795942784e+04 + -1.257146022000000e+04 + -1.258890063824787e+04 + -1.260635923174130e+04 + -1.262383601348307e+04 + -1.264133098848631e+04 + -1.265884416999999e+04 + -1.267637557565735e+04 + -1.269392521755929e+04 + -1.271149310837373e+04 + -1.272907926320349e+04 + -1.274668368999999e+04 + -1.276430639533002e+04 + -1.278194739917555e+04 + -1.279960671868295e+04 + -1.281728436173200e+04 + -1.283498034000000e+04 + -1.285269466774105e+04 + -1.287042735763161e+04 + -1.288817842259290e+04 + -1.290594787633268e+04 + -1.292373573000000e+04 + -1.294154199365508e+04 + -1.295936667981993e+04 + -1.297720980264566e+04 + -1.299507137776778e+04 + -1.301295141999999e+04 + -1.303084994203483e+04 + -1.304876695061188e+04 + -1.306670245535235e+04 + -1.308465647270176e+04 + -1.310262901999999e+04 + -1.312062011245361e+04 + -1.313862975604251e+04 + -1.315665796055223e+04 + -1.317470474462057e+04 + -1.319277012000000e+04 + -1.321085409533674e+04 + -1.322895668661698e+04 + -1.324707790969205e+04 + -1.326521777718879e+04 + -1.328337630000000e+04 + -1.330155348851537e+04 + -1.331974935491385e+04 + -1.333796391362056e+04 + -1.335619718140170e+04 + -1.337444916999999e+04 + -1.339271988874384e+04 + -1.341100935082290e+04 + -1.342931756987852e+04 + -1.344764455872234e+04 + -1.346599032999999e+04 + -1.348435489664480e+04 + -1.350273827293541e+04 + -1.352114047045672e+04 + -1.353956149695641e+04 + -1.355800137000000e+04 + -1.357646010972569e+04 + -1.359493772045508e+04 + -1.361343421245217e+04 + -1.363194960986463e+04 + -1.365048392000000e+04 + -1.366903714425879e+04 + -1.368760930691950e+04 + -1.370620042690972e+04 + -1.372481050688227e+04 + -1.374343955999999e+04 + -1.376208760473736e+04 + -1.378075465005087e+04 + -1.379944070672178e+04 + -1.381814579185646e+04 + -1.383686991999999e+04 + -1.385561310323773e+04 + -1.387437535187516e+04 + -1.389315667691767e+04 + -1.391195709130158e+04 + -1.393077661000000e+04 + -1.394961524865979e+04 + -1.396847302075987e+04 + -1.398734993803961e+04 + -1.400624601054151e+04 + -1.402516125000000e+04 + -1.404409566989268e+04 + -1.406304928589068e+04 + -1.408202211294199e+04 + -1.410101416389908e+04 + -1.412002544999999e+04 + -1.413905598164077e+04 + -1.415810576964489e+04 + -1.417717482985645e+04 + -1.419626318438561e+04 + -1.421537083999999e+04 + -1.423449779870650e+04 + -1.425364408441530e+04 + -1.427280971628262e+04 + -1.429199469812906e+04 + -1.431119904000000e+04 + -1.433042275698022e+04 + -1.434966586475864e+04 + -1.436892837784737e+04 + -1.438821030874347e+04 + -1.440751167000000e+04 + -1.442683247464153e+04 + -1.444617273693482e+04 + -1.446553246845352e+04 + -1.448491167696990e+04 + -1.450431037999999e+04 + -1.452372859766022e+04 + -1.454316633455064e+04 + -1.456262360064903e+04 + -1.458210041914065e+04 + -1.460159679999998e+04 + -1.462111274782618e+04 + -1.464064828275210e+04 + -1.466020342227678e+04 + -1.467977817411577e+04 + -1.469937255000000e+04 + -1.471898656425745e+04 + -1.473862022905967e+04 + -1.475827355783788e+04 + -1.477794656667906e+04 + -1.479763927000000e+04 + -1.481735168022591e+04 + -1.483708380678435e+04 + -1.485683566151257e+04 + -1.487660726097236e+04 + -1.489639861999999e+04 + -1.491620975095499e+04 + -1.493604066191429e+04 + -1.495589136614776e+04 + -1.497576188555746e+04 + -1.499565223000000e+04 + -1.501556240451360e+04 + -1.503549242791256e+04 + -1.505544231896391e+04 + -1.507541209051985e+04 + -1.509540175000000e+04 + -1.511541130465759e+04 + -1.513544077375174e+04 + -1.515549017500820e+04 + -1.517555951933053e+04 + -1.519564881999999e+04 + -1.521575809081157e+04 + -1.523588734003694e+04 + -1.525603658036919e+04 + -1.527620583277191e+04 + -1.529639510999998e+04 + -1.531660442034102e+04 + -1.533683377728779e+04 + -1.535708319443415e+04 + -1.537735268372974e+04 + -1.539764226000000e+04 + -1.541795193921473e+04 + -1.543828173411788e+04 + -1.545863165713528e+04 + -1.547900172144165e+04 + -1.549939194000000e+04 + -1.551980232569260e+04 + -1.554023289178161e+04 + -1.556068365093051e+04 + -1.558115461506463e+04 + -1.560164579999999e+04 + -1.562215722197032e+04 + -1.564268888875843e+04 + -1.566324081295743e+04 + -1.568381301679686e+04 + -1.570440550999999e+04 + -1.572501829745089e+04 + -1.574565139909988e+04 + -1.576630483256334e+04 + -1.578697860613842e+04 + -1.580767273000000e+04 + -1.582838721786272e+04 + -1.584912209052793e+04 + -1.586987736246908e+04 + -1.589065303680525e+04 + -1.591144913000000e+04 + -1.593226566336701e+04 + -1.595310264072689e+04 + -1.597396007256353e+04 + -1.599483798518879e+04 + -1.601573638999999e+04 + -1.603665529105288e+04 + -1.605759470511345e+04 + -1.607855464838671e+04 + -1.609953513161670e+04 + -1.612053616999998e+04 + -1.614155778004294e+04 + -1.616259997081388e+04 + -1.618366275348782e+04 + -1.620474614514243e+04 + -1.622585016000000e+04 + -1.624697481007329e+04 + -1.626812010728422e+04 + -1.628928606592670e+04 + -1.631047270337715e+04 + -1.633168003000000e+04 + -1.635290805454565e+04 + -1.637415679800609e+04 + -1.639542627592296e+04 + -1.641671649164588e+04 + -1.643802745999998e+04 + -1.645935920179283e+04 + -1.648071172935930e+04 + -1.650208505431875e+04 + -1.652347919039644e+04 + -1.654489414999998e+04 + -1.656632994566724e+04 + -1.658778659427021e+04 + -1.660926410974226e+04 + -1.663076250029903e+04 + -1.665228178000000e+04 + -1.667382196574960e+04 + -1.669538306946903e+04 + -1.671696510420377e+04 + -1.673856808645374e+04 + -1.676019203000000e+04 + -1.678183694695306e+04 + -1.680350285053965e+04 + -1.682518975366077e+04 + -1.684689766850574e+04 + -1.686862660999998e+04 + -1.689037659405139e+04 + -1.691214763326874e+04 + -1.693393974023558e+04 + -1.695575292860696e+04 + -1.697758720999999e+04 + -1.699944259677470e+04 + -1.702131910964586e+04 + -1.704321676352935e+04 + -1.706513556209216e+04 + -1.708707552000000e+04 + -1.710903665790321e+04 + -1.713101898938551e+04 + -1.715302252479822e+04 + -1.717504727292061e+04 + -1.719709325000000e+04 + -1.721916047545785e+04 + -1.724124896174670e+04 + -1.726335872072246e+04 + -1.728548976595536e+04 + -1.730764210999998e+04 + -1.732981576548871e+04 + -1.735201074834581e+04 + -1.737422707347338e+04 + -1.739646475299950e+04 + -1.741872379999999e+04 + -1.744100422776899e+04 + -1.746330604747148e+04 + -1.748562927386205e+04 + -1.750797392735042e+04 + -1.753034002000000e+04 + -1.755272756012299e+04 + -1.757513656385630e+04 + -1.759756704698003e+04 + -1.762001902165879e+04 + -1.764249250000000e+04 + -1.766498749503509e+04 + -1.768750402279969e+04 + -1.771004209797560e+04 + -1.773260173223158e+04 + -1.775518293999998e+04 + -1.777778573725219e+04 + -1.780041013823985e+04 + -1.782305615380817e+04 + -1.784572379125812e+04 + -1.786841306999998e+04 + -1.789112401297027e+04 + -1.791385662520795e+04 + -1.793661091551944e+04 + -1.795938690506571e+04 + -1.798218461000000e+04 + -1.800500404157250e+04 + -1.802784520726593e+04 + -1.805070812126097e+04 + -1.807359280836476e+04 + -1.809649928000000e+04 + -1.811942754077938e+04 + -1.814237760512274e+04 + -1.816534949154946e+04 + -1.818834322012563e+04 + -1.821135879999998e+04 + -1.823439623670011e+04 + -1.825745554979138e+04 + -1.828053675712656e+04 + -1.830363986886144e+04 + -1.832676489999998e+04 + -1.834991186721417e+04 + -1.837308077964562e+04 + -1.839627164834952e+04 + -1.841948449011883e+04 + -1.844271932000000e+04 + -1.846597615248214e+04 + -1.848925500551537e+04 + -1.851255589141731e+04 + -1.853587881367952e+04 + -1.855922379000000e+04 + -1.858259084403384e+04 + -1.860597998459446e+04 + -1.862939122130964e+04 + -1.865282457086503e+04 + -1.867628004999998e+04 + -1.869975767367864e+04 + -1.872325745125265e+04 + -1.874677939400045e+04 + -1.877032351829797e+04 + -1.879388983999998e+04 + -1.881747837395618e+04 + -1.884108913346766e+04 + -1.886472213066709e+04 + -1.888837737699991e+04 + -1.891205489000000e+04 + -1.893575468867888e+04 + -1.895947678210836e+04 + -1.898322118083019e+04 + -1.900698790146692e+04 + -1.903077696000000e+04 + -1.905458837122149e+04 + -1.907842214817702e+04 + -1.910227830328199e+04 + -1.912615684887913e+04 + -1.915005779999998e+04 + -1.917398117252360e+04 + -1.919792697876641e+04 + -1.922189523162585e+04 + -1.924588594641727e+04 + -1.926989913999998e+04 + -1.929393482787026e+04 + -1.931799301676958e+04 + -1.934207372057390e+04 + -1.936617696590223e+04 + -1.939030276000000e+04 + -1.941445110358184e+04 + -1.943862202401437e+04 + -1.946281554213054e+04 + -1.948703165938676e+04 + -1.951127038999998e+04 + -1.953553175531375e+04 + -1.955981576784185e+04 + -1.958412243887696e+04 + -1.960845178142548e+04 + -1.963280380999998e+04 + -1.965717854010317e+04 + -1.968157598764983e+04 + -1.970599616682758e+04 + -1.973043908928781e+04 + -1.975490477000000e+04 + -1.977939322508017e+04 + -1.980390446612710e+04 + -1.982843850672890e+04 + -1.985299536464853e+04 + -1.987757505000000e+04 + -1.990217757132781e+04 + -1.992680295149982e+04 + -1.995145120910892e+04 + -1.997612235106852e+04 + -2.000081638999998e+04 + -2.002553334291399e+04 + -2.005027322692212e+04 + -2.007503605456619e+04 + -2.009982183242756e+04 + -2.012463057999998e+04 + -2.014946232050470e+04 + -2.017431705750974e+04 + -2.019919480128925e+04 + -2.022409557869796e+04 + -2.024901940000000e+04 + -2.027396626876084e+04 + -2.029893620825063e+04 + -2.032392923791251e+04 + -2.034894536415857e+04 + -2.037398460000000e+04 + -2.039904696273123e+04 + -2.042413246681671e+04 + -2.044924112749899e+04 + -2.047437296177629e+04 + -2.049952797999998e+04 + -2.052470619105544e+04 + -2.054990761544010e+04 + -2.057513226917888e+04 + -2.060038015795331e+04 + -2.062565129999998e+04 + -2.065094571854568e+04 + -2.067626342205840e+04 + -2.070160442063917e+04 + -2.072696873225083e+04 + -2.075235637000000e+04 + -2.077776734528838e+04 + -2.080320167696219e+04 + -2.082865938214449e+04 + -2.085414047244347e+04 + -2.087964496000000e+04 + -2.090517285805777e+04 + -2.093072418166662e+04 + -2.095629894648643e+04 + -2.098189716820891e+04 + -2.100751885999998e+04 + -2.103316403410750e+04 + -2.105883270565323e+04 + -2.108452489048020e+04 + -2.111024060421226e+04 + -2.113597985999998e+04 + -2.116174267011207e+04 + -2.118752904964539e+04 + -2.121333901446366e+04 + -2.123917258027344e+04 + -2.126502976000000e+04 + -2.129091056582378e+04 + -2.131681501406569e+04 + -2.134274311932949e+04 + -2.136869489244911e+04 + -2.139467035000000e+04 + -2.142066951097057e+04 + -2.144669238829770e+04 + -2.147273899344209e+04 + -2.149880933837191e+04 + -2.152490343999998e+04 + -2.155102131698669e+04 + -2.157716298229367e+04 + -2.160332844744082e+04 + -2.162951772437019e+04 + -2.165573082999998e+04 + -2.168196778300156e+04 + -2.170822859627157e+04 + -2.173451328139519e+04 + -2.176082185053812e+04 + -2.178715432000000e+04 + -2.181351070821111e+04 + -2.183989103150391e+04 + -2.186629530288904e+04 + -2.189272353140778e+04 + -2.191917573000000e+04 + -2.194565191497987e+04 + -2.197215210512397e+04 + -2.199867631637375e+04 + -2.202522455969623e+04 + -2.205179684999998e+04 + -2.207839320363563e+04 + -2.210501363148952e+04 + -2.213165814783550e+04 + -2.215832677350680e+04 + -2.218501951999998e+04 + -2.221173639532642e+04 + -2.223847741910094e+04 + -2.226524260850044e+04 + -2.229203197306979e+04 + -2.231884553000000e+04 + -2.234568329890952e+04 + -2.237254528774074e+04 + -2.239943150806158e+04 + -2.242634198094407e+04 + -2.245327672000000e+04 + -2.248023573585785e+04 + -2.250721904853940e+04 + -2.253422667413636e+04 + -2.256125861967753e+04 + -2.258831489999998e+04 + -2.261539553480252e+04 + -2.264250054093014e+04 + -2.266962993052693e+04 + -2.269678371062339e+04 + -2.272396189999998e+04 + -2.275116452153000e+04 + -2.277839158301363e+04 + -2.280564309534358e+04 + -2.283291907950545e+04 + -2.286021955000000e+04 + -2.288754451771620e+04 + -2.291489399812011e+04 + -2.294226800704672e+04 + -2.296966655917324e+04 + -2.299708967000000e+04 + -2.302453735490683e+04 + -2.305200962641978e+04 + -2.307950649734620e+04 + -2.310702798236498e+04 + -2.313457409999998e+04 + -2.316214486856879e+04 + -2.318974029613145e+04 + -2.321736039447026e+04 + -2.324500518448656e+04 + -2.327267467999998e+04 + -2.330036889175501e+04 + -2.332808783850466e+04 + -2.335583153764027e+04 + -2.338360000121357e+04 + -2.341139324000000e+04 + -2.343921126672951e+04 + -2.346705410394532e+04 + -2.349492176951476e+04 + -2.352281427072812e+04 + -2.355073162000000e+04 + -2.357867383414989e+04 + -2.360664093193761e+04 + -2.363463292919098e+04 + -2.366264983684289e+04 + -2.369069166999998e+04 + -2.371875844507638e+04 + -2.374685017193912e+04 + -2.377496686591387e+04 + -2.380310855200050e+04 + -2.383127523999998e+04 + -2.385946693403709e+04 + -2.388768365690810e+04 + -2.391592542840760e+04 + -2.394419225713039e+04 + -2.397248416000000e+04 + -2.400080115691363e+04 + -2.402914325571854e+04 + -2.405751046800998e+04 + -2.408590281513924e+04 + -2.411432031000000e+04 + -2.414276296293371e+04 + -2.417123079796694e+04 + -2.419972383203242e+04 + -2.422824206706797e+04 + -2.425678551999998e+04 + -2.428535421485334e+04 + -2.431394816200073e+04 + -2.434256737430352e+04 + -2.437121187300517e+04 + -2.439988166999998e+04 + -2.442857677325164e+04 + -2.445729720103540e+04 + -2.448604297194451e+04 + -2.451481410082247e+04 + -2.454361060000000e+04 + -2.457243248182022e+04 + -2.460127976431730e+04 + -2.463015246522100e+04 + -2.465905059942621e+04 + -2.468797417999998e+04 + -2.471692321945616e+04 + -2.474589773222362e+04 + -2.477489773480251e+04 + -2.480392324581322e+04 + -2.483297427999998e+04 + -2.486205084917968e+04 + -2.489115296405446e+04 + -2.492028064024748e+04 + -2.494943390068509e+04 + -2.497861275999998e+04 + -2.500781722863925e+04 + -2.503704732334474e+04 + -2.506630305872901e+04 + -2.509558444448151e+04 + -2.512489150000000e+04 + -2.515422424757818e+04 + -2.518358269497819e+04 + -2.521296685328307e+04 + -2.524237674371633e+04 + -2.527181237999998e+04 + -2.530127377273551e+04 + -2.533076094162915e+04 + -2.536027390311031e+04 + -2.538981266594794e+04 + -2.541937724999998e+04 + -2.544896767801531e+04 + -2.547858395406606e+04 + -2.550822608802068e+04 + -2.553789410507042e+04 + -2.556758802000000e+04 + -2.559730784246711e+04 + -2.562705359189478e+04 + -2.565682528525429e+04 + -2.568662293247029e+04 + -2.571644655000000e+04 + -2.574629615713407e+04 + -2.577617176604459e+04 + -2.580607338891535e+04 + -2.583600104095400e+04 + -2.586595473999998e+04 + -2.589593450400127e+04 + -2.592594034522632e+04 + -2.595597227610649e+04 + -2.598603031176017e+04 + -2.601611446999998e+04 + -2.604622476879967e+04 + -2.607636122042706e+04 + -2.610652383730750e+04 + -2.613671263455693e+04 + -2.616692763000000e+04 + -2.619716884161440e+04 + -2.622743628160562e+04 + -2.625772996246335e+04 + -2.628804989951923e+04 + -2.631839611000000e+04 + -2.634876861165059e+04 + -2.637916741999917e+04 + -2.640959254907723e+04 + -2.644004401148305e+04 + -2.647052181999998e+04 + -2.650102598983489e+04 + -2.653155654491748e+04 + -2.656211350103963e+04 + -2.659269685960091e+04 + -2.662330663999998e+04 + -2.665394286925502e+04 + -2.668460555517195e+04 + -2.671529470758218e+04 + -2.674601034670874e+04 + -2.677675249000000e+04 + -2.680752115154265e+04 + -2.683831634157976e+04 + -2.686913807564717e+04 + -2.689998637786448e+04 + -2.693086126000000e+04 + -2.696176272916689e+04 + -2.699269080763921e+04 + -2.702364551500381e+04 + -2.705462686113689e+04 + -2.708563485999998e+04 + -2.711666952809900e+04 + -2.714773087942122e+04 + -2.717881892991400e+04 + -2.720993369901594e+04 + -2.724107519999998e+04 + -2.727224344400496e+04 + -2.730343845029767e+04 + -2.733466023562860e+04 + -2.736590881019403e+04 + -2.739718419000000e+04 + -2.742848639412166e+04 + -2.745981543741902e+04 + -2.749117133427855e+04 + -2.752255410001475e+04 + -2.755396375000000e+04 + -2.758540029934807e+04 + -2.761686376223195e+04 + -2.764835415476579e+04 + -2.767987149599797e+04 + -2.771141579999997e+04 + -2.774298707825893e+04 + -2.777458534547961e+04 + -2.780621061813955e+04 + -2.783786291408934e+04 + -2.786954224999998e+04 + -2.790124864062599e+04 + -2.793298209650868e+04 + -2.796474263269793e+04 + -2.799653027215609e+04 + -2.802834503000000e+04 + -2.806018691678819e+04 + -2.809205594694776e+04 + -2.812395213748840e+04 + -2.815587550739401e+04 + -2.818782607000000e+04 + -2.821980383662756e+04 + -2.825180882556652e+04 + -2.828384105454396e+04 + -2.831590053775406e+04 + -2.834798728999998e+04 + -2.838010132665882e+04 + -2.841224266303304e+04 + -2.844441131398464e+04 + -2.847660729395342e+04 + -2.850883061999998e+04 + -2.854108131015271e+04 + -2.857335937943549e+04 + -2.860566484233479e+04 + -2.863799771378746e+04 + -2.867035801000000e+04 + -2.870274574643904e+04 + -2.873516093256026e+04 + -2.876760358436649e+04 + -2.880007372909855e+04 + -2.883257138000000e+04 + -2.886509654365616e+04 + -2.889764923870952e+04 + -2.893022948372628e+04 + -2.896283729252707e+04 + -2.899547267999998e+04 + -2.902813566168059e+04 + -2.906082625190441e+04 + -2.909354446650575e+04 + -2.912629032389195e+04 + -2.915906383999998e+04 + -2.919186502919167e+04 + -2.922469390650106e+04 + -2.925755048733507e+04 + -2.929043478725063e+04 + -2.932334682000000e+04 + -2.935628660019129e+04 + -2.938925415043921e+04 + -2.942224948806936e+04 + -2.945527261979538e+04 + -2.948832356000000e+04 + -2.952140232838030e+04 + -2.955450894361826e+04 + -2.958764342089699e+04 + -2.962080577090842e+04 + -2.965399600999997e+04 + -2.968721415715336e+04 + -2.972046022629084e+04 + -2.975373423280592e+04 + -2.978703619592538e+04 + -2.982036612999997e+04 + -2.985372404652352e+04 + -2.988710995909819e+04 + -2.992052388535540e+04 + -2.995396584761164e+04 + -2.998743586000000e+04 + -3.002093393306212e+04 + -3.005446008516229e+04 + -3.008801433415196e+04 + -3.012159669410924e+04 + -3.015520717999998e+04 + -3.018884580728562e+04 + -3.022251259030155e+04 + -3.025620754490144e+04 + -3.028993068950620e+04 + -3.032368203999997e+04 + -3.035746161072607e+04 + -3.039126941699663e+04 + -3.042510547393189e+04 + -3.045896979612689e+04 + -3.049286239999998e+04 + -3.052678330335746e+04 + -3.056073252467363e+04 + -3.059471007984014e+04 + -3.062871598061521e+04 + -3.066275024000000e+04 + -3.069681287356475e+04 + -3.073090390293319e+04 + -3.076502334621974e+04 + -3.079917121404217e+04 + -3.083334751999997e+04 + -3.086755228023463e+04 + -3.090178551167921e+04 + -3.093604723225209e+04 + -3.097033746063074e+04 + -3.100465620999998e+04 + -3.103900349175437e+04 + -3.107337932446759e+04 + -3.110778372555016e+04 + -3.114221670819769e+04 + -3.117667828999997e+04 + -3.121116848957679e+04 + -3.124568731744623e+04 + -3.128023478769976e+04 + -3.131481092268889e+04 + -3.134941574000000e+04 + -3.138404925321087e+04 + -3.141871147408727e+04 + -3.145340241670125e+04 + -3.148812209933996e+04 + -3.152287053999997e+04 + -3.155764775577736e+04 + -3.159245376185456e+04 + -3.162728857279384e+04 + -3.166215220315186e+04 + -3.169704466999997e+04 + -3.173196599132658e+04 + -3.176691618227545e+04 + -3.180189525721778e+04 + -3.183690323068162e+04 + -3.187194012000000e+04 + -3.190700594323912e+04 + -3.194210071422084e+04 + -3.197722444868163e+04 + -3.201237716636405e+04 + -3.204755888000000e+04 + -3.208276960047787e+04 + -3.211800935023939e+04 + -3.215327814773632e+04 + -3.218857600158763e+04 + -3.222390292999997e+04 + -3.225925895535199e+04 + -3.229464408973093e+04 + -3.233005834462857e+04 + -3.236550173499735e+04 + -3.240097427999997e+04 + -3.243647600017841e+04 + -3.247200691143220e+04 + -3.250756702669824e+04 + -3.254315635678932e+04 + -3.257877492000000e+04 + -3.261442273792899e+04 + -3.265009982555360e+04 + -3.268580619650297e+04 + -3.272154186523808e+04 + -3.275730685000000e+04 + -3.279310116911321e+04 + -3.282892483177912e+04 + -3.286477785354587e+04 + -3.290066026202923e+04 + -3.293657206999998e+04 + -3.297251328378459e+04 + -3.300848392497173e+04 + -3.304448401407080e+04 + -3.308051356412061e+04 + -3.311657258999998e+04 + -3.315266110752813e+04 + -3.318877912993135e+04 + -3.322492667413289e+04 + -3.326110376297835e+04 + -3.329731041000000e+04 + -3.333354662539115e+04 + -3.336981243131684e+04 + -3.340610784669931e+04 + -3.344243288124504e+04 + -3.347878755000000e+04 + -3.351517187197135e+04 + -3.355158586607149e+04 + -3.358802954742114e+04 + -3.362450292603845e+04 + -3.366100601999997e+04 + -3.369753885113907e+04 + -3.373410143432544e+04 + -3.377069378324004e+04 + -3.380731591267414e+04 + -3.384396783999997e+04 + -3.388064958319757e+04 + -3.391736115628571e+04 + -3.395410257481684e+04 + -3.399087385786547e+04 + -3.402767502000000e+04 + -3.406450607498593e+04 + -3.410136704569621e+04 + -3.413825794940452e+04 + -3.417517879208190e+04 + -3.421212959000000e+04 + -3.424911036577375e+04 + -3.428612113814977e+04 + -3.432316192150666e+04 + -3.436023272579379e+04 + -3.439733356999998e+04 + -3.443446447621852e+04 + -3.447162545467578e+04 + -3.450881651888796e+04 + -3.454603769159935e+04 + -3.458328898999997e+04 + -3.462057042742187e+04 + -3.465788201822977e+04 + -3.469522377879945e+04 + -3.473259572783916e+04 + -3.476999788000000e+04 + -3.480743024920274e+04 + -3.484489285737491e+04 + -3.488238572274076e+04 + -3.491990885516477e+04 + -3.495746227000000e+04 + -3.499504598627476e+04 + -3.503266002163332e+04 + -3.507030439252002e+04 + -3.510797911399110e+04 + -3.514568419999997e+04 + -3.518341966538968e+04 + -3.522118553105182e+04 + -3.525898181605689e+04 + -3.529680853433678e+04 + -3.533466569999997e+04 + -3.537255332834070e+04 + -3.541047143775194e+04 + -3.544842004484196e+04 + -3.548639916273562e+04 + -3.552440881000000e+04 + -3.556244900769237e+04 + -3.560051977192599e+04 + -3.563862111528000e+04 + -3.567675304773126e+04 + -3.571491559000000e+04 + -3.575310876677514e+04 + -3.579133259025522e+04 + -3.582958707123636e+04 + -3.586787222387752e+04 + -3.590618806999997e+04 + -3.594453463328037e+04 + -3.598291192576281e+04 + -3.602131995866808e+04 + -3.605975874687618e+04 + -3.609822830999997e+04 + -3.613672866872384e+04 + -3.617525983663845e+04 + -3.621382182906358e+04 + -3.625241466608051e+04 + -3.629103836000000e+04 + -3.632969292147818e+04 + -3.636837837559645e+04 + -3.640709474306420e+04 + -3.644584203277781e+04 + -3.648462026000000e+04 + -3.652342944409004e+04 + -3.656226960170399e+04 + -3.660114075023598e+04 + -3.664004290875195e+04 + -3.667897608999997e+04 + -3.671794030512006e+04 + -3.675693557547587e+04 + -3.679596192058374e+04 + -3.683501935339895e+04 + -3.687410788999997e+04 + -3.691322754870546e+04 + -3.695237834723054e+04 + -3.699156030212950e+04 + -3.703077342834224e+04 + -3.707001774000000e+04 + -3.710929325203417e+04 + -3.714859998429457e+04 + -3.718793795692067e+04 + -3.722730718808447e+04 + -3.726670768999997e+04 + -3.730613947399677e+04 + -3.734560256273038e+04 + -3.738509697431393e+04 + -3.742462271654350e+04 + -3.746417980999997e+04 + -3.750376828013417e+04 + -3.754338813660564e+04 + -3.758303939267375e+04 + -3.762272207245150e+04 + -3.766243618999997e+04 + -3.770218175518502e+04 + -3.774195878945678e+04 + -3.778176731257176e+04 + -3.782160733741773e+04 + -3.786147888000000e+04 + -3.790138195864309e+04 + -3.794131659132003e+04 + -3.798128279431332e+04 + -3.802128058166662e+04 + -3.806130996999997e+04 + -3.810137097737330e+04 + -3.814146362046005e+04 + -3.818158791697343e+04 + -3.822174388615621e+04 + -3.826193153999997e+04 + -3.830215088938532e+04 + -3.834240195971708e+04 + -3.838268477130731e+04 + -3.842299933188884e+04 + -3.846334565999997e+04 + -3.850372377842891e+04 + -3.854413369633216e+04 + -3.858457542816681e+04 + -3.862504900090371e+04 + -3.866555443000000e+04 + -3.870609172486966e+04 + -3.874666090332934e+04 + -3.878726198409722e+04 + -3.882789498399841e+04 + -3.886855991999997e+04 + -3.890925680946781e+04 + -3.894998567054054e+04 + -3.899074651957592e+04 + -3.903153937021765e+04 + -3.907236423999997e+04 + -3.911322114736439e+04 + -3.915411010384250e+04 + -3.919503112747372e+04 + -3.923598424716172e+04 + -3.927696946999997e+04 + -3.931798680038228e+04 + -3.935903628964794e+04 + -3.940011793259103e+04 + -3.944123164809230e+04 + -3.948237770000000e+04 + -3.952355620675491e+04 + -3.956476584662273e+04 + -3.960600801403134e+04 + -3.964728735507886e+04 + -3.968859104999997e+04 + 1.819886919039960e+01 + 1.816405130921966e+01 + 1.812918280091599e+01 + 1.809426424653407e+01 + 1.805929622711938e+01 + 1.802427932371738e+01 + 1.798921411737356e+01 + 1.795410118913339e+01 + 1.791894112004234e+01 + 1.788373449114589e+01 + 1.784848188348950e+01 + 1.781318387811866e+01 + 1.777784105607885e+01 + 1.774245399841551e+01 + 1.770702328617416e+01 + 1.767154950040025e+01 + 1.763603322213927e+01 + 1.760047503243666e+01 + 1.756487551233793e+01 + 1.752923524288855e+01 + 1.749355480513398e+01 + 1.745783478011969e+01 + 1.742207574889119e+01 + 1.738627829249391e+01 + 1.735044299197335e+01 + 1.731457042837498e+01 + 1.727866118274428e+01 + 1.724271583612671e+01 + 1.720673496956776e+01 + 1.717071916411290e+01 + 1.713466900080759e+01 + 1.709858506069732e+01 + 1.706246792482757e+01 + 1.702631817424381e+01 + 1.699013638999150e+01 + 1.695392315311613e+01 + 1.691767904466317e+01 + 1.688140464567809e+01 + 1.684510053720637e+01 + 1.680876730029348e+01 + 1.677240551598491e+01 + 1.673601576532611e+01 + 1.669959862936258e+01 + 1.666315468913977e+01 + 1.662668452570318e+01 + 1.659018872009826e+01 + 1.655366785337049e+01 + 1.651712250656536e+01 + 1.648055326072833e+01 + 1.644396069690488e+01 + 1.640734539614049e+01 + 1.637070793948063e+01 + 1.633404890797076e+01 + 1.629736888265638e+01 + 1.626066844458294e+01 + 1.622394817479593e+01 + 1.618720865434082e+01 + 1.615045046426309e+01 + 1.611367418560821e+01 + 1.607688039942165e+01 + 1.604006968674890e+01 + 1.600324262863542e+01 + 1.596639980612668e+01 + 1.592954180026817e+01 + 1.589266919210536e+01 + 1.585578256268371e+01 + 1.581888249304872e+01 + 1.578196956424585e+01 + 1.574504435732057e+01 + 1.570810745331836e+01 + 1.567115943328470e+01 + 1.563420087826506e+01 + 1.559723236930490e+01 + 1.556025448744972e+01 + 1.552326781374499e+01 + 1.548627292923617e+01 + 1.544927041496875e+01 + 1.541226085198818e+01 + 1.537524482133997e+01 + 1.533822290406956e+01 + 1.530119568122245e+01 + 1.526416373384410e+01 + 1.522712764298000e+01 + 1.519008798967561e+01 + 1.515304535497640e+01 + 1.511600031992786e+01 + 1.507895346557546e+01 + 1.504190537296467e+01 + 1.500485662314096e+01 + 1.496780779714983e+01 + 1.493075947603672e+01 + 1.489371224084713e+01 + 1.485666667262652e+01 + 1.481962335242037e+01 + 1.478258286127416e+01 + 1.474554578023335e+01 + 1.470851269034343e+01 + 1.467148417264987e+01 + 1.463446080819813e+01 + 1.459744317803371e+01 + 1.456043186320207e+01 + 1.452342744474869e+01 + 1.448643050371903e+01 + 1.444944162115858e+01 + 1.441246137811282e+01 + 1.437549035562720e+01 + 1.433852913474722e+01 + 1.430157829651834e+01 + 1.426463842198604e+01 + 1.422771009219579e+01 + 1.419079388819307e+01 + 1.415389039102334e+01 + 1.411700018173210e+01 + 1.408012384136481e+01 + 1.404326195096694e+01 + 1.400641509158398e+01 + 1.396958384426139e+01 + 1.393276879004464e+01 + 1.389597050997923e+01 + 1.385918958511061e+01 + 1.382242659648426e+01 + 1.378568212514566e+01 + 1.374895675214029e+01 + 1.371225105851361e+01 + 1.367556562531110e+01 + 1.363890103357824e+01 + 1.360225786436050e+01 + 1.356563669870335e+01 + 1.352903811765228e+01 + 1.349246270225274e+01 + 1.345591103355023e+01 + 1.341938369259021e+01 + 1.338288126041816e+01 + 1.334640431807955e+01 + 1.330995344661985e+01 + 1.327352922708455e+01 + 1.323713224051912e+01 + 1.320076306796902e+01 + 1.316442229047974e+01 + 1.312811048909675e+01 + 1.309182824486553e+01 + 1.305557613883154e+01 + 1.301935475204027e+01 + 1.298316466553718e+01 + 1.294700646036776e+01 + 1.291088071757748e+01 + 1.287478801821181e+01 + 1.283872894331622e+01 + 1.280270407393619e+01 + 1.276671399111720e+01 + 1.273075927590472e+01 + 1.269484050934423e+01 + 1.265895827248119e+01 + 1.262311314636109e+01 + 1.258730571202939e+01 + 1.255153655053158e+01 + 1.251580624291313e+01 + 1.248011537021950e+01 + 1.244446451349619e+01 + 1.240885425378865e+01 + 1.237328517214237e+01 + 1.233775784960283e+01 + 1.230227286721548e+01 + 1.226683080602582e+01 + 1.223143224707931e+01 + 1.219607777142142e+01 + 1.216076796009764e+01 + 1.212550339415344e+01 + 1.209028465463429e+01 + 1.205511232258567e+01 + 1.201998697905305e+01 + 1.198490920508190e+01 + 1.194987958171770e+01 + 1.191489869000593e+01 + 1.187996711099206e+01 + 1.184508542572157e+01 + 1.181025421523992e+01 + 1.177547406059259e+01 + 1.174074554282507e+01 + 1.170606924298281e+01 + 1.167144574211130e+01 + 1.163687562125602e+01 + 1.160235946146243e+01 + 1.156789784377601e+01 + 1.153349134924223e+01 + 1.149914055890657e+01 + 1.146484605381452e+01 + 1.143060841501152e+01 + 1.139642822354308e+01 + 1.136230606045464e+01 + 1.132824250679170e+01 + 1.129423814359974e+01 + 1.126029355192421e+01 + 1.122640931281059e+01 + 1.119258600730437e+01 + 1.115882421645102e+01 + 1.112512452129600e+01 + 1.109148750288479e+01 + 1.105791374226288e+01 + 1.102440382047573e+01 + 1.099095831856882e+01 + 1.095757781758762e+01 + 1.092426289857761e+01 + 1.089101414258426e+01 + 1.085783213065305e+01 + 1.082471744382945e+01 + 1.079167066315892e+01 + 1.075869236968697e+01 + 1.072578314445905e+01 + 1.069294356852064e+01 + 1.066017422291721e+01 + 1.062747568869424e+01 + 1.059484854689720e+01 + 1.056229337857157e+01 + 1.052981076476282e+01 + 1.049740128651643e+01 + 1.046506552487787e+01 + 1.043280406089261e+01 + 1.040061747560614e+01 + 1.036850635006392e+01 + 1.033647126531143e+01 + 1.030451280239414e+01 + 1.027263154235754e+01 + 1.024082806624708e+01 + 1.020910295510825e+01 + 1.017745678998653e+01 + 1.014589015192738e+01 + 1.011440362197629e+01 + 1.008299778117872e+01 + 1.005167321058014e+01 + 1.002043049122605e+01 + 9.989270204161905e+00 + 9.958192930433185e+00 + 9.927199251085362e+00 + 9.896289747163912e+00 + 9.865464999714312e+00 + 9.834725589782032e+00 + 9.804072098412551e+00 + 9.773505106651340e+00 + 9.743025195543877e+00 + 9.712632946135633e+00 + 9.682328939472086e+00 + 9.652113756598707e+00 + 9.621987978560973e+00 + 9.591952186404358e+00 + 9.562006961174337e+00 + 9.532152883916382e+00 + 9.502390535675973e+00 + 9.472720497498580e+00 + 9.443143350429677e+00 + 9.413659675514742e+00 + 9.384270053799249e+00 + 9.354975066328668e+00 + 9.325775294148480e+00 + 9.296671318304156e+00 + 9.267663719841170e+00 + 9.238753079804999e+00 + 9.209939979241115e+00 + 9.181224999194994e+00 + 9.152608720712111e+00 + 9.124091724837941e+00 + 9.095674592617954e+00 + 9.067357905097630e+00 + 9.039142243322441e+00 + 9.011028188337864e+00 + 8.983016292445305e+00 + 8.955106689123129e+00 + 8.927299195109086e+00 + 8.899593619314148e+00 + 8.871989770649284e+00 + 8.844487458025464e+00 + 8.817086490353660e+00 + 8.789786676544839e+00 + 8.762587825509975e+00 + 8.735489746160036e+00 + 8.708492247405990e+00 + 8.681595138158814e+00 + 8.654798227329470e+00 + 8.628101323828931e+00 + 8.601504236568170e+00 + 8.575006774458155e+00 + 8.548608746409855e+00 + 8.522309961334242e+00 + 8.496110228142285e+00 + 8.470009355744955e+00 + 8.444007153053219e+00 + 8.418103428978053e+00 + 8.392297992430423e+00 + 8.366590652321301e+00 + 8.340981217561657e+00 + 8.315469497062459e+00 + 8.290055299734679e+00 + 8.264738434489287e+00 + 8.239518710237252e+00 + 8.214395935889545e+00 + 8.189369920357137e+00 + 8.164440472550996e+00 + 8.139607401382095e+00 + 8.114870515761401e+00 + 8.090229624599887e+00 + 8.065684536808520e+00 + 8.041235061298275e+00 + 8.016881006980118e+00 + 7.992622182765018e+00 + 7.968458397563950e+00 + 7.944389460287880e+00 + 7.920415179847782e+00 + 7.896535365154623e+00 + 7.872749825119373e+00 + 7.849058368653004e+00 + 7.825460804666484e+00 + 7.801956942070785e+00 + 7.778546589776878e+00 + 7.755229556695729e+00 + 7.732005651738313e+00 + 7.708874683815599e+00 + 7.685836461838553e+00 + 7.662890794718152e+00 + 7.640037491365358e+00 + 7.617276360691148e+00 + 7.594607211606489e+00 + 7.572029853022352e+00 + 7.549544093849708e+00 + 7.527149742999527e+00 + 7.504846609382777e+00 + 7.482634501910429e+00 + 7.460513229493455e+00 + 7.438482601042820e+00 + 7.416542425469502e+00 + 7.394692511684465e+00 + 7.372932668598682e+00 + 7.351262705123121e+00 + 7.329682430168755e+00 + 7.308191652646553e+00 + 7.286790181467484e+00 + 7.265477825542517e+00 + 7.244254393782626e+00 + 7.223119695098779e+00 + 7.202073538401945e+00 + 7.181115732603097e+00 + 7.160246086613204e+00 + 7.139464409343233e+00 + 7.118770509704159e+00 + 7.098164196606950e+00 + 7.077645278962576e+00 + 7.057213565682006e+00 + 7.036868865676214e+00 + 7.016610987856166e+00 + 6.996439741132834e+00 + 6.976354934417186e+00 + 6.956356376620197e+00 + 6.936443876652832e+00 + 6.916617243426063e+00 + 6.896876285850862e+00 + 6.877220812838198e+00 + 6.857650633299039e+00 + 6.838165556144358e+00 + 6.818765390285123e+00 + 6.799449944632306e+00 + 6.780219028096876e+00 + 6.761072449589803e+00 + 6.742010018022058e+00 + 6.723031542304612e+00 + 6.704136831348434e+00 + 6.685325694064492e+00 + 6.666597939363759e+00 + 6.647953376157203e+00 + 6.629391813355797e+00 + 6.610913059870509e+00 + 6.592516924612310e+00 + 6.574203216492170e+00 + 6.555971744421058e+00 + 6.537822317309946e+00 + 6.519754744069803e+00 + 6.501768833611600e+00 + 6.483864394846306e+00 + 6.466041236684891e+00 + 6.448299168038327e+00 + 6.430637997817582e+00 + 6.413057534933630e+00 + 6.395557588297437e+00 + 6.378137966819973e+00 + 6.360798479412210e+00 + 6.343538934985117e+00 + 6.326359142449666e+00 + 6.309258910716826e+00 + 6.292238048697567e+00 + 6.275296365302859e+00 + 6.258433669443673e+00 + 6.241649770030978e+00 + 6.224944475975745e+00 + 6.208317596188945e+00 + 6.191768939581546e+00 + 6.175298315064517e+00 + 6.158905531548833e+00 + 6.142590397945462e+00 + 6.126352723165372e+00 + 6.110192316119535e+00 + 6.094108985718921e+00 + 6.078102540874499e+00 + 6.062172790497243e+00 + 6.046319543498117e+00 + 6.030542608788095e+00 + 6.014841795278149e+00 + 5.999216911879244e+00 + 5.983667767502356e+00 + 5.968194171058448e+00 + 5.952795931458497e+00 + 5.937472857613470e+00 + 5.922224758434336e+00 + 5.907051442832068e+00 + 5.891952719717633e+00 + 5.876928398002004e+00 + 5.861978286596150e+00 + 5.847102194411042e+00 + 5.832299930357648e+00 + 5.817571303346941e+00 + 5.802916122289888e+00 + 5.788334196097460e+00 + 5.773825333680629e+00 + 5.759389343950365e+00 + 5.745026035817635e+00 + 5.730735218193414e+00 + 5.716516699988668e+00 + 5.702370290114368e+00 + 5.688295797481486e+00 + 5.674293031000990e+00 + 5.660361799583851e+00 + 5.646501912141039e+00 + 5.632713177583526e+00 + 5.618995404822279e+00 + 5.605348402768269e+00 + 5.591771980332467e+00 + 5.578265946425844e+00 + 5.564830109959368e+00 + 5.551464279844009e+00 + 5.538168264990741e+00 + 5.524941874310529e+00 + 5.511784916714348e+00 + 5.498697201113163e+00 + 5.485678536417949e+00 + 5.472728731539672e+00 + 5.459847595389306e+00 + 5.447031950668924e+00 + 5.434258417587328e+00 + 5.421509378931341e+00 + 5.408790202589383e+00 + 5.396101725502927e+00 + 5.383442552149385e+00 + 5.370812984345576e+00 + 5.358213038980171e+00 + 5.345642546622406e+00 + 5.333101463094469e+00 + 5.320589727671893e+00 + 5.308107264570896e+00 + 5.295654006874094e+00 + 5.283229887334054e+00 + 5.270834837715376e+00 + 5.258468789944041e+00 + 5.246131676000384e+00 + 5.233823428164784e+00 + 5.221543979540164e+00 + 5.209293263422158e+00 + 5.197071212820756e+00 + 5.184877759775030e+00 + 5.172712837552656e+00 + 5.160576380473933e+00 + 5.148468322098453e+00 + 5.136388595958436e+00 + 5.124337135726472e+00 + 5.112313875048248e+00 + 5.100318748362747e+00 + 5.088351690727316e+00 + 5.076412636057685e+00 + 5.064501518839616e+00 + 5.052618274474592e+00 + 5.040762837425790e+00 + 5.028935142799786e+00 + 5.017135126690616e+00 + 5.005362723409764e+00 + 4.993617868159293e+00 + 4.981900498164465e+00 + 4.970210548621045e+00 + 4.958547954855285e+00 + 4.946912653797631e+00 + 4.935304581602165e+00 + 4.923723674430540e+00 + 4.912169869219803e+00 + 4.900643102722201e+00 + 4.889143311682699e+00 + 4.877670433140317e+00 + 4.866224404407401e+00 + 4.854805162838305e+00 + 4.843412645620055e+00 + 4.832046790470948e+00 + 4.820707535433044e+00 + 4.809394818449872e+00 + 4.798108577205163e+00 + 4.786848749644187e+00 + 4.775615274885523e+00 + 4.764408091170769e+00 + 4.753227136475494e+00 + 4.742072350593207e+00 + 4.730943672320340e+00 + 4.719841039761676e+00 + 4.708764392793848e+00 + 4.697713670947889e+00 + 4.686688813308958e+00 + 4.675689760077749e+00 + 4.664716450776882e+00 + 4.653768824355144e+00 + 4.642846821849820e+00 + 4.631950383864691e+00 + 4.621079449911801e+00 + 4.610233960271208e+00 + 4.599413855814052e+00 + 4.588619077726644e+00 + 4.577849566594633e+00 + 4.567105263324451e+00 + 4.556386109499332e+00 + 4.545692045920192e+00 + 4.535023013830533e+00 + 4.524378955558838e+00 + 4.513759812087895e+00 + 4.503165524982657e+00 + 4.492596037765430e+00 + 4.482051291649467e+00 + 4.471531227881555e+00 + 4.461035790191556e+00 + 4.450564920902143e+00 + 4.440118561957067e+00 + 4.429696656605228e+00 + 4.419299148008472e+00 + 4.408925979093191e+00 + 4.398577092681390e+00 + 4.388252432408600e+00 + 4.377951941955149e+00 + 4.367675563647617e+00 + 4.357423241911437e+00 + 4.347194921954801e+00 + 4.336990545878941e+00 + 4.326810057907454e+00 + 4.316653403787664e+00 + 4.306520526395858e+00 + 4.296411370095723e+00 + 4.286325880845187e+00 + 4.276264002847886e+00 + 4.266225680614548e+00 + 4.256210859410026e+00 + 4.246219484255977e+00 + 4.236251500111860e+00 + 4.226306852214246e+00 + 4.216385487075638e+00 + 4.206487350454588e+00 + 4.196612386873075e+00 + 4.186760542346591e+00 + 4.176931763412669e+00 + 4.167125996517556e+00 + 4.157343187931778e+00 + 4.147583283659849e+00 + 4.137846229598449e+00 + 4.128131973158434e+00 + 4.118440461620333e+00 + 4.108771640965568e+00 + 4.099125458381889e+00 + 4.089501861348100e+00 + 4.079900796648449e+00 + 4.070322211621146e+00 + 4.060766054068578e+00 + 4.051232271987449e+00 + 4.041720812903752e+00 + 4.032231624389123e+00 + 4.022764654839604e+00 + 4.013319852102482e+00 + 4.003897163974346e+00 + 3.994496539375360e+00 + 3.985117927005318e+00 + 3.975761275162846e+00 + 3.966426531955261e+00 + 3.957113646635731e+00 + 3.947822569087183e+00 + 3.938553247905866e+00 + 3.929305631685672e+00 + 3.920079669524097e+00 + 3.910875311427285e+00 + 3.901692507178291e+00 + 3.892531206204140e+00 + 3.883391358396833e+00 + 3.874272913657344e+00 + 3.865175821793835e+00 + 3.856100032879644e+00 + 3.847045497225832e+00 + 3.838012165320722e+00 + 3.828999987558789e+00 + 3.820008914413926e+00 + 3.811038896595444e+00 + 3.802089885254187e+00 + 3.793161831237620e+00 + 3.784254684890362e+00 + 3.775368398252875e+00 + 3.766502923100440e+00 + 3.757658209579520e+00 + 3.748834209295840e+00 + 3.740030874433959e+00 + 3.731248156753105e+00 + 3.722486008117639e+00 + 3.713744380387517e+00 + 3.705023225280349e+00 + 3.696322495070079e+00 + 3.687642142318910e+00 + 3.678982119432947e+00 + 3.670342378801804e+00 + 3.661722872867589e+00 + 3.653123554235752e+00 + 3.644544376023881e+00 + 3.635985291634798e+00 + 3.627446254192749e+00 + 3.618927216218316e+00 + 3.610428130335504e+00 + 3.601948951051534e+00 + 3.593489632231203e+00 + 3.585050126838500e+00 + 3.576630388153148e+00 + 3.568230370457688e+00 + 3.559850028500160e+00 + 3.551489315016799e+00 + 3.543148183918080e+00 + 3.534826590807976e+00 + 3.526524489935861e+00 + 3.518241835011243e+00 + 3.509978579991082e+00 + 3.501734680809670e+00 + 3.493510092697708e+00 + 3.485304769188242e+00 + 3.477118665289576e+00 + 3.468951736765370e+00 + 3.460803939385451e+00 + 3.452675227557384e+00 + 3.444565556219660e+00 + 3.436474882078371e+00 + 3.428403160505571e+00 + 3.420350346590145e+00 + 3.412316396304707e+00 + 3.404301265792477e+00 + 3.396304911176187e+00 + 3.388327288491305e+00 + 3.380368354160161e+00 + 3.372428064623584e+00 + 3.364506375833607e+00 + 3.356603244211646e+00 + 3.348718626640740e+00 + 3.340852480236101e+00 + 3.333004761693424e+00 + 3.325175427644580e+00 + 3.317364435527293e+00 + 3.309571742493616e+00 + 3.301797305408617e+00 + 3.294041081414062e+00 + 3.286303028194412e+00 + 3.278583103726798e+00 + 3.270881265443555e+00 + 3.263197471011956e+00 + 3.255531678390248e+00 + 3.247883845209004e+00 + 3.240253929642225e+00 + 3.232641890347850e+00 + 3.225047684908113e+00 + 3.217471271425511e+00 + 3.209912608923088e+00 + 3.202371655667555e+00 + 3.194848370181972e+00 + 3.187342711658790e+00 + 3.179854638881827e+00 + 3.172384110380890e+00 + 3.164931084707051e+00 + 3.157495521384956e+00 + 3.150077380058776e+00 + 3.142676619877814e+00 + 3.135293200073600e+00 + 3.127927079946843e+00 + 3.120578218905895e+00 + 3.113246577218404e+00 + 3.105932114937815e+00 + 3.098634790846833e+00 + 3.091354565058342e+00 + 3.084091398271724e+00 + 3.076845250485478e+00 + 3.069616081755534e+00 + 3.062403852244847e+00 + 3.055208522175611e+00 + 3.048030052549899e+00 + 3.040868404439169e+00 + 3.033723537524932e+00 + 3.026595412527113e+00 + 3.019483991059011e+00 + 3.012389234177664e+00 + 3.005311102428361e+00 + 2.998249556405217e+00 + 2.991204558170310e+00 + 2.984176069441633e+00 + 2.977164051262350e+00 + 2.970168464749357e+00 + 2.963189271462946e+00 + 2.956226433370632e+00 + 2.949279912203107e+00 + 2.942349669912605e+00 + 2.935435668735732e+00 + 2.928537870478263e+00 + 2.921656236941307e+00 + 2.914790730227295e+00 + 2.907941313056430e+00 + 2.901107948114753e+00 + 2.894290597705669e+00 + 2.887489224186201e+00 + 2.880703790004707e+00 + 2.873934257778331e+00 + 2.867180590721052e+00 + 2.860442752157053e+00 + 2.853720705060128e+00 + 2.847014412004529e+00 + 2.840323835735203e+00 + 2.833648939802342e+00 + 2.826989688360529e+00 + 2.820346045051714e+00 + 2.813717971838249e+00 + 2.807105432809830e+00 + 2.800508392739872e+00 + 2.793926814285955e+00 + 2.787360661385112e+00 + 2.780809898806732e+00 + 2.774274490318202e+00 + 2.767754399418414e+00 + 2.761249589980850e+00 + 2.754760027276096e+00 + 2.748285675987487e+00 + 2.741826499952791e+00 + 2.735382462817888e+00 + 2.728953529693038e+00 + 2.722539666591989e+00 + 2.716140837199994e+00 + 2.709757005928732e+00 + 2.703388138477217e+00 + 2.697034199719776e+00 + 2.690695154612969e+00 + 2.684370968516638e+00 + 2.678061606722120e+00 + 2.671767034388748e+00 + 2.665487216663756e+00 + 2.659222119420964e+00 + 2.652971708466898e+00 + 2.646735949170681e+00 + 2.640514807328682e+00 + 2.634308248813400e+00 + 2.628116239351709e+00 + 2.621938745026414e+00 + 2.615775731978081e+00 + 2.609627166192403e+00 + 2.603493014125890e+00 + 2.597373242133807e+00 + 2.591267815972944e+00 + 2.585176702619338e+00 + 2.579099869106871e+00 + 2.573037281022092e+00 + 2.566988905295047e+00 + 2.560954709471109e+00 + 2.554934660124372e+00 + 2.548928723658388e+00 + 2.542936866838776e+00 + 2.536959057454498e+00 + 2.530995262744329e+00 + 2.525045449466102e+00 + 2.519109584724339e+00 + 2.513187636248425e+00 + 2.507279571988880e+00 + 2.501385358975255e+00 + 2.495504964649074e+00 + 2.489638357011514e+00 + 2.483785503877440e+00 + 2.477946372993181e+00 + 2.472120932153270e+00 + 2.466309149445497e+00 + 2.460510993028623e+00 + 2.454726431004756e+00 + 2.448955431279121e+00 + 2.443197962222979e+00 + 2.437453992749473e+00 + 2.431723490873920e+00 + 2.426006424702946e+00 + 2.420302763016215e+00 + 2.414612474849150e+00 + 2.408935529060812e+00 + 2.403271894158963e+00 + 2.397621539164549e+00 + 2.391984432987164e+00 + 2.386360544006294e+00 + 2.380749841635097e+00 + 2.375152295549876e+00 + 2.369567874852914e+00 + 2.363996548806182e+00 + 2.358438286639424e+00 + 2.352893057314327e+00 + 2.347360830870365e+00 + 2.341841577523073e+00 + 2.336335266205249e+00 + 2.330841866367291e+00 + 2.325361348038680e+00 + 2.319893681430557e+00 + 2.314438836350213e+00 + 2.308996782525400e+00 + 2.303567490482044e+00 + 2.298150930506590e+00 + 2.292747072570897e+00 + 2.287355886755380e+00 + 2.281977343203661e+00 + 2.276611412298537e+00 + 2.271258065216148e+00 + 2.265917272602903e+00 + 2.260589004572198e+00 + 2.255273232290290e+00 + 2.249969926489583e+00 + 2.244679057245269e+00 + 2.239400595715412e+00 + 2.234134513404378e+00 + 2.228880781619142e+00 + 2.223639370658599e+00 + 2.218410251587366e+00 + 2.213193396784469e+00 + 2.207988776668006e+00 + 2.202796362022329e+00 + 2.197616125439223e+00 + 2.192448038696543e+00 + 2.187292072961057e+00 + 2.182148199232985e+00 + 2.177016389630072e+00 + 2.171896616273634e+00 + 2.166788850281173e+00 + 2.161693063906081e+00 + 2.156609229682959e+00 + 2.151537319229026e+00 + 2.146477304323076e+00 + 2.141429157241363e+00 + 2.136392850965788e+00 + 2.131368357474257e+00 + 2.126355648357996e+00 + 2.121354696293322e+00 + 2.116365474412895e+00 + 2.111387955644702e+00 + 2.106422111701483e+00 + 2.101467915161587e+00 + 2.096525339256323e+00 + 2.091594356269759e+00 + 2.086674939419586e+00 + 2.081767062556874e+00 + 2.076870697792070e+00 + 2.071985817585338e+00 + 2.067112395385072e+00 + 2.062250405333861e+00 + 2.057399820685575e+00 + 2.052560613690088e+00 + 2.047732757766644e+00 + 2.042916226752948e+00 + 2.038110994473384e+00 + 2.033317034456518e+00 + 2.028534320143143e+00 + 2.023762825026766e+00 + 2.019002522786289e+00 + 2.014253387371010e+00 + 2.009515392980107e+00 + 2.004788513366984e+00 + 2.000072722331970e+00 + 1.995367994084281e+00 + 1.990674302528751e+00 + 1.985991621875054e+00 + 1.981319927135009e+00 + 1.976659192144684e+00 + 1.972009390577169e+00 + 1.967370497271926e+00 + 1.962742487042732e+00 + 1.958125334423246e+00 + 1.953519013586483e+00 + 1.948923499307053e+00 + 1.944338766657414e+00 + 1.939764790288013e+00 + 1.935201544608222e+00 + 1.930649004305472e+00 + 1.926107145248571e+00 + 1.921575942429786e+00 + 1.917055370205972e+00 + 1.912545403920843e+00 + 1.908046018818900e+00 + 1.903557190023903e+00 + 1.899078893306255e+00 + 1.894611103753838e+00 + 1.890153795996532e+00 + 1.885706946418488e+00 + 1.881270530851602e+00 + 1.876844524133795e+00 + 1.872428902057808e+00 + 1.868023640555410e+00 + 1.863628715350075e+00 + 1.859244102235784e+00 + 1.854869776995271e+00 + 1.850505715382929e+00 + 1.846151893336109e+00 + 1.841808287113597e+00 + 1.837474873250078e+00 + 1.833151627500553e+00 + 1.828838525627810e+00 + 1.824535544058799e+00 + 1.820242659570220e+00 + 1.815959848693184e+00 + 1.811687087278560e+00 + 1.807424351697017e+00 + 1.803171618832717e+00 + 1.798928865872756e+00 + 1.794696068965575e+00 + 1.790473204047284e+00 + 1.786260248130819e+00 + 1.782057178544553e+00 + 1.777863972462110e+00 + 1.773680606381003e+00 + 1.769507057021776e+00 + 1.765343301422396e+00 + 1.761189316819223e+00 + 1.757045080431908e+00 + 1.752910569457795e+00 + 1.748785761158555e+00 + 1.744670632638435e+00 + 1.740565161064786e+00 + 1.736469324420783e+00 + 1.732383100261040e+00 + 1.728306465595185e+00 + 1.724239397824929e+00 + 1.720181874717039e+00 + 1.716133874165168e+00 + 1.712095373335323e+00 + 1.708066350172305e+00 + 1.704046783495335e+00 + 1.700036650108550e+00 + 1.696035927584721e+00 + 1.692044595228792e+00 + 1.688062630443701e+00 + 1.684090010658294e+00 + 1.680126714513943e+00 + 1.676172720475745e+00 + 1.672228006742128e+00 + 1.668292551293985e+00 + 1.664366332353613e+00 + 1.660449328407859e+00 + 1.656541518177628e+00 + 1.652642880144113e+00 + 1.648753392536404e+00 + 1.644873033459887e+00 + 1.641001782122826e+00 + 1.637139617814172e+00 + 1.633286518510287e+00 + 1.629442462986198e+00 + 1.625607430332010e+00 + 1.621781398816242e+00 + 1.617964347766720e+00 + 1.614156256896447e+00 + 1.610357104501046e+00 + 1.606566869528149e+00 + 1.602785531410984e+00 + 1.599013068753329e+00 + 1.595249460964023e+00 + 1.591494688099418e+00 + 1.587748729316499e+00 + 1.584011563661260e+00 + 1.580283170366302e+00 + 1.576563528988570e+00 + 1.572852618951484e+00 + 1.569150419635379e+00 + 1.565456911345500e+00 + 1.561772073862832e+00 + 1.558095886092752e+00 + 1.554428327545355e+00 + 1.550769378486889e+00 + 1.547119019605569e+00 + 1.543477229774924e+00 + 1.539843988467248e+00 + 1.536219276909035e+00 + 1.532603074921508e+00 + 1.528995362108325e+00 + 1.525396118748853e+00 + 1.521805324624735e+00 + 1.518222960029189e+00 + 1.514649006474386e+00 + 1.511083443428444e+00 + 1.507526250258141e+00 + 1.503977408416516e+00 + 1.500436898413773e+00 + 1.496904700394939e+00 + 1.493380795245864e+00 + 1.489865163193006e+00 + 1.486357784560608e+00 + 1.482858641108495e+00 + 1.479367713055452e+00 + 1.475884980161266e+00 + 1.472410424507970e+00 + 1.468944026943859e+00 + 1.465485767394345e+00 + 1.462035627142284e+00 + 1.458593587684006e+00 + 1.455159630175685e+00 + 1.451733734815932e+00 + 1.448315882934573e+00 + 1.444906056735074e+00 + 1.441504236540235e+00 + 1.438110403156542e+00 + 1.434724538391589e+00 + 1.431346623689521e+00 + 1.427976640530080e+00 + 1.424614570482120e+00 + 1.421260394583763e+00 + 1.417914094214416e+00 + 1.414575651360096e+00 + 1.411245047472556e+00 + 1.407922264020345e+00 + 1.404607282835409e+00 + 1.401300085863839e+00 + 1.398000654669480e+00 + 1.394708970355802e+00 + 1.391425015768138e+00 + 1.388148773421020e+00 + 1.384880223867722e+00 + 1.381619349200565e+00 + 1.378366132070768e+00 + 1.375120554406545e+00 + 1.371882598139520e+00 + 1.368652245409923e+00 + 1.365429478679455e+00 + 1.362214280063096e+00 + 1.359006631544336e+00 + 1.355806515436745e+00 + 1.352613914349998e+00 + 1.349428810943258e+00 + 1.346251187518860e+00 + 1.343081026346772e+00 + 1.339918309785753e+00 + 1.336763020411036e+00 + 1.333615141050152e+00 + 1.330474654669161e+00 + 1.327341544046787e+00 + 1.324215791384803e+00 + 1.321097378844990e+00 + 1.317986290481943e+00 + 1.314882509034005e+00 + 1.311786015892630e+00 + 1.308696795174431e+00 + 1.305614830505543e+00 + 1.302540104058677e+00 + 1.299472598550161e+00 + 1.296412297281467e+00 + 1.293359183920592e+00 + 1.290313241355424e+00 + 1.287274452437336e+00 + 1.284242800425524e+00 + 1.281218268758570e+00 + 1.278200840971845e+00 + 1.275190500607243e+00 + 1.272187230813499e+00 + 1.269191014801793e+00 + 1.266201836184906e+00 + 1.263219678244913e+00 + 1.260244524601725e+00 + 1.257276359774731e+00 + 1.254315166659556e+00 + 1.251360928048363e+00 + 1.248413628680403e+00 + 1.245473252809453e+00 + 1.242539783866816e+00 + 1.239613204552696e+00 + 1.236693499248504e+00 + 1.233780652886690e+00 + 1.230874648458874e+00 + 1.227975469917747e+00 + 1.225083101914005e+00 + 1.222197528080320e+00 + 1.219318732110022e+00 + 1.216446698153004e+00 + 1.213581411119631e+00 + 1.210722855169846e+00 + 1.207871013783067e+00 + 1.205025871252235e+00 + 1.202187412198626e+00 + 1.199355621285526e+00 + 1.196530482820713e+00 + 1.193711981112556e+00 + 1.190900100540550e+00 + 1.188094825240966e+00 + 1.185296139829988e+00 + 1.182504029475636e+00 + 1.179718478336080e+00 + 1.176939470878376e+00 + 1.174166992368249e+00 + 1.171401026812090e+00 + 1.168641558834780e+00 + 1.165888574598542e+00 + 1.163142057955574e+00 + 1.160401992993886e+00 + 1.157668365957760e+00 + 1.154941161055026e+00 + 1.152220362585152e+00 + 1.149505957086653e+00 + 1.146797929075238e+00 + 1.144096262593592e+00 + 1.141400943461258e+00 + 1.138711957183691e+00 + 1.136029288721929e+00 + 1.133352922646698e+00 + 1.130682844729473e+00 + 1.128019040871752e+00 + 1.125361495026299e+00 + 1.122710192813904e+00 + 1.120065120718956e+00 + 1.117426263156838e+00 + 1.114793605478939e+00 + 1.112167133798269e+00 + 1.109546832911627e+00 + 1.106932688501257e+00 + 1.104324686965964e+00 + 1.101722813149718e+00 + 1.099127052563592e+00 + 1.096537391606932e+00 + 1.093953815681381e+00 + 1.091376310184498e+00 + 1.088804860939953e+00 + 1.086239454210304e+00 + 1.083680075729654e+00 + 1.081126710640132e+00 + 1.078579345517792e+00 + 1.076037966489376e+00 + 1.073502558540405e+00 + 1.070973108232868e+00 + 1.068449602380304e+00 + 1.065932027008114e+00 + 1.063420367175204e+00 + 1.060914608734753e+00 + 1.058414739347230e+00 + 1.055920744292059e+00 + 1.053432609053344e+00 + 1.050950321750202e+00 + 1.048473868029229e+00 + 1.046003233045439e+00 + 1.043538404185668e+00 + 1.041079368203120e+00 + 1.038626111099421e+00 + 1.036178618557632e+00 + 1.033736877638818e+00 + 1.031300875738098e+00 + 1.028870598541307e+00 + 1.026446032279889e+00 + 1.024027163804816e+00 + 1.021613979927458e+00 + 1.019206467749757e+00 + 1.016804614201984e+00 + 1.014408404816817e+00 + 1.012017826403038e+00 + 1.009632866898133e+00 + 1.007253512894913e+00 + 1.004879750675442e+00 + 1.002511566831996e+00 + 1.000148949144661e+00 + 9.977918847145494e-01 + 9.954403597206510e-01 + 9.930943614528360e-01 + 9.907538771638876e-01 + 9.884188936740620e-01 + 9.860893982148410e-01 + 9.837653780132122e-01 + 9.814468200970017e-01 + 9.791337116433633e-01 + 9.768260399168340e-01 + 9.745237922184445e-01 + 9.722269558574113e-01 + 9.699355181124353e-01 + 9.676494662336389e-01 + 9.653687876978898e-01 + 9.630934699744002e-01 + 9.608235003289306e-01 + 9.585588662885656e-01 + 9.562995554410040e-01 + 9.540455551859868e-01 + 9.517968530931009e-01 + 9.495534368089285e-01 + 9.473152938858177e-01 + 9.450824119343849e-01 + 9.428547786515334e-01 + 9.406323818265262e-01 + 9.384152091378831e-01 + 9.362032482272097e-01 + 9.339964869112888e-01 + 9.317949130473703e-01 + 9.295985145023118e-01 + 9.274072791539573e-01 + 9.252211948367504e-01 + 9.230402494034374e-01 + 9.208644309329761e-01 + 9.186937274260252e-01 + 9.165281267878318e-01 + 9.143676170947915e-01 + 9.122121864230248e-01 + 9.100618228160294e-01 + 9.079165144228375e-01 + 9.057762494101416e-01 + 9.036410159434681e-01 + 9.015108022852656e-01 + 8.993855966397851e-01 + 8.972653871281971e-01 + 8.951501621417270e-01 + 8.930399100640233e-01 + 8.909346191274636e-01 + 8.888342777366450e-01 + 8.867388743114113e-01 + 8.846483971822027e-01 + 8.825628348501290e-01 + 8.804821758633001e-01 + 8.784064086961259e-01 + 8.763355218399448e-01 + 8.742695038377348e-01 + 8.722083433110044e-01 + 8.701520288632569e-01 + 8.681005490981059e-01 + 8.660538926659707e-01 + 8.640120483003465e-01 + 8.619750047630993e-01 + 8.599427507590320e-01 + 8.579152750196485e-01 + 8.558925663402844e-01 + 8.538746136199826e-01 + 8.518614056598554e-01 + 8.498529312415619e-01 + 8.478491793952468e-01 + 8.458501390477595e-01 + 8.438557990450950e-01 + 8.418661484572341e-01 + 8.398811762963594e-01 + 8.379008715007146e-01 + 8.359252231704962e-01 + 8.339542204095658e-01 + 8.319878522912720e-01 + 8.300261079597043e-01 + 8.280689765769632e-01 + 8.261164473121154e-01 + 8.241685093974929e-01 + 8.222251520716921e-01 + 8.202863645582569e-01 + 8.183521361157176e-01 + 8.164224560764958e-01 + 8.144973138478017e-01 + 8.125766987264839e-01 + 8.106606000443977e-01 + 8.087490072689457e-01 + 8.068419098143643e-01 + 8.049392971238847e-01 + 8.030411587461223e-01 + 8.011474841608780e-01 + 7.992582628576694e-01 + 7.973734844327619e-01 + 7.954931384582111e-01 + 7.936172145055983e-01 + 7.917457022017643e-01 + 7.898785912282000e-01 + 7.880158712799969e-01 + 7.861575320154112e-01 + 7.843035631939634e-01 + 7.824539546080472e-01 + 7.806086959262468e-01 + 7.787677769423338e-01 + 7.769311875560446e-01 + 7.750989176225404e-01 + 7.732709569908990e-01 + 7.714472955211897e-01 + 7.696279231052234e-01 + 7.678128297211314e-01 + 7.660020053954494e-01 + 7.641954400346342e-01 + 7.623931236302374e-01 + 7.605950462809293e-01 + 7.588011979868697e-01 + 7.570115688113509e-01 + 7.552261489288551e-01 + 7.534449284973839e-01 + 7.516678976340316e-01 + 7.498950464317325e-01 + 7.481263651057625e-01 + 7.463618439154999e-01 + 7.446014731170009e-01 + 7.428452429690515e-01 + 7.410931437272748e-01 + 7.393451656574506e-01 + 7.376012991597445e-01 + 7.358615346418724e-01 + 7.341258624174279e-01 + 7.323942728636854e-01 + 7.306667564033948e-01 + 7.289433034811010e-01 + 7.272239046205010e-01 + 7.255085503342084e-01 + 7.237972310231134e-01 + 7.220899372662432e-01 + 7.203866596915919e-01 + 7.186873887451811e-01 + 7.169921150530395e-01 + 7.153008293485266e-01 + 7.136135222426403e-01 + 7.119301843519804e-01 + 7.102508063464710e-01 + 7.085753789925818e-01 + 7.069038930034782e-01 + 7.052363390742027e-01 + 7.035727080638011e-01 + 7.019129907750439e-01 + 7.002571779481277e-01 + 6.986052604437083e-01 + 6.969572291357313e-01 + 6.953130748988353e-01 + 6.936727887020229e-01 + 6.920363614522840e-01 + 6.904037839913317e-01 + 6.887750473773369e-01 + 6.871501426160253e-01 + 6.855290605871678e-01 + 6.839117924040222e-01 + 6.822983292013320e-01 + 6.806886620123715e-01 + 6.790827818618205e-01 + 6.774806798407976e-01 + 6.758823471448717e-01 + 6.742877749381793e-01 + 6.726969543916550e-01 + 6.711098767181217e-01 + 6.695265330883416e-01 + 6.679469147191758e-01 + 6.663710129607012e-01 + 6.647988190816884e-01 + 6.632303243195694e-01 + 6.616655199754741e-01 + 6.601043974442724e-01 + 6.585469481315596e-01 + 6.569931633409668e-01 + 6.554430344698831e-01 + 6.538965529680326e-01 + 6.523537102216029e-01 + 6.508144977390269e-01 + 6.492789070902356e-01 + 6.477469297088367e-01 + 6.462185570471877e-01 + 6.446937806333431e-01 + 6.431725921186934e-01 + 6.416549831028872e-01 + 6.401409451291091e-01 + 6.386304698205402e-01 + 6.371235488250266e-01 + 6.356201737964903e-01 + 6.341203364004929e-01 + 6.326240283504814e-01 + 6.311312414023209e-01 + 6.296419672600960e-01 + 6.281561976359686e-01 + 6.266739242956670e-01 + 6.251951391194286e-01 + 6.237198339322255e-01 + 6.222480004492102e-01 + 6.207796305731890e-01 + 6.193147162378443e-01 + 6.178532493027741e-01 + 6.163952216645050e-01 + 6.149406252489860e-01 + 6.134894519984210e-01 + 6.120416938491114e-01 + 6.105973427932988e-01 + 6.091563909278429e-01 + 6.077188302116784e-01 + 6.062846526061169e-01 + 6.048538502614210e-01 + 6.034264152733694e-01 + 6.020023397068369e-01 + 6.005816156757759e-01 + 5.991642353027361e-01 + 5.977501907311656e-01 + 5.963394741611191e-01 + 5.949320777981851e-01 + 5.935279938331314e-01 + 5.921272144271874e-01 + 5.907297318514260e-01 + 5.893355384452119e-01 + 5.879446264415406e-01 + 5.865569881004723e-01 + 5.851726157460215e-01 + 5.837915017610071e-01 + 5.824136384878618e-01 + 5.810390182425506e-01 + 5.796676334763571e-01 + 5.782994765837256e-01 + 5.769345398883866e-01 + 5.755728159164172e-01 + 5.742142971757567e-01 + 5.728589760824728e-01 + 5.715068451452269e-01 + 5.701578968814059e-01 + 5.688121237820786e-01 + 5.674695184134964e-01 + 5.661300733449661e-01 + 5.647937811172017e-01 + 5.634606343850478e-01 + 5.621306258016221e-01 + 5.608037479323033e-01 + 5.594799934047830e-01 + 5.581593548937616e-01 + 5.568418251014721e-01 + 5.555273967809645e-01 + 5.542160626584934e-01 + 5.529078153599613e-01 + 5.516026476869607e-01 + 5.503005524879265e-01 + 5.490015224489507e-01 + 5.477055503706971e-01 + 5.464126291394721e-01 + 5.451227516109934e-01 + 5.438359105997491e-01 + 5.425520989229732e-01 + 5.412713094911021e-01 + 5.399935352390902e-01 + 5.387187691034094e-01 + 5.374470040087604e-01 + 5.361782329113639e-01 + 5.349124487848638e-01 + 5.336496445514358e-01 + 5.323898132502169e-01 + 5.311329480044003e-01 + 5.298790417206248e-01 + 5.286280874266170e-01 + 5.273800783238198e-01 + 5.261350074387794e-01 + 5.248928678300107e-01 + 5.236536526638225e-01 + 5.224173550711992e-01 + 5.211839681868924e-01 + 5.199534851768893e-01 + 5.187258992380968e-01 + 5.175012035574781e-01 + 5.162793913049719e-01 + 5.150604557657682e-01 + 5.138443902158373e-01 + 5.126311878387027e-01 + 5.114208419243966e-01 + 5.102133457996210e-01 + 5.090086927559088e-01 + 5.078068761246483e-01 + 5.066078892445277e-01 + 5.054117254264059e-01 + 5.042183780707232e-01 + 5.030278406156905e-01 + 5.018401064512819e-01 + 5.006551689399448e-01 + 4.994730214827681e-01 + 4.982936576173500e-01 + 4.971170708196209e-01 + 4.959432545161117e-01 + 4.947722021784954e-01 + 4.936039073158257e-01 + 4.924383634713059e-01 + 4.912755642215030e-01 + 4.901155031029966e-01 + 4.889581736357913e-01 + 4.878035694426762e-01 + 4.866516841617155e-01 + 4.855025114095978e-01 + 4.843560447562852e-01 + 4.832122778219742e-01 + 4.820712043014287e-01 + 4.809328179063641e-01 + 4.797971123302187e-01 + 4.786640812406847e-01 + 4.775337183184862e-01 + 4.764060173050261e-01 + 4.752809720122764e-01 + 4.741585762111217e-01 + 4.730388236459319e-01 + 4.719217080626099e-01 + 4.708072233097221e-01 + 4.696953632391141e-01 + 4.685861216425942e-01 + 4.674794923970907e-01 + 4.663754693817960e-01 + 4.652740464091041e-01 + 4.641752174102053e-01 + 4.630789763262008e-01 + 4.619853169869301e-01 + 4.608942333544824e-01 + 4.598057194461572e-01 + 4.587197692038988e-01 + 4.576363766028796e-01 + 4.565555356433393e-01 + 4.554772403132010e-01 + 4.544014845983438e-01 + 4.533282625263808e-01 + 4.522575682421708e-01 + 4.511893957834002e-01 + 4.501237391381080e-01 + 4.490605924783677e-01 + 4.479999499003729e-01 + 4.469418054464263e-01 + 4.458861533467765e-01 + 4.448329877503518e-01 + 4.437823027182840e-01 + 4.427340924804984e-01 + 4.416883512661691e-01 + 4.406450732493647e-01 + 4.396042525949710e-01 + 4.385658835318161e-01 + 4.375299603591805e-01 + 4.364964773180230e-01 + 4.354654286529535e-01 + 4.344368086449737e-01 + 4.334106115882546e-01 + 4.323868317973252e-01 + 4.313654636098987e-01 + 4.303465013528135e-01 + 4.293299393608014e-01 + 4.283157719943348e-01 + 4.273039936389254e-01 + 4.262945986862648e-01 + 4.252875815207442e-01 + 4.242829365388910e-01 + 4.232806581737543e-01 + 4.222807409163533e-01 + 4.212831792180283e-01 + 4.202879675160039e-01 + 4.192951002821390e-01 + 4.183045720190468e-01 + 4.173163772308877e-01 + 4.163305103895081e-01 + 4.153469660738844e-01 + 4.143657388899517e-01 + 4.133868232899575e-01 + 4.124102138463511e-01 + 4.114359052122226e-01 + 4.104638919136763e-01 + 4.094941685453110e-01 + 4.085267297797091e-01 + 4.075615702608746e-01 + 4.065986846247451e-01 + 4.056380675149961e-01 + 4.046797136100241e-01 + 4.037236175791848e-01 + 4.027697740867438e-01 + 4.018181778668817e-01 + 4.008688236537185e-01 + 3.999217061664900e-01 + 3.989768201804763e-01 + 3.980341604347806e-01 + 3.970937216243767e-01 + 3.961554986257766e-01 + 3.952194862561716e-01 + 3.942856791749246e-01 + 3.933540722797152e-01 + 3.924246604937682e-01 + 3.914974386057962e-01 + 3.905724013991230e-01 + 3.896495437210211e-01 + 3.887288605266558e-01 + 3.878103467384126e-01 + 3.868939972364765e-01 + 3.859798068708922e-01 + 3.850677705778728e-01 + 3.841578833358520e-01 + 3.832501400994028e-01 + 3.823445358203205e-01 + 3.814410654593080e-01 + 3.805397240004402e-01 + 3.796405064500310e-01 + 3.787434078281850e-01 + 3.778484231550217e-01 + 3.769555474293507e-01 + 3.760647756653435e-01 + 3.751761029803938e-01 + 3.742895244689736e-01 + 3.734050351897463e-01 + 3.725226302131253e-01 + 3.716423046174657e-01 + 3.707640535037849e-01 + 3.698878720450124e-01 + 3.690137553825144e-01 + 3.681416986254492e-01 + 3.672716969834960e-01 + 3.664037456169829e-01 + 3.655378396196581e-01 + 3.646739742353085e-01 + 3.638121447254376e-01 + 3.629523462980791e-01 + 3.620945741078652e-01 + 3.612388233782960e-01 + 3.603850894480597e-01 + 3.595333675652969e-01 + 3.586836529581271e-01 + 3.578359408972814e-01 + 3.569902267099870e-01 + 3.561465057284826e-01 + 3.553047732480240e-01 + 3.544650245489661e-01 + 3.536272549462773e-01 + 3.527914598373741e-01 + 3.519576346281267e-01 + 3.511257746893044e-01 + 3.502958753213151e-01 + 3.494679319027064e-01 + 3.486419398734283e-01 + 3.478178946841421e-01 + 3.469957917542031e-01 + 3.461756264906874e-01 + 3.453573943334770e-01 + 3.445410907466465e-01 + 3.437267112053919e-01 + 3.429142511753236e-01 + 3.421037061357636e-01 + 3.412950715914754e-01 + 3.404883430871680e-01 + 3.396835161220511e-01 + 3.388805861778083e-01 + 3.380795488742585e-01 + 3.372803997559251e-01 + 3.364831342820857e-01 + 3.356877480644493e-01 + 3.348942367473924e-01 + 3.341025959445759e-01 + 3.333128211902062e-01 + 3.325249080644813e-01 + 3.317388522349806e-01 + 3.309546493345364e-01 + 3.301722950123494e-01 + 3.293917849546400e-01 + 3.286131147929292e-01 + 3.278362801760134e-01 + 3.270612768173198e-01 + 3.262881004205997e-01 + 3.255167466776044e-01 + 3.247472112772871e-01 + 3.239794899591130e-01 + 3.232135784952971e-01 + 3.224494726613195e-01 + 3.216871681318617e-01 + 3.209266606147330e-01 + 3.201679460077643e-01 + 3.194110200806869e-01 + 3.186558785671545e-01 + 3.179025173467112e-01 + 3.171509322207308e-01 + 3.164011189537259e-01 + 3.156530734030461e-01 + 3.149067914068442e-01 + 3.141622688005587e-01 + 3.134195014964591e-01 + 3.126784853846941e-01 + 3.119392163234083e-01 + 3.112016901727622e-01 + 3.104659028247113e-01 + 3.097318502149898e-01 + 3.089995283319454e-01 + 3.082689330721005e-01 + 3.075400602663161e-01 + 3.068129059444455e-01 + 3.060874661234649e-01 + 3.053637367362504e-01 + 3.046417137079979e-01 + 3.039213930418579e-01 + 3.032027708160949e-01 + 3.024858429572295e-01 + 3.017706054435225e-01 + 3.010570543910775e-01 + 3.003451858060657e-01 + 2.996349956970866e-01 + 2.989264801519126e-01 + 2.982196352437723e-01 + 2.975144570351090e-01 + 2.968109415899733e-01 + 2.961090850010212e-01 + 2.954088833712375e-01 + 2.947103327960572e-01 + 2.940134293733013e-01 + 2.933181692489545e-01 + 2.926245486550200e-01 + 2.919325636534309e-01 + 2.912422103009782e-01 + 2.905534849067459e-01 + 2.898663836341845e-01 + 2.891809025716850e-01 + 2.884970379499296e-01 + 2.878147859946116e-01 + 2.871341429080666e-01 + 2.864551048929091e-01 + 2.857776681836698e-01 + 2.851018290276999e-01 + 2.844275836247271e-01 + 2.837549281975342e-01 + 2.830838590248334e-01 + 2.824143724756855e-01 + 2.817464648036653e-01 + 2.810801321714598e-01 + 2.804153709678348e-01 + 2.797521775256636e-01 + 2.790905480675208e-01 + 2.784304789557904e-01 + 2.777719665515425e-01 + 2.771150071618110e-01 + 2.764595971293084e-01 + 2.758057328280932e-01 + 2.751534106502780e-01 + 2.745026269461392e-01 + 2.738533780697315e-01 + 2.732056604090851e-01 + 2.725594703611444e-01 + 2.719148043516682e-01 + 2.712716588414341e-01 + 2.706300302048149e-01 + 2.699899148252890e-01 + 2.693513091897057e-01 + 2.687142097803046e-01 + 2.680786130525787e-01 + 2.674445154290628e-01 + 2.668119133864937e-01 + 2.661808034370396e-01 + 2.655511820876730e-01 + 2.649230457933240e-01 + 2.642963910212357e-01 + 2.636712143601993e-01 + 2.630475123173707e-01 + 2.624252813696634e-01 + 2.618045181320832e-01 + 2.611852191534089e-01 + 2.605673809346502e-01 + 2.599510000768786e-01 + 2.593360731482656e-01 + 2.587225966857610e-01 + 2.581105673020023e-01 + 2.574999816218594e-01 + 2.568908362541323e-01 + 2.562831277681717e-01 + 2.556768527608984e-01 + 2.550720078808991e-01 + 2.544685898158210e-01 + 2.538665952029344e-01 + 2.532660206235198e-01 + 2.526668627769018e-01 + 2.520691183671673e-01 + 2.514727840477828e-01 + 2.508778564832650e-01 + 2.502843323450471e-01 + 2.496922083184503e-01 + 2.491014811741685e-01 + 2.485121476502302e-01 + 2.479242043768022e-01 + 2.473376480750936e-01 + 2.467524755330193e-01 + 2.461686835577825e-01 + 2.455862688342147e-01 + 2.450052280577099e-01 + 2.444255580815592e-01 + 2.438472556710649e-01 + 2.432703175621061e-01 + 2.426947405763887e-01 + 2.421205215374063e-01 + 2.415476572469543e-01 + 2.409761444715131e-01 + 2.404059800179439e-01 + 2.398371607406130e-01 + 2.392696835287547e-01 + 2.387035451998669e-01 + 2.381387425460481e-01 + 2.375752724876774e-01 + 2.370131319015487e-01 + 2.364523176042576e-01 + 2.358928264341899e-01 + 2.353346553264918e-01 + 2.347778012643444e-01 + 2.342222610470114e-01 + 2.336680315481888e-01 + 2.331151097630285e-01 + 2.325634926002070e-01 + 2.320131769932555e-01 + 2.314641599195968e-01 + 2.309164382391606e-01 + 2.303700088852327e-01 + 2.298248689205450e-01 + 2.292810152383314e-01 + 2.287384447851956e-01 + 2.281971546639490e-01 + 2.276571417710518e-01 + 2.271184030444375e-01 + 2.265809356255705e-01 + 2.260447364485637e-01 + 2.255098024678978e-01 + 2.249761308577043e-01 + 2.244437185775247e-01 + 2.239125625576918e-01 + 2.233826599324974e-01 + 2.228540078061592e-01 + 2.223266032159731e-01 + 2.218004431318401e-01 + 2.212755246546400e-01 + 2.207518449320089e-01 + 2.202294009962271e-01 + 2.197081899542086e-01 + 2.191882089472320e-01 + 2.186694550184439e-01 + 2.181519252528444e-01 + 2.176356167922184e-01 + 2.171205268020538e-01 + 2.166066524055817e-01 + 2.160939907132310e-01 + 2.155825389406823e-01 + 2.150722942170351e-01 + 2.145632535999474e-01 + 2.140554143128364e-01 + 2.135487735713488e-01 + 2.130433285376680e-01 + 2.125390763961598e-01 + 2.120360143225986e-01 + 2.115341394866721e-01 + 2.110334491301104e-01 + 2.105339404911263e-01 + 2.100356107692428e-01 + 2.095384571593845e-01 + 2.090424768875718e-01 + 2.085476672286868e-01 + 2.080540254649787e-01 + 2.075615488420417e-01 + 2.070702345483775e-01 + 2.065800798220710e-01 + 2.060910819584345e-01 + 2.056032383022174e-01 + 2.051165461242101e-01 + 2.046310026800104e-01 + 2.041466052775261e-01 + 2.036633511978056e-01 + 2.031812377402102e-01 + 2.027002622847228e-01 + 2.022204221329609e-01 + 2.017417145657832e-01 + 2.012641369577989e-01 + 2.007876866366623e-01 + 2.003123609141739e-01 + 1.998381571847571e-01 + 1.993650728223044e-01 + 1.988931051658790e-01 + 1.984222515391579e-01 + 1.979525093566432e-01 + 1.974838760798574e-01 + 1.970163490394448e-01 + 1.965499255924087e-01 + 1.960846031634822e-01 + 1.956203792216227e-01 + 1.951572511456189e-01 + 1.946952162469976e-01 + 1.942342720345011e-01 + 1.937744160087473e-01 + 1.933156455855021e-01 + 1.928579581617409e-01 + 1.924013511601302e-01 + 1.919458220512063e-01 + 1.914913683490730e-01 + 1.910379875218400e-01 + 1.905856769690048e-01 + 1.901344342343178e-01 + 1.896842568245793e-01 + 1.892351421122624e-01 + 1.887870876905098e-01 + 1.883400911382459e-01 + 1.878941498265747e-01 + 1.874492612709913e-01 + 1.870054230608876e-01 + 1.865626327533365e-01 + 1.861208878413260e-01 + 1.856801858136446e-01 + 1.852405242290795e-01 + 1.848019006561455e-01 + 1.843643126785636e-01 + 1.839277579090688e-01 + 1.834922338318696e-01 + 1.830577379260484e-01 + 1.826242679175190e-01 + 1.821918214082343e-01 + 1.817603959062191e-01 + 1.813299890419420e-01 + 1.809005984096844e-01 + 1.804722215715290e-01 + 1.800448561676805e-01 + 1.796184998471616e-01 + 1.791931502428031e-01 + 1.787688049585256e-01 + 1.783454616010165e-01 + 1.779231178061076e-01 + 1.775017712916434e-01 + 1.770814197185338e-01 + 1.766620606643876e-01 + 1.762436917707559e-01 + 1.758263107475443e-01 + 1.754099153420786e-01 + 1.749945031691721e-01 + 1.745800718793870e-01 + 1.741666192327871e-01 + 1.737541428912765e-01 + 1.733426405128906e-01 + 1.729321098263452e-01 + 1.725225486037993e-01 + 1.721139545737653e-01 + 1.717063253659770e-01 + 1.712996586978482e-01 + 1.708939523524782e-01 + 1.704892041326627e-01 + 1.700854117308264e-01 + 1.696825728315613e-01 + 1.692806852413719e-01 + 1.688797467224855e-01 + 1.684797550119726e-01 + 1.680807078836749e-01 + 1.676826031516292e-01 + 1.672854386236076e-01 + 1.668892120257625e-01 + 1.664939211364053e-01 + 1.660995637803862e-01 + 1.657061377653481e-01 + 1.653136408872399e-01 + 1.649220709518862e-01 + 1.645314258210089e-01 + 1.641417032838678e-01 + 1.637529010922642e-01 + 1.633650171697226e-01 + 1.629780493848234e-01 + 1.625919955117979e-01 + 1.622068533722165e-01 + 1.618226208359533e-01 + 1.614392958070308e-01 + 1.610568761629498e-01 + 1.606753597398519e-01 + 1.602947443525945e-01 + 1.599150279328092e-01 + 1.595362083976419e-01 + 1.591582835876941e-01 + 1.587812513806259e-01 + 1.584051096895935e-01 + 1.580298564486488e-01 + 1.576554895336735e-01 + 1.572820068494865e-01 + 1.569094063844064e-01 + 1.565376859935812e-01 + 1.561668435389086e-01 + 1.557968770322644e-01 + 1.554277844430587e-01 + 1.550595637013555e-01 + 1.546922127252644e-01 + 1.543257294352004e-01 + 1.539601117962810e-01 + 1.535953578647676e-01 + 1.532314655336160e-01 + 1.528684326732572e-01 + 1.525062574192351e-01 + 1.521449377506904e-01 + 1.517844715391133e-01 + 1.514248568043021e-01 + 1.510660915767141e-01 + 1.507081738546517e-01 + 1.503511015850067e-01 + 1.499948728018673e-01 + 1.496394855976185e-01 + 1.492849379312514e-01 + 1.489312277844460e-01 + 1.485783532079666e-01 + 1.482263122946834e-01 + 1.478751030581660e-01 + 1.475247234388987e-01 + 1.471751715323068e-01 + 1.468264454469171e-01 + 1.464785432366154e-01 + 1.461314629165984e-01 + 1.457852025452889e-01 + 1.454397602433564e-01 + 1.450951340257908e-01 + 1.447513219420110e-01 + 1.444083221508569e-01 + 1.440661327109938e-01 + 1.437247516822454e-01 + 1.433841772072484e-01 + 1.430444073901374e-01 + 1.427054403077353e-01 + 1.423672740371242e-01 + 1.420299067307786e-01 + 1.416933365389871e-01 + 1.413575615237460e-01 + 1.410225798197031e-01 + 1.406883895901710e-01 + 1.403549889435149e-01 + 1.400223760032770e-01 + 1.396905489310608e-01 + 1.393595059398338e-01 + 1.390292451224472e-01 + 1.386997645496112e-01 + 1.383710625136722e-01 + 1.380431371824567e-01 + 1.377159866326167e-01 + 1.373896091055998e-01 + 1.370640027752214e-01 + 1.367391657592925e-01 + 1.364150963290927e-01 + 1.360917926783734e-01 + 1.357692529162330e-01 + 1.354474752994454e-01 + 1.351264580487222e-01 + 1.348061993122180e-01 + 1.344866973181728e-01 + 1.341679503229942e-01 + 1.338499565682958e-01 + 1.335327142013654e-01 + 1.332162214320106e-01 + 1.329004765902875e-01 + 1.325854779258548e-01 + 1.322712236439755e-01 + 1.319577119415115e-01 + 1.316449410589970e-01 + 1.313329092858229e-01 + 1.310216149439378e-01 + 1.307110562100912e-01 + 1.304012313067827e-01 + 1.300921386594473e-01 + 1.297837764741680e-01 + 1.294761429396481e-01 + 1.291692364743583e-01 + 1.288630553353233e-01 + 1.285575977268189e-01 + 1.282528620001082e-01 + 1.279488464604472e-01 + 1.276455493846629e-01 + 1.273429690961771e-01 + 1.270411039265308e-01 + 1.267399521912149e-01 + 1.264395121606617e-01 + 1.261397821624623e-01 + 1.258407605612416e-01 + 1.255424456647032e-01 + 1.252448358003494e-01 + 1.249479293256652e-01 + 1.246517245985584e-01 + 1.243562199440227e-01 + 1.240614136699035e-01 + 1.237673041530004e-01 + 1.234738897833225e-01 + 1.231811689369122e-01 + 1.228891399541342e-01 + 1.225978011897444e-01 + 1.223071510285007e-01 + 1.220171878485729e-01 + 1.217279100221359e-01 + 1.214393159216827e-01 + 1.211514039479249e-01 + 1.208641725134118e-01 + 1.205776200272462e-01 + 1.202917448643591e-01 + 1.200065453999493e-01 + 1.197220200388544e-01 + 1.194381672203820e-01 + 1.191549853738593e-01 + 1.188724728867774e-01 + 1.185906281982059e-01 + 1.183094497403516e-01 + 1.180289358802264e-01 + 1.177490851172322e-01 + 1.174698959467804e-01 + 1.171913666738082e-01 + 1.169134957966049e-01 + 1.166362818664215e-01 + 1.163597231976005e-01 + 1.160838182295854e-01 + 1.158085654926154e-01 + 1.155339634301463e-01 + 1.152600105152270e-01 + 1.149867052503320e-01 + 1.147140461081585e-01 + 1.144420315115333e-01 + 1.141706598827868e-01 + 1.138999297876022e-01 + 1.136298397724347e-01 + 1.133603883121320e-01 + 1.130915738073908e-01 + 1.128233947404400e-01 + 1.125558496925181e-01 + 1.122889371862514e-01 + 1.120226557224089e-01 + 1.117570037925019e-01 + 1.114919798574519e-01 + 1.112275824488763e-01 + 1.109638101830490e-01 + 1.107006615250319e-01 + 1.104381349837027e-01 + 1.101762291935612e-01 + 1.099149425817940e-01 + 1.096542736337556e-01 + 1.093942210611990e-01 + 1.091347833750166e-01 + 1.088759590309823e-01 + 1.086177465849946e-01 + 1.083601446468375e-01 + 1.081031518187680e-01 + 1.078467666394599e-01 + 1.075909876365943e-01 + 1.073358133695979e-01 + 1.070812424691808e-01 + 1.068272734935852e-01 + 1.065739049762412e-01 + 1.063211355273724e-01 + 1.060689637753818e-01 + 1.058173883167701e-01 + 1.055664076522099e-01 + 1.053160204177178e-01 + 1.050662253083730e-01 + 1.048170208178578e-01 + 1.045684055544151e-01 + 1.043203782231123e-01 + 1.040729373745031e-01 + 1.038260815947437e-01 + 1.035798095388146e-01 + 1.033341198496129e-01 + 1.030890111353506e-01 + 1.028444819874746e-01 + 1.026005310752415e-01 + 1.023571570589419e-01 + 1.021143585605686e-01 + 1.018721342063349e-01 + 1.016304826397723e-01 + 1.013894025261397e-01 + 1.011488925387934e-01 + 1.009089513225976e-01 + 1.006695774897203e-01 + 1.004307697412456e-01 + 1.001925267601519e-01 + 9.995484714928184e-02 + 9.971772961029543e-02 + 9.948117285600222e-02 + 9.924517553207431e-02 + 9.900973631553432e-02 + 9.877485389836264e-02 + 9.854052696321766e-02 + 9.830675420513860e-02 + 9.807353432053509e-02 + 9.784086599274402e-02 + 9.760874792461041e-02 + 9.737717882679013e-02 + 9.714615739564664e-02 + 9.691568233864475e-02 + 9.668575237168565e-02 + 9.645636620785530e-02 + 9.622752256709627e-02 + 9.599922017186617e-02 + 9.577145773476621e-02 + 9.554423398421420e-02 + 9.531754766036228e-02 + 9.509139748951084e-02 + 9.486578220406576e-02 + 9.464070054491446e-02 + 9.441615125061270e-02 + 9.419213306609350e-02 + 9.396864474309072e-02 + 9.374568502973517e-02 + 9.352325267700241e-02 + 9.330134644110695e-02 + 9.307996508088016e-02 + 9.285910736057421e-02 + 9.263877204899980e-02 + 9.241895790580944e-02 + 9.219966369924239e-02 + 9.198088821288498e-02 + 9.176263021506216e-02 + 9.154488848131032e-02 + 9.132766180778470e-02 + 9.111094897456264e-02 + 9.089474876186458e-02 + 9.067905996494766e-02 + 9.046388137755315e-02 + 9.024921179550818e-02 + 9.003505002157913e-02 + 8.982139485394958e-02 + 8.960824509461397e-02 + 8.939559956013639e-02 + 8.918345705664489e-02 + 8.897181638949078e-02 + 8.876067638185769e-02 + 8.855003585659860e-02 + 8.833989363307079e-02 + 8.813024852794749e-02 + 8.792109937217552e-02 + 8.771244500411417e-02 + 8.750428424959714e-02 + 8.729661594151820e-02 + 8.708943892038122e-02 + 8.688275202450782e-02 + 8.667655409894154e-02 + 8.647084399500883e-02 + 8.626562056133152e-02 + 8.606088264727801e-02 + 8.585662910508592e-02 + 8.565285879275303e-02 + 8.544957056834111e-02 + 8.524676329099723e-02 + 8.504443583456130e-02 + 8.484258706983651e-02 + 8.464121585953427e-02 + 8.444032107793580e-02 + 8.423990160173883e-02 + 8.403995630660985e-02 + 8.384048408000297e-02 + 8.364148380739438e-02 + 8.344295436518863e-02 + 8.324489464319758e-02 + 8.304730353631981e-02 + 8.285017993778075e-02 + 8.265352274723796e-02 + 8.245733086610936e-02 + 8.226160319325253e-02 + 8.206633863307464e-02 + 8.187153609323948e-02 + 8.167719448173898e-02 + 8.148331271120264e-02 + 8.128988969775824e-02 + 8.109692435907066e-02 + 8.090441561287741e-02 + 8.071236238052074e-02 + 8.052076359329230e-02 + 8.032961817777003e-02 + 8.013892505875637e-02 + 7.994868317110082e-02 + 7.975889145271257e-02 + 7.956954884117817e-02 + 7.938065427030368e-02 + 7.919220668490153e-02 + 7.900420503735811e-02 + 7.881664826626886e-02 + 7.862953532075569e-02 + 7.844286516277023e-02 + 7.825663674598571e-02 + 7.807084902471831e-02 + 7.788550095723919e-02 + 7.770059150356423e-02 + 7.751611963202924e-02 + 7.733208431808952e-02 + 7.714848452080215e-02 + 7.696531920714285e-02 + 7.678258736267791e-02 + 7.660028796508990e-02 + 7.641841999039113e-02 + 7.623698241865759e-02 + 7.605597423593993e-02 + 7.587539442823407e-02 + 7.569524197859213e-02 + 7.551551588287067e-02 + 7.533621514035872e-02 + 7.515733874432233e-02 + 7.497888568830134e-02 + 7.480085497230264e-02 + 7.462324560850588e-02 + 7.444605660139321e-02 + 7.426928695221670e-02 + 7.409293566891996e-02 + 7.391700176673981e-02 + 7.374148426439617e-02 + 7.356638217848553e-02 + 7.339169452681224e-02 + 7.321742033027905e-02 + 7.304355861552074e-02 + 7.287010840815517e-02 + 7.269706873346385e-02 + 7.252443862192341e-02 + 7.235221710900047e-02 + 7.218040323296335e-02 + 7.200899602923243e-02 + 7.183799453782064e-02 + 7.166739780328910e-02 + 7.149720486628935e-02 + 7.132741477295296e-02 + 7.115802657678706e-02 + 7.098903933028558e-02 + 7.082045208266741e-02 + 7.065226388294948e-02 + 7.048447379884398e-02 + 7.031708089501212e-02 + 7.015008422428966e-02 + 6.998348285005500e-02 + 6.981727584178653e-02 + 6.965146227138255e-02 + 6.948604121024822e-02 + 6.932101172952453e-02 + 6.915637290177927e-02 + 6.899212381049466e-02 + 6.882826353831664e-02 + 6.866479115768206e-02 + 6.850170575553913e-02 + 6.833900642241270e-02 + 6.817669223943085e-02 + 6.801476230171261e-02 + 6.785321570853013e-02 + 6.769205154760372e-02 + 6.753126891476798e-02 + 6.737086691265436e-02 + 6.721084464422118e-02 + 6.705120121123045e-02 + 6.689193571639933e-02 + 6.673304726820108e-02 + 6.657453497707803e-02 + 6.641639795452120e-02 + 6.625863531334618e-02 + 6.610124617104705e-02 + 6.594422964866814e-02 + 6.578758486543299e-02 + 6.563131094165708e-02 + 6.547540700031643e-02 + 6.531987216906585e-02 + 6.516470557837332e-02 + 6.500990635997374e-02 + 6.485547364331327e-02 + 6.470140656204018e-02 + 6.454770425495959e-02 + 6.439436585565442e-02 + 6.424139050518185e-02 + 6.408877735587415e-02 + 6.393652554954243e-02 + 6.378463422860429e-02 + 6.363310254304622e-02 + 6.348192964216136e-02 + 6.333111467891862e-02 + 6.318065681259971e-02 + 6.303055519506789e-02 + 6.288080898167979e-02 + 6.273141734118348e-02 + 6.258237943694549e-02 + 6.243369443027962e-02 + 6.228536148636779e-02 + 6.213737977420020e-02 + 6.198974846629291e-02 + 6.184246673830131e-02 + 6.169553375986874e-02 + 6.154894870195880e-02 + 6.140271075002260e-02 + 6.125681908485271e-02 + 6.111127288458502e-02 + 6.096607133498923e-02 + 6.082121362330186e-02 + 6.067669893778977e-02 + 6.053252647002724e-02 + 6.038869541057570e-02 + 6.024520494995262e-02 + 6.010205428441878e-02 + 5.995924261312039e-02 + 5.981676913606451e-02 + 5.967463305068377e-02 + 5.953283356000520e-02 + 5.939136987405440e-02 + 5.925024120274393e-02 + 5.910944675075109e-02 + 5.896898572024133e-02 + 5.882885733338872e-02 + 5.868906080796419e-02 + 5.854959534943303e-02 + 5.841046018167366e-02 + 5.827165452667381e-02 + 5.813317759547484e-02 + 5.799502862000658e-02 + 5.785720683145133e-02 + 5.771971144629215e-02 + 5.758254169802959e-02 + 5.744569682168989e-02 + 5.730917604038396e-02 + 5.717297859181381e-02 + 5.703710371771920e-02 + 5.690155065150715e-02 + 5.676631863514842e-02 + 5.663140691176988e-02 + 5.649681471550019e-02 + 5.636254129818796e-02 + 5.622858591623749e-02 + 5.609494780637568e-02 + 5.596162621719655e-02 + 5.582862040612746e-02 + 5.569592962469259e-02 + 5.556355312969728e-02 + 5.543149018219354e-02 + 5.529974004038473e-02 + 5.516830196534387e-02 + 5.503717522021812e-02 + 5.490635906464716e-02 + 5.477585276550550e-02 + 5.464565559606692e-02 + 5.451576682310098e-02 + 5.438618571514530e-02 + 5.425691154653305e-02 + 5.412794360021059e-02 + 5.399928115030668e-02 + 5.387092346220070e-02 + 5.374286982722595e-02 + 5.361511953114147e-02 + 5.348767184376742e-02 + 5.336052605560348e-02 + 5.323368145967647e-02 + 5.310713734132736e-02 + 5.298089299097181e-02 + 5.285494769998652e-02 + 5.272930075794034e-02 + 5.260395145882699e-02 + 5.247889910164562e-02 + 5.235414299012888e-02 + 5.222968242359068e-02 + 5.210551670083303e-02 + 5.198164512479836e-02 + 5.185806699751586e-02 + 5.173478162567831e-02 + 5.161178832737659e-02 + 5.148908640731603e-02 + 5.136667516904735e-02 + 5.124455393713079e-02 + 5.112272202541198e-02 + 5.100117874145336e-02 + 5.087992340415878e-02 + 5.075895533879782e-02 + 5.063827387090040e-02 + 5.051787831631814e-02 + 5.039776799903027e-02 + 5.027794224956293e-02 + 5.015840039018700e-02 + 5.003914174911826e-02 + 4.992016566217095e-02 + 4.980147146451835e-02 + 4.968305848565534e-02 + 4.956492605368701e-02 + 4.944707351707827e-02 + 4.932950021762891e-02 + 4.921220548403189e-02 + 4.909518865563611e-02 + 4.897844908008390e-02 + 4.886198610990230e-02 + 4.874579909103274e-02 + 4.862988736593407e-02 + 4.851425027765261e-02 + 4.839888718215429e-02 + 4.828379743688660e-02 + 4.816898039344981e-02 + 4.805443540666063e-02 + 4.794016183433197e-02 + 4.782615903628512e-02 + 4.771242637003347e-02 + 4.759896319572905e-02 + 4.748576888074151e-02 + 4.737284278529100e-02 + 4.726018427180961e-02 + 4.714779271742187e-02 + 4.703566749191604e-02 + 4.692380796083007e-02 + 4.681221349433169e-02 + 4.670088346648602e-02 + 4.658981725459431e-02 + 4.647901423861556e-02 + 4.636847379587147e-02 + 4.625819530165282e-02 + 4.614817813294777e-02 + 4.603842167462240e-02 + 4.592892531626435e-02 + 4.581968844113481e-02 + 4.571071043517037e-02 + 4.560199068716930e-02 + 4.549352858172214e-02 + 4.538532351139604e-02 + 4.527737487551017e-02 + 4.516968206047602e-02 + 4.506224446147816e-02 + 4.495506148526871e-02 + 4.484813252205796e-02 + 4.474145696749865e-02 + 4.463503423038843e-02 + 4.452886371233129e-02 + 4.442294481444441e-02 + 4.431727694184067e-02 + 4.421185950249874e-02 + 4.410669190344264e-02 + 4.400177354999718e-02 + 4.389710385863231e-02 + 4.379268224561385e-02 + 4.368850811842072e-02 + 4.358458089102907e-02 + 4.348089998166713e-02 + 4.337746480946211e-02 + 4.327427479290449e-02 + 4.317132935104961e-02 + 4.306862790538591e-02 + 4.296616988021409e-02 + 4.286395470044128e-02 + 4.276198178905291e-02 + 4.266025057462805e-02 + 4.255876048879523e-02 + 4.245751096050290e-02 + 4.235650142249979e-02 + 4.225573130868781e-02 + 4.215520004666404e-02 + 4.205490707118502e-02 + 4.195485182346907e-02 + 4.185503374212908e-02 + 4.175545226547943e-02 + 4.165610683241490e-02 + 4.155699688308426e-02 + 4.145812186261490e-02 + 4.135948121993713e-02 + 4.126107439911159e-02 + 4.116290084444992e-02 + 4.106496000347922e-02 + 4.096725133001662e-02 + 4.086977427671171e-02 + 4.077252829257245e-02 + 4.067551282880393e-02 + 4.057872734137224e-02 + 4.048217129089726e-02 + 4.038584413229004e-02 + 4.028974532214836e-02 + 4.019387432351806e-02 + 4.009823059884492e-02 + 4.000281360871143e-02 + 3.990762281263916e-02 + 3.981265768084040e-02 + 3.971791768404272e-02 + 3.962340228400397e-02 + 3.952911094818634e-02 + 3.943504314911767e-02 + 3.934119836154264e-02 + 3.924757605343093e-02 + 3.915417569553582e-02 + 3.906099677494528e-02 + 3.896803876462888e-02 + 3.887530113159737e-02 + 3.878278335807980e-02 + 3.869048493136602e-02 + 3.859840533540703e-02 + 3.850654403935494e-02 + 3.841490053099362e-02 + 3.832347430867683e-02 + 3.823226484521238e-02 + 3.814127162485314e-02 + 3.805049414568009e-02 + 3.795993189727254e-02 + 3.786958436838804e-02 + 3.777945104950908e-02 + 3.768953143238456e-02 + 3.759982501241896e-02 + 3.751033128919119e-02 + 3.742104976390229e-02 + 3.733197993101921e-02 + 3.724312127956968e-02 + 3.715447331892652e-02 + 3.706603555538170e-02 + 3.697780748276271e-02 + 3.688978860838169e-02 + 3.680197844047203e-02 + 3.671437648083043e-02 + 3.662698223859462e-02 + 3.653979522528233e-02 + 3.645281495050149e-02 + 3.636604092401768e-02 + 3.627947265609178e-02 + 3.619310965841749e-02 + 3.610695145022991e-02 + 3.602099755007137e-02 + 3.593524746722872e-02 + 3.584970072102484e-02 + 3.576435683502647e-02 + 3.567921532660211e-02 + 3.559427571435374e-02 + 3.550953752116312e-02 + 3.542500027693078e-02 + 3.534066350384148e-02 + 3.525652672170999e-02 + 3.517258946309138e-02 + 3.508885125628003e-02 + 3.500531162678291e-02 + 3.492197010940873e-02 + 3.483882623525817e-02 + 3.475587953263527e-02 + 3.467312953919303e-02 + 3.459057578876406e-02 + 3.450821781198862e-02 + 3.442605515251815e-02 + 3.434408734999198e-02 + 3.426231393723483e-02 + 3.418073445581654e-02 + 3.409934844816886e-02 + 3.401815545480084e-02 + 3.393715501912345e-02 + 3.385634668688416e-02 + 3.377573000557660e-02 + 3.369530452244349e-02 + 3.361506978424043e-02 + 3.353502533757853e-02 + 3.345517073173793e-02 + 3.337550551749302e-02 + 3.329602924688495e-02 + 3.321674147756795e-02 + 3.313764176353938e-02 + 3.305872964905672e-02 + 3.298000469915312e-02 + 3.290146647870388e-02 + 3.282311452928582e-02 + 3.274494841443989e-02 + 3.266696770524984e-02 + 3.258917195579586e-02 + 3.251156072835543e-02 + 3.243413358906379e-02 + 3.235689009597382e-02 + 3.227982981604406e-02 + 3.220295232152501e-02 + 3.212625717796667e-02 + 3.204974395335523e-02 + 3.197341221860803e-02 + 3.189726154412311e-02 + 3.182129149811844e-02 + 3.174550164996338e-02 + 3.166989158019418e-02 + 3.159446086552403e-02 + 3.151920907660396e-02 + 3.144413578635114e-02 + 3.136924057603768e-02 + 3.129452303273958e-02 + 3.121998273049806e-02 + 3.114561924523454e-02 + 3.107143216011856e-02 + 3.099742105799270e-02 + 3.092358552474080e-02 + 3.084992514925186e-02 + 3.077643951122557e-02 + 3.070312819305190e-02 + 3.062999078663620e-02 + 3.055702688363699e-02 + 3.048423607160382e-02 + 3.041161793396379e-02 + 3.033917207015753e-02 + 3.026689807846873e-02 + 3.019479554128978e-02 + 3.012286404982057e-02 + 3.005110320361238e-02 + 2.997951260681279e-02 + 2.990809185255966e-02 + 2.983684053226568e-02 + 2.976575824919644e-02 + 2.969484460522792e-02 + 2.962409920087884e-02 + 2.955352163824170e-02 + 2.948311151647172e-02 + 2.941286843696999e-02 + 2.934279201383451e-02 + 2.927288185012002e-02 + 2.920313754405053e-02 + 2.913355871301240e-02 + 2.906414496442064e-02 + 2.899489589842786e-02 + 2.892581113440367e-02 + 2.885689028447655e-02 + 2.878813295173805e-02 + 2.871953875071393e-02 + 2.865110730129668e-02 + 2.858283822298987e-02 + 2.851473112218837e-02 + 2.844678561226344e-02 + 2.837900131834860e-02 + 2.831137785774055e-02 + 2.824391484910513e-02 + 2.817661191561155e-02 + 2.810946867322990e-02 + 2.804248474261467e-02 + 2.797565975421473e-02 + 2.790899332866750e-02 + 2.784248508684310e-02 + 2.777613465729838e-02 + 2.770994166752542e-02 + 2.764390574354870e-02 + 2.757802651096095e-02 + 2.751230360164656e-02 + 2.744673664902718e-02 + 2.738132528259552e-02 + 2.731606912935157e-02 + 2.725096782054994e-02 + 2.718602099881922e-02 + 2.712122829441404e-02 + 2.705658933533107e-02 + 2.699210376654040e-02 + 2.692777122629619e-02 + 2.686359134766074e-02 + 2.679956376855050e-02 + 2.673568813034148e-02 + 2.667196407449100e-02 + 2.660839123653040e-02 + 2.654496925932325e-02 + 2.648169779209303e-02 + 2.641857647953893e-02 + 2.635560496230991e-02 + 2.629278288047702e-02 + 2.623010988365326e-02 + 2.616758562366618e-02 + 2.610520975031933e-02 + 2.604298190560967e-02 + 2.598090173551063e-02 + 2.591896889422694e-02 + 2.585718303968204e-02 + 2.579554382343284e-02 + 2.573405088890056e-02 + 2.567270388987125e-02 + 2.561150248485746e-02 + 2.555044633271165e-02 + 2.548953508798725e-02 + 2.542876840653832e-02 + 2.536814594843316e-02 + 2.530766736730480e-02 + 2.524733231969847e-02 + 2.518714047250964e-02 + 2.512709148651639e-02 + 2.506718502244662e-02 + 2.500742074738894e-02 + 2.494779831751132e-02 + 2.488831739145854e-02 + 2.482897764751662e-02 + 2.476977875068464e-02 + 2.471072035836960e-02 + 2.465180213557270e-02 + 2.459302375909131e-02 + 2.453438490494015e-02 + 2.447588522684439e-02 + 2.441752439632618e-02 + 2.435930209686822e-02 + 2.430121799563831e-02 + 2.424327176191257e-02 + 2.418546306971884e-02 + 2.412779159320392e-02 + 2.407025701015483e-02 + 2.401285899981317e-02 + 2.395559723318545e-02 + 2.389847138352894e-02 + 2.384148113010493e-02 + 2.378462615809818e-02 + 2.372790614977233e-02 + 2.367132078182184e-02 + 2.361486973012364e-02 + 2.355855267628842e-02 + 2.350236930912776e-02 + 2.344631931433045e-02 + 2.339040237260871e-02 + 2.333461816188979e-02 + 2.327896637661412e-02 + 2.322344670789084e-02 + 2.316805883183489e-02 + 2.311280243515378e-02 + 2.305767721100430e-02 + 2.300268285332459e-02 + 2.294781904779433e-02 + 2.289308548135576e-02 + 2.283848185163481e-02 + 2.278400785322520e-02 + 2.272966317657758e-02 + 2.267544750906289e-02 + 2.262136054576819e-02 + 2.256740198705395e-02 + 2.251357153295969e-02 + 2.245986887460843e-02 + 2.240629370372433e-02 + 2.235284572997816e-02 + 2.229952464835474e-02 + 2.224633014749699e-02 + 2.219326193897085e-02 + 2.214031972536749e-02 + 2.208750320104361e-02 + 2.203481207216736e-02 + 2.198224604054712e-02 + 2.192980480490446e-02 + 2.187748807809197e-02 + 2.182529556587840e-02 + 2.177322696546572e-02 + 2.172128198614426e-02 + 2.166946034002218e-02 + 2.161776173674180e-02 + 2.156618587797776e-02 + 2.151473246970182e-02 + 2.146340122648612e-02 + 2.141219186176581e-02 + 2.136110408595206e-02 + 2.131013760699968e-02 + 2.125929213929720e-02 + 2.120856739943827e-02 + 2.115796310282077e-02 + 2.110747896148574e-02 + 2.105711468773744e-02 + 2.100686999708853e-02 + 2.095674460640470e-02 + 2.090673823470377e-02 + 2.085685060390323e-02 + 2.080708143284909e-02 + 2.075743043865024e-02 + 2.070789733896461e-02 + 2.065848185338519e-02 + 2.060918370358626e-02 + 2.056000261352416e-02 + 2.051093830731156e-02 + 2.046199050841276e-02 + 2.041315893918832e-02 + 2.036444332670061e-02 + 2.031584339811113e-02 + 2.026735887003067e-02 + 2.021898947028043e-02 + 2.017073493461869e-02 + 2.012259498752183e-02 + 2.007456935661649e-02 + 2.002665777297167e-02 + 1.997885996051357e-02 + 1.993117564943755e-02 + 1.988360457741823e-02 + 1.983614647798445e-02 + 1.978880107994216e-02 + 1.974156811085027e-02 + 1.969444731166336e-02 + 1.964743841723802e-02 + 1.960054115227784e-02 + 1.955375525837354e-02 + 1.950708047707909e-02 + 1.946051654129042e-02 + 1.941406318497287e-02 + 1.936772014474174e-02 + 1.932148716119990e-02 + 1.927536398140690e-02 + 1.922935034564512e-02 + 1.918344598004621e-02 + 1.913765063001423e-02 + 1.909196404425976e-02 + 1.904638595925336e-02 + 1.900091612254435e-02 + 1.895555428061772e-02 + 1.891030016469744e-02 + 1.886515352637163e-02 + 1.882011411999479e-02 + 1.877518167521970e-02 + 1.873035594232305e-02 + 1.868563668110157e-02 + 1.864102363246424e-02 + 1.859651653826993e-02 + 1.855211514697215e-02 + 1.850781921692858e-02 + 1.846362849811174e-02 + 1.841954273425713e-02 + 1.837556167722399e-02 + 1.833168508274460e-02 + 1.828791270638760e-02 + 1.824424429587613e-02 + 1.820067960186819e-02 + 1.815721838079814e-02 + 1.811386039242822e-02 + 1.807060539155448e-02 + 1.802745312809132e-02 + 1.798440336027523e-02 + 1.794145584667328e-02 + 1.789861034396235e-02 + 1.785586661445831e-02 + 1.781322441713358e-02 + 1.777068350457842e-02 + 1.772824363606736e-02 + 1.768590457480606e-02 + 1.764366608520215e-02 + 1.760152792777148e-02 + 1.755948986314423e-02 + 1.751755165530868e-02 + 1.747571306959304e-02 + 1.743397386795286e-02 + 1.739233380686453e-02 + 1.735079266012027e-02 + 1.730935019969591e-02 + 1.726800617515195e-02 + 1.722676035861175e-02 + 1.718561252878033e-02 + 1.714456244423200e-02 + 1.710360987245781e-02 + 1.706275458478830e-02 + 1.702199634263978e-02 + 1.698133492251816e-02 + 1.694077010510486e-02 + 1.690030164730331e-02 + 1.685992932068962e-02 + 1.681965290804600e-02 + 1.677947217681714e-02 + 1.673938689521877e-02 + 1.669939683666028e-02 + 1.665950177991986e-02 + 1.661970150205939e-02 + 1.657999577711447e-02 + 1.654038437906895e-02 + 1.650086708478969e-02 + 1.646144367321789e-02 + 1.642211391798679e-02 + 1.638287759543946e-02 + 1.634373448754158e-02 + 1.630468437732129e-02 + 1.626572704186934e-02 + 1.622686225251448e-02 + 1.618808979718675e-02 + 1.614940946170872e-02 + 1.611082101996680e-02 + 1.607232425440962e-02 + 1.603391895041710e-02 + 1.599560489122978e-02 + 1.595738185903658e-02 + 1.591924963852954e-02 + 1.588120801872957e-02 + 1.584325677844977e-02 + 1.580539569909259e-02 + 1.576762457813194e-02 + 1.572994319938502e-02 + 1.569235134492561e-02 + 1.565484881311209e-02 + 1.561743538476679e-02 + 1.558011083839993e-02 + 1.554287497958433e-02 + 1.550572759639263e-02 + 1.546866846769255e-02 + 1.543169739218311e-02 + 1.539481416304957e-02 + 1.535801856691179e-02 + 1.532131039389481e-02 + 1.528468944222785e-02 + 1.524815551153465e-02 + 1.521170838270258e-02 + 1.517534784896265e-02 + 1.513907371629201e-02 + 1.510288577422132e-02 + 1.506678381600347e-02 + 1.503076764230081e-02 + 1.499483704594779e-02 + 1.495899182406707e-02 + 1.492323178010325e-02 + 1.488755670734373e-02 + 1.485196640180961e-02 + 1.481646066682016e-02 + 1.478103929846549e-02 + 1.474570209596410e-02 + 1.471044886603309e-02 + 1.467527940456368e-02 + 1.464019351058384e-02 + 1.460519099481427e-02 + 1.457027165550810e-02 + 1.453543528903122e-02 + 1.450068170024789e-02 + 1.446601069487559e-02 + 1.443142207965322e-02 + 1.439691566271563e-02 + 1.436249124410753e-02 + 1.432814862414793e-02 + 1.429388761459182e-02 + 1.425970802314729e-02 + 1.422560965483165e-02 + 1.419159231701311e-02 + 1.415765581387266e-02 + 1.412379995194592e-02 + 1.409002455116132e-02 + 1.405632941792580e-02 + 1.402271435207008e-02 + 1.398917917303820e-02 + 1.395572369124169e-02 + 1.392234770932106e-02 + 1.388905104392205e-02 + 1.385583350997071e-02 + 1.382269491788422e-02 + 1.378963507987753e-02 + 1.375665380791922e-02 + 1.372375091366015e-02 + 1.369092621121959e-02 + 1.365817951715469e-02 + 1.362551064978645e-02 + 1.359291942581090e-02 + 1.356040565686591e-02 + 1.352796915179558e-02 + 1.349560973812908e-02 + 1.346332723540702e-02 + 1.343112144483463e-02 + 1.339899219360082e-02 + 1.336693930891053e-02 + 1.333496260043193e-02 + 1.330306188734133e-02 + 1.327123699224855e-02 + 1.323948773476910e-02 + 1.320781393642416e-02 + 1.317621541705235e-02 + 1.314469199166208e-02 + 1.311324348463446e-02 + 1.308186972415766e-02 + 1.305057053367403e-02 + 1.301934573289238e-02 + 1.298819514247371e-02 + 1.295711859027024e-02 + 1.292611590142988e-02 + 1.289518689817567e-02 + 1.286433140253433e-02 + 1.283354924175516e-02 + 1.280284024626999e-02 + 1.277220424320462e-02 + 1.274164105512193e-02 + 1.271115050454398e-02 + 1.268073242580564e-02 + 1.265038664885281e-02 + 1.262011299771964e-02 + 1.258991130045622e-02 + 1.255978138928737e-02 + 1.252972309744999e-02 + 1.249973624798444e-02 + 1.246982067106488e-02 + 1.243997620568067e-02 + 1.241020267461013e-02 + 1.238049990730756e-02 + 1.235086774662455e-02 + 1.232130602051964e-02 + 1.229181455798258e-02 + 1.226239319685087e-02 + 1.223304176770951e-02 + 1.220376010347082e-02 + 1.217454804486660e-02 + 1.214540542341299e-02 + 1.211633207079968e-02 + 1.208732782681391e-02 + 1.205839252771904e-02 + 1.202952600673308e-02 + 1.200072809652295e-02 + 1.197199864009422e-02 + 1.194333748171596e-02 + 1.191474445503102e-02 + 1.188621939116683e-02 + 1.185776212675846e-02 + 1.182937251288597e-02 + 1.180105038815069e-02 + 1.177279558400601e-02 + 1.174460794107161e-02 + 1.171648730396891e-02 + 1.168843351609301e-02 + 1.166044641171630e-02 + 1.163252583370262e-02 + 1.160467163102319e-02 + 1.157688364263099e-02 + 1.154916170933044e-02 + 1.152150567570712e-02 + 1.149391538555893e-02 + 1.146639068095789e-02 + 1.143893140447911e-02 + 1.141153740781534e-02 + 1.138420853673168e-02 + 1.135694462854312e-02 + 1.132974552839056e-02 + 1.130261108533809e-02 + 1.127554114921167e-02 + 1.124853556496432e-02 + 1.122159417997002e-02 + 1.119471684585664e-02 + 1.116790340283271e-02 + 1.114115369710312e-02 + 1.111446758944324e-02 + 1.108784492177717e-02 + 1.106128553754357e-02 + 1.103478929728668e-02 + 1.100835604711145e-02 + 1.098198563223047e-02 + 1.095567791133747e-02 + 1.092943273475230e-02 + 1.090324994987072e-02 + 1.087712940981543e-02 + 1.085107096608238e-02 + 1.082507447053145e-02 + 1.079913977931768e-02 + 1.077326674497864e-02 + 1.074745521859533e-02 + 1.072170505538783e-02 + 1.069601611066200e-02 + 1.067038823886727e-02 + 1.064482129354728e-02 + 1.061931513219808e-02 + 1.059386961330933e-02 + 1.056848458681972e-02 + 1.054315990640127e-02 + 1.051789543133304e-02 + 1.049269102209236e-02 + 1.046754653920045e-02 + 1.044246184044743e-02 + 1.041743677288729e-02 + 1.039247119540387e-02 + 1.036756497988909e-02 + 1.034271797805630e-02 + 1.031793004647734e-02 + 1.029320105343422e-02 + 1.026853085073464e-02 + 1.024391929539164e-02 + 1.021936625852857e-02 + 1.019487159885800e-02 + 1.017043517132599e-02 + 1.014605683522427e-02 + 1.012173646293816e-02 + 1.009747392058842e-02 + 1.007326905588179e-02 + 1.004912173629082e-02 + 1.002503183346208e-02 + 1.000099920653610e-02 + 9.977023717087764e-03 + 9.953105229386335e-03 + 9.929243608872406e-03 + 9.905438720973011e-03 + 9.881690430827046e-03 + 9.857998603258316e-03 + 9.834363104786413e-03 + 9.810783802355094e-03 + 9.787260560990621e-03 + 9.763793247942843e-03 + 9.740381731591760e-03 + 9.717025877967826e-03 + 9.693725554510663e-03 + 9.670480630152000e-03 + 9.647290973567844e-03 + 9.624156453317901e-03 + 9.601076938062841e-03 + 9.578052297039983e-03 + 9.555082399653962e-03 + 9.532167115690399e-03 + 9.509306316445141e-03 + 9.486499872512266e-03 + 9.463747653553711e-03 + 9.441049530896879e-03 + 9.418405376662433e-03 + 9.395815063104617e-03 + 9.373278461427175e-03 + 9.350795443499088e-03 + 9.328365882617906e-03 + 9.305989651970981e-03 + 9.283666624824176e-03 + 9.261396674723778e-03 + 9.239179675408710e-03 + 9.217015500757821e-03 + 9.194904024947497e-03 + 9.172845123632957e-03 + 9.150838672277813e-03 + 9.128884544858834e-03 + 9.106982617538152e-03 + 9.085132766987650e-03 + 9.063334868396023e-03 + 9.041588798714301e-03 + 9.019894435479832e-03 + 8.998251654894885e-03 + 8.976660334271958e-03 + 8.955120351755344e-03 + 8.933631585369030e-03 + 8.912193913264732e-03 + 8.890807213885918e-03 + 8.869471366214972e-03 + 8.848186249259440e-03 + 8.826951742251321e-03 + 8.805767725383001e-03 + 8.784634078744540e-03 + 8.763550682317645e-03 + 8.742517416743529e-03 + 8.721534162832835e-03 + 8.700600801502468e-03 + 8.679717214101214e-03 + 8.658883282492529e-03 + 8.638098888885700e-03 + 8.617363915025602e-03 + 8.596678243443276e-03 + 8.576041757693640e-03 + 8.555454340593486e-03 + 8.534915875110577e-03 + 8.514426244844865e-03 + 8.493985333674233e-03 + 8.473593025909724e-03 + 8.453249206335058e-03 + 8.432953759374154e-03 + 8.412706569850796e-03 + 8.392507523503252e-03 + 8.372356505624580e-03 + 8.352253401838657e-03 + 8.332198098800467e-03 + 8.312190482450988e-03 + 8.292230438871008e-03 + 8.272317855399050e-03 + 8.252452619663047e-03 + 8.232634619050869e-03 + 8.212863740331295e-03 + 8.193139871755427e-03 + 8.173462902270600e-03 + 8.153832719956143e-03 + 8.134249213438031e-03 + 8.114712271802299e-03 + 8.095221784035302e-03 + 8.075777639824947e-03 + 8.056379729417138e-03 + 8.037027942938956e-03 + 8.017722170154111e-03 + 7.998462301028743e-03 + 7.979248227381455e-03 + 7.960079840401237e-03 + 7.940957030626293e-03 + 7.921879690329333e-03 + 7.902847711453821e-03 + 7.883860985421759e-03 + 7.864919405368889e-03 + 7.846022864037674e-03 + 7.827171253341911e-03 + 7.808364466777525e-03 + 7.789602397948123e-03 + 7.770884940051978e-03 + 7.752211987488857e-03 + 7.733583434567862e-03 + 7.714999174963997e-03 + 7.696459103766686e-03 + 7.677963116338556e-03 + 7.659511107448910e-03 + 7.641102971817726e-03 + 7.622738605046243e-03 + 7.604417904226960e-03 + 7.586140765207791e-03 + 7.567907083543369e-03 + 7.549716755813157e-03 + 7.531569679511053e-03 + 7.513465752109027e-03 + 7.495404870064185e-03 + 7.477386931039745e-03 + 7.459411833357052e-03 + 7.441479474804043e-03 + 7.423589753808636e-03 + 7.405742569108545e-03 + 7.387937818890154e-03 + 7.370175402616828e-03 + 7.352455220325825e-03 + 7.334777170378787e-03 + 7.317141152312192e-03 + 7.299547066866589e-03 + 7.281994814230038e-03 + 7.264484294760120e-03 + 7.247015409129196e-03 + 7.229588058070396e-03 + 7.212202142721502e-03 + 7.194857564639825e-03 + 7.177554225359851e-03 + 7.160292026920549e-03 + 7.143070871828701e-03 + 7.125890661781023e-03 + 7.108751299151369e-03 + 7.091652687463649e-03 + 7.074594729473165e-03 + 7.057577328092346e-03 + 7.040600386920140e-03 + 7.023663809440293e-03 + 7.006767499621119e-03 + 6.989911362283671e-03 + 6.973095301761098e-03 + 6.956319222246642e-03 + 6.939583028242014e-03 + 6.922886624933184e-03 + 6.906229917977133e-03 + 6.889612813237578e-03 + 6.873035215956658e-03 + 6.856497031627250e-03 + 6.839998167124653e-03 + 6.823538528849983e-03 + 6.807118022950776e-03 + 6.790736556102037e-03 + 6.774394036012678e-03 + 6.758090370503938e-03 + 6.741825465874382e-03 + 6.725599229831710e-03 + 6.709411571076730e-03 + 6.693262397262002e-03 + 6.677151616823099e-03 + 6.661079138883049e-03 + 6.645044871881905e-03 + 6.629048724424302e-03 + 6.613090605727208e-03 + 6.597170426169362e-03 + 6.581288095176429e-03 + 6.565443521393368e-03 + 6.549636616015259e-03 + 6.533867289690503e-03 + 6.518135451870410e-03 + 6.502441013826720e-03 + 6.486783886693610e-03 + 6.471163980906190e-03 + 6.455581208648475e-03 + 6.440035481784445e-03 + 6.424526710952946e-03 + 6.409054808194274e-03 + 6.393619686148648e-03 + 6.378221257389647e-03 + 6.362859434216829e-03 + 6.347534129405275e-03 + 6.332245256702231e-03 + 6.316992728926398e-03 + 6.301776458848883e-03 + 6.286596360249587e-03 + 6.271452347303485e-03 + 6.256344334335297e-03 + 6.241272235637497e-03 + 6.226235965289892e-03 + 6.211235437617355e-03 + 6.196270567881593e-03 + 6.181341271239764e-03 + 6.166447462747857e-03 + 6.151589057705613e-03 + 6.136765971896623e-03 + 6.121978121401951e-03 + 6.107225422158738e-03 + 6.092507790132517e-03 + 6.077825141615326e-03 + 6.063177393866219e-03 + 6.048564463915550e-03 + 6.033986268442907e-03 + 6.019442724444331e-03 + 6.004933749322879e-03 + 5.990459260846786e-03 + 5.976019176895193e-03 + 5.961613415583682e-03 + 5.947241895231883e-03 + 5.932904533953020e-03 + 5.918601250164555e-03 + 5.904331962789744e-03 + 5.890096590861083e-03 + 5.875895053571317e-03 + 5.861727270328322e-03 + 5.847593160694864e-03 + 5.833492644473149e-03 + 5.819425641715960e-03 + 5.805392072163253e-03 + 5.791391855962741e-03 + 5.777424914193254e-03 + 5.763491167389796e-03 + 5.749590536018355e-03 + 5.735722941127108e-03 + 5.721888304505554e-03 + 5.708086547766434e-03 + 5.694317591457392e-03 + 5.680581357894303e-03 + 5.666877769868901e-03 + 5.653206748498332e-03 + 5.639568216215936e-03 + 5.625962096156960e-03 + 5.612388310444156e-03 + 5.598846782244085e-03 + 5.585337435180374e-03 + 5.571860191464977e-03 + 5.558414974887664e-03 + 5.545001710146245e-03 + 5.531620319651646e-03 + 5.518270727342278e-03 + 5.504952858629092e-03 + 5.491666637149790e-03 + 5.478411987013346e-03 + 5.465188833301630e-03 + 5.451997101098349e-03 + 5.438836715460566e-03 + 5.425707601454353e-03 + 5.412609684284744e-03 + 5.399542889597323e-03 + 5.386507143481766e-03 + 5.373502371672819e-03 + 5.360528500347250e-03 + 5.347585456320476e-03 + 5.334673165238440e-03 + 5.321791553290573e-03 + 5.308940548233661e-03 + 5.296120077396591e-03 + 5.283330067535706e-03 + 5.270570445063373e-03 + 5.257841138170363e-03 + 5.245142075198895e-03 + 5.232473183108559e-03 + 5.219834389656078e-03 + 5.207225623450342e-03 + 5.194646813662877e-03 + 5.182097888019903e-03 + 5.169578774433769e-03 + 5.157089403379224e-03 + 5.144629703265777e-03 + 5.132199601938705e-03 + 5.119799030212023e-03 + 5.107427917986021e-03 + 5.095086194230444e-03 + 5.082773788422456e-03 + 5.070490630851877e-03 + 5.058236652191947e-03 + 5.046011782411770e-03 + 5.033815952050060e-03 + 5.021649092096963e-03 + 5.009511132708167e-03 + 4.997402004872952e-03 + 4.985321640554899e-03 + 4.973269971256597e-03 + 4.961246927915243e-03 + 4.949252441362114e-03 + 4.937286444284467e-03 + 4.925348869112448e-03 + 4.913439647407981e-03 + 4.901558711796497e-03 + 4.889705994683389e-03 + 4.877881427891544e-03 + 4.866084944974813e-03 + 4.854316479151838e-03 + 4.842575962313763e-03 + 4.830863328342665e-03 + 4.819178511323663e-03 + 4.807521444209243e-03 + 4.795892060902922e-03 + 4.784290295384500e-03 + 4.772716080911616e-03 + 4.761169351999607e-03 + 4.749650043687660e-03 + 4.738158090505544e-03 + 4.726693426328077e-03 + 4.715255985490762e-03 + 4.703845704177584e-03 + 4.692462517255530e-03 + 4.681106359266913e-03 + 4.669777166671382e-03 + 4.658474874679097e-03 + 4.647199418051047e-03 + 4.635950733693824e-03 + 4.624728757814554e-03 + 4.613533426035275e-03 + 4.602364675085555e-03 + 4.591222441193854e-03 + 4.580106660340010e-03 + 4.569017270133631e-03 + 4.557954207729684e-03 + 4.546917409606468e-03 + 4.535906813063030e-03 + 4.524922355662980e-03 + 4.513963975024341e-03 + 4.503031608847450e-03 + 4.492125194604706e-03 + 4.481244669721014e-03 + 4.470389973075004e-03 + 4.459561043403103e-03 + 4.448757818627942e-03 + 4.437980237157132e-03 + 4.427228237584914e-03 + 4.416501758569245e-03 + 4.405800739475310e-03 + 4.395125119625431e-03 + 4.384474837838549e-03 + 4.373849833628650e-03 + 4.363250046735974e-03 + 4.352675416676511e-03 + 4.342125883231542e-03 + 4.331601386396421e-03 + 4.321101866314548e-03 + 4.310627263408189e-03 + 4.300177518182486e-03 + 4.289752570999768e-03 + 4.279352362569397e-03 + 4.268976833692708e-03 + 4.258625924837488e-03 + 4.248299577436886e-03 + 4.237997733292803e-03 + 4.227720333115902e-03 + 4.217467318344088e-03 + 4.207238631130676e-03 + 4.197034213451204e-03 + 4.186854006892898e-03 + 4.176697952964437e-03 + 4.166565994071401e-03 + 4.156458073034852e-03 + 4.146374132669977e-03 + 4.136314114934178e-03 + 4.126277962231265e-03 + 4.116265617785051e-03 + 4.106277025130100e-03 + 4.096312127225507e-03 + 4.086370866498099e-03 + 4.076453186851077e-03 + 4.066559032318193e-03 + 4.056688346384167e-03 + 4.046841072375552e-03 + 4.037017154234894e-03 + 4.027216536753351e-03 + 4.017439163691562e-03 + 4.007684978984143e-03 + 3.997953927616382e-03 + 3.988245954341580e-03 + 3.978561003853811e-03 + 3.968899020987378e-03 + 3.959259950174110e-03 + 3.949643736363211e-03 + 3.940050325917084e-03 + 3.930479663940112e-03 + 3.920931695287086e-03 + 3.911406366153988e-03 + 3.901903622192779e-03 + 3.892423408886956e-03 + 3.882965672529350e-03 + 3.873530359308857e-03 + 3.864117415340676e-03 + 3.854726787040829e-03 + 3.845358421069152e-03 + 3.836012264203419e-03 + 3.826688263119290e-03 + 3.817386364639579e-03 + 3.808106515834866e-03 + 3.798848664119608e-03 + 3.789612756624806e-03 + 3.780398740246358e-03 + 3.771206562411011e-03 + 3.762036171140922e-03 + 3.752887514765301e-03 + 3.743760540740341e-03 + 3.734655196719529e-03 + 3.725571430980986e-03 + 3.716509192135922e-03 + 3.707468428387338e-03 + 3.698449087559120e-03 + 3.689451119263604e-03 + 3.680474472533827e-03 + 3.671519094850921e-03 + 3.662584935894309e-03 + 3.653671945297336e-03 + 3.644780071212910e-03 + 3.635909263417586e-03 + 3.627059471971036e-03 + 3.618230645951014e-03 + 3.609422734605362e-03 + 3.600635687799489e-03 + 3.591869456282116e-03 + 3.583123989955615e-03 + 3.574399238584414e-03 + 3.565695152845392e-03 + 3.557011682930304e-03 + 3.548348778903569e-03 + 3.539706391532652e-03 + 3.531084471778191e-03 + 3.522482970666699e-03 + 3.513901839239252e-03 + 3.505341028388093e-03 + 3.496800489111114e-03 + 3.488280173119797e-03 + 3.479780031892506e-03 + 3.471300016604386e-03 + 3.462840078578430e-03 + 3.454400170006921e-03 + 3.445980243580072e-03 + 3.437580250715228e-03 + 3.429200142992534e-03 + 3.420839872683964e-03 + 3.412499392748279e-03 + 3.404178655562702e-03 + 3.395877612913789e-03 + 3.387596218304602e-03 + 3.379334424898988e-03 + 3.371092184764359e-03 + 3.362869450661228e-03 + 3.354666175991870e-03 + 3.346482314543412e-03 + 3.338317819175880e-03 + 3.330172642972068e-03 + 3.322046740007612e-03 + 3.313940064235304e-03 + 3.305852569386344e-03 + 3.297784208976364e-03 + 3.289734936574093e-03 + 3.281704706272533e-03 + 3.273693473070924e-03 + 3.265701191257636e-03 + 3.257727814803984e-03 + 3.249773298074863e-03 + 3.241837595979994e-03 + 3.233920663596386e-03 + 3.226022455622204e-03 + 3.218142926555998e-03 + 3.210282031163023e-03 + 3.202439725215123e-03 + 3.194615964134223e-03 + 3.186810703067027e-03 + 3.179023897602838e-03 + 3.171255503398923e-03 + 3.163505475998486e-03 + 3.155773770652277e-03 + 3.148060343553685e-03 + 3.140365151430291e-03 + 3.132688149313277e-03 + 3.125029293443853e-03 + 3.117388541383308e-03 + 3.109765848736372e-03 + 3.102161171408776e-03 + 3.094574466373093e-03 + 3.087005690728782e-03 + 3.079454801280003e-03 + 3.071921754555924e-03 + 3.064406507880629e-03 + 3.056909018530570e-03 + 3.049429243338136e-03 + 3.041967139485151e-03 + 3.034522664377966e-03 + 3.027095775562892e-03 + 3.019686430816400e-03 + 3.012294588055786e-03 + 3.004920205212209e-03 + 2.997563239705405e-03 + 2.990223649173174e-03 + 2.982901392198357e-03 + 2.975596427311395e-03 + 2.968308712822513e-03 + 2.961038206776656e-03 + 2.953784867392937e-03 + 2.946548653201694e-03 + 2.939329523194637e-03 + 2.932127436582907e-03 + 2.924942352269057e-03 + 2.917774228121468e-03 + 2.910623023643702e-03 + 2.903488699056754e-03 + 2.896371212514376e-03 + 2.889270523304112e-03 + 2.882186591679052e-03 + 2.875119376658924e-03 + 2.868068837676741e-03 + 2.861034934776638e-03 + 2.854017627809473e-03 + 2.847016876513213e-03 + 2.840032640686938e-03 + 2.833064880681456e-03 + 2.826113557103426e-03 + 2.819178630449384e-03 + 2.812260060105674e-03 + 2.805357806481411e-03 + 2.798471831428214e-03 + 2.791602094948384e-03 + 2.784748557109701e-03 + 2.777911179108391e-03 + 2.771089922400815e-03 + 2.764284748007964e-03 + 2.757495616304825e-03 + 2.750722488782177e-03 + 2.743965327115693e-03 + 2.737224092409578e-03 + 2.730498746008289e-03 + 2.723789249648372e-03 + 2.717095565426717e-03 + 2.710417654358784e-03 + 2.703755477800617e-03 + 2.697108999073383e-03 + 2.690478179939920e-03 + 2.683862981660020e-03 + 2.677263366970269e-03 + 2.670679298051182e-03 + 2.664110736894467e-03 + 2.657557646411518e-03 + 2.651019989061974e-03 + 2.644497727126834e-03 + 2.637990823748049e-03 + 2.631499241776077e-03 + 2.625022943767310e-03 + 2.618561892619121e-03 + 2.612116051581085e-03 + 2.605685383984310e-03 + 2.599269852507916e-03 + 2.592869420565223e-03 + 2.586484052278204e-03 + 2.580113710814501e-03 + 2.573758359383719e-03 + 2.567417961609419e-03 + 2.561092481362380e-03 + 2.554781882413309e-03 + 2.548486128464411e-03 + 2.542205184036424e-03 + 2.535939013422343e-03 + 2.529687580238901e-03 + 2.523450848436006e-03 + 2.517228782352248e-03 + 2.511021346720505e-03 + 2.504828506444104e-03 + 2.498650225958320e-03 + 2.492486468967693e-03 + 2.486337200680032e-03 + 2.480202386472821e-03 + 2.474081990653096e-03 + 2.467975978460781e-03 + 2.461884315215007e-03 + 2.455806965243642e-03 + 2.449743893777428e-03 + 2.443695066608286e-03 + 2.437660449328877e-03 + 2.431640007119714e-03 + 2.425633705211946e-03 + 2.419641509634817e-03 + 2.413663385966202e-03 + 2.407699299632079e-03 + 2.401749216909934e-03 + 2.395813103777745e-03 + 2.389890925926915e-03 + 2.383982649399740e-03 + 2.378088240561300e-03 + 2.372207665865560e-03 + 2.366340891258303e-03 + 2.360487883037754e-03 + 2.354648608018378e-03 + 2.348823033118129e-03 + 2.343011124539103e-03 + 2.337212848054228e-03 + 2.331428171390765e-03 + 2.325657062040943e-03 + 2.319899486412222e-03 + 2.314155410784699e-03 + 2.308424802407836e-03 + 2.302707629600105e-03 + 2.297003858807791e-03 + 2.291313456815245e-03 + 2.285636391934661e-03 + 2.279972631361444e-03 + 2.274322142294874e-03 + 2.268684892803898e-03 + 2.263060850445441e-03 + 2.257449982675230e-03 + 2.251852257331842e-03 + 2.246267642469826e-03 + 2.240696106224612e-03 + 2.235137616698782e-03 + 2.229592141925271e-03 + 2.224059650030012e-03 + 2.218540109447705e-03 + 2.213033488566449e-03 + 2.207539755750870e-03 + 2.202058879448589e-03 + 2.196590827868759e-03 + 2.191135569406817e-03 + 2.185693073538250e-03 + 2.180263309295804e-03 + 2.174846245146934e-03 + 2.169441849486111e-03 + 2.164050091629877e-03 + 2.158670941358884e-03 + 2.153304367034930e-03 + 2.147950337590323e-03 + 2.142608822828710e-03 + 2.137279792340324e-03 + 2.131963215445541e-03 + 2.126659061301170e-03 + 2.121367299363035e-03 + 2.116087899310018e-03 + 2.110820830985273e-03 + 2.105566064252215e-03 + 2.100323568822542e-03 + 2.095093314356113e-03 + 2.089875271479949e-03 + 2.084669410319118e-03 + 2.079475699908646e-03 + 2.074294110902819e-03 + 2.069124614088778e-03 + 2.063967179294363e-03 + 2.058821776747180e-03 + 2.053688376991423e-03 + 2.048566950718572e-03 + 2.043457468582260e-03 + 2.038359901240009e-03 + 2.033274219398007e-03 + 2.028200393563254e-03 + 2.023138394473247e-03 + 2.018088193625467e-03 + 2.013049761808302e-03 + 2.008023069533293e-03 + 2.003008088023720e-03 + 1.998004788922133e-03 + 1.993013143739461e-03 + 1.988033123025423e-03 + 1.983064698401174e-03 + 1.978107841939004e-03 + 1.973162523959716e-03 + 1.968228716337500e-03 + 1.963306391971906e-03 + 1.958395521239848e-03 + 1.953496075620802e-03 + 1.948608028019931e-03 + 1.943731350228688e-03 + 1.938866013789267e-03 + 1.934011990485791e-03 + 1.929169252901841e-03 + 1.924337773155020e-03 + 1.919517522737516e-03 + 1.914708474346116e-03 + 1.909910600724645e-03 + 1.905123874118434e-03 + 1.900348266781407e-03 + 1.895583751238830e-03 + 1.890830300301618e-03 + 1.886087886140893e-03 + 1.881356481302734e-03 + 1.876636059328014e-03 + 1.871926593048255e-03 + 1.867228054886164e-03 + 1.862540417322874e-03 + 1.857863653843670e-03 + 1.853197738035545e-03 + 1.848542642643349e-03 + 1.843898340659710e-03 + 1.839264805482605e-03 + 1.834642010884087e-03 + 1.830029929675155e-03 + 1.825428534756660e-03 + 1.820837800826725e-03 + 1.816257700994342e-03 + 1.811688207883416e-03 + 1.807129296515363e-03 + 1.802580940478343e-03 + 1.798043112569654e-03 + 1.793515787775185e-03 + 1.788998939681262e-03 + 1.784492541016137e-03 + 1.779996567290557e-03 + 1.775510992946437e-03 + 1.771035790955467e-03 + 1.766570935188881e-03 + 1.762116400368900e-03 + 1.757672161650980e-03 + 1.753238193179788e-03 + 1.748814468721301e-03 + 1.744400962234815e-03 + 1.739997649147064e-03 + 1.735604504535336e-03 + 1.731221502449809e-03 + 1.726848617398671e-03 + 1.722485824273258e-03 + 1.718133098140551e-03 + 1.713790413411143e-03 + 1.709457744960287e-03 + 1.705135068710917e-03 + 1.700822359316501e-03 + 1.696519591326764e-03 + 1.692226740206531e-03 + 1.687943780998710e-03 + 1.683670689031842e-03 + 1.679407440586236e-03 + 1.675154010255336e-03 + 1.670910372590760e-03 + 1.666676504276520e-03 + 1.662452380680008e-03 + 1.658237976786625e-03 + 1.654033269138933e-03 + 1.649838233189823e-03 + 1.645652843894769e-03 + 1.641477077483747e-03 + 1.637310910181093e-03 + 1.633154317979306e-03 + 1.629007276710043e-03 + 1.624869761977525e-03 + 1.620741749539434e-03 + 1.616623216289868e-03 + 1.612514138626602e-03 + 1.608414492386588e-03 + 1.604324254066237e-03 + 1.600243399893093e-03 + 1.596171905775712e-03 + 1.592109748332882e-03 + 1.588056904380057e-03 + 1.584013350591343e-03 + 1.579979063015354e-03 + 1.575954018417919e-03 + 1.571938194458784e-03 + 1.567931567017785e-03 + 1.563934112240304e-03 + 1.559945807641685e-03 + 1.555966630779868e-03 + 1.551996558434039e-03 + 1.548035566441649e-03 + 1.544083632647983e-03 + 1.540140734902817e-03 + 1.536206849423688e-03 + 1.532281953055403e-03 + 1.528366023373284e-03 + 1.524459038476067e-03 + 1.520560975283225e-03 + 1.516671810636623e-03 + 1.512791522697568e-03 + 1.508920088807869e-03 + 1.505057486015793e-03 + 1.501203692209861e-03 + 1.497358685322736e-03 + 1.493522443062933e-03 + 1.489694942724099e-03 + 1.485876162194853e-03 + 1.482066079697174e-03 + 1.478264672890618e-03 + 1.474471919644568e-03 + 1.470687798121700e-03 + 1.466912286519041e-03 + 1.463145362594138e-03 + 1.459387003990816e-03 + 1.455637189563730e-03 + 1.451895897787254e-03 + 1.448163106500405e-03 + 1.444438793947980e-03 + 1.440722938760434e-03 + 1.437015519703072e-03 + 1.433316514691718e-03 + 1.429625901910583e-03 + 1.425943660193772e-03 + 1.422269768088744e-03 + 1.418604204318803e-03 + 1.414946947952337e-03 + 1.411297977567125e-03 + 1.407657271824714e-03 + 1.404024809794961e-03 + 1.400400570139807e-03 + 1.396784531434119e-03 + 1.393176672533590e-03 + 1.389576972970703e-03 + 1.385985412235932e-03 + 1.382401969107074e-03 + 1.378826622443783e-03 + 1.375259351219489e-03 + 1.371700134514839e-03 + 1.368148952079336e-03 + 1.364605783757036e-03 + 1.361070608632290e-03 + 1.357543405835987e-03 + 1.354024154621594e-03 + 1.350512834317403e-03 + 1.347009424934256e-03 + 1.343513906652181e-03 + 1.340026258589547e-03 + 1.336546459739890e-03 + 1.333074489680093e-03 + 1.329610329681042e-03 + 1.326153959375566e-03 + 1.322705357081586e-03 + 1.319264503327096e-03 + 1.315831378649060e-03 + 1.312405962942001e-03 + 1.308988235813027e-03 + 1.305578177302038e-03 + 1.302175767936687e-03 + 1.298780987783750e-03 + 1.295393816813521e-03 + 1.292014235092867e-03 + 1.288642222828844e-03 + 1.285277760581296e-03 + 1.281920829182064e-03 + 1.278571408463430e-03 + 1.275229478684235e-03 + 1.271895021211065e-03 + 1.268568016022571e-03 + 1.265248443161927e-03 + 1.261936283866222e-03 + 1.258631518851812e-03 + 1.255334128626978e-03 + 1.252044093890807e-03 + 1.248761395151665e-03 + 1.245486013113075e-03 + 1.242217929129610e-03 + 1.238957124307338e-03 + 1.235703579418827e-03 + 1.232457274959363e-03 + 1.229218191838193e-03 + 1.225986311430220e-03 + 1.222761615473843e-03 + 1.219544084856006e-03 + 1.216333699988417e-03 + 1.213130441964228e-03 + 1.209934292565758e-03 + 1.206745233676102e-03 + 1.203563245963205e-03 + 1.200388310525061e-03 + 1.197220409053557e-03 + 1.194059523052293e-03 + 1.190905633902454e-03 + 1.187758723052301e-03 + 1.184618772531461e-03 + 1.181485763720917e-03 + 1.178359677471923e-03 + 1.175240496495629e-03 + 1.172128202790389e-03 + 1.169022777016151e-03 + 1.165924201427151e-03 + 1.162832457974330e-03 + 1.159747527667428e-03 + 1.156669393342115e-03 + 1.153598037513975e-03 + 1.150533441195656e-03 + 1.147475586343000e-03 + 1.144424455192698e-03 + 1.141380029693999e-03 + 1.138342292070108e-03 + 1.135311224784675e-03 + 1.132286810371483e-03 + 1.129269030382949e-03 + 1.126257866687504e-03 + 1.123253302806332e-03 + 1.120255320585528e-03 + 1.117263901594027e-03 + 1.114279029333705e-03 + 1.111300685926414e-03 + 1.108328853214485e-03 + 1.105363515126287e-03 + 1.102404653695880e-03 + 1.099452250276090e-03 + 1.096506289065756e-03 + 1.093566752790314e-03 + 1.090633622975802e-03 + 1.087706882683815e-03 + 1.084786514992603e-03 + 1.081872502714538e-03 + 1.078964828755093e-03 + 1.076063476054719e-03 + 1.073168427526746e-03 + 1.070279665938792e-03 + 1.067397174442816e-03 + 1.064520936417689e-03 + 1.061650934087830e-03 + 1.058787150577005e-03 + 1.055929570192933e-03 + 1.053078175257412e-03 + 1.050232948596978e-03 + 1.047393874491941e-03 + 1.044560935952646e-03 + 1.041734115621369e-03 + 1.038913396581877e-03 + 1.036098763274602e-03 + 1.033290199394467e-03 + 1.030487686748966e-03 + 1.027691209748195e-03 + 1.024900752824401e-03 + 1.022116298137994e-03 + 1.019337829638554e-03 + 1.016565331581354e-03 + 1.013798786681050e-03 + 1.011038178744547e-03 + 1.008283492069452e-03 + 1.005534710312413e-03 + 1.002791817167788e-03 + 1.000054796401326e-03 + 9.973236317525795e-04 + 9.945983071579347e-04 + 9.918788066535795e-04 + 9.891651141399331e-04 + 9.864572135795289e-04 + 9.837550890187182e-04 + 9.810587245440681e-04 + 9.783681042165289e-04 + 9.756832121036981e-04 + 9.730040324093254e-04 + 9.703305493295432e-04 + 9.676627470403841e-04 + 9.650006098016114e-04 + 9.623441219373007e-04 + 9.596932677964791e-04 + 9.570480316324495e-04 + 9.544083978205768e-04 + 9.517743508934125e-04 + 9.491458752396962e-04 + 9.465229553012878e-04 + 9.439055756501852e-04 + 9.412937207895038e-04 + 9.386873752809525e-04 + 9.360865238076440e-04 + 9.334911509791107e-04 + 9.309012414140627e-04 + 9.283167798171848e-04 + 9.257377509839119e-04 + 9.231641396993918e-04 + 9.205959306774516e-04 + 9.180331088080401e-04 + 9.154756590138127e-04 + 9.129235660962607e-04 + 9.103768149701878e-04 + 9.078353906321772e-04 + 9.052992780955343e-04 + 9.027684623946166e-04 + 9.002429285800444e-04 + 8.977226617159664e-04 + 8.952076469091944e-04 + 8.926978693026045e-04 + 8.901933140640126e-04 + 8.876939664445818e-04 + 8.851998117353022e-04 + 8.827108351548252e-04 + 8.802270219825557e-04 + 8.777483575705740e-04 + 8.752748272927118e-04 + 8.728064165680979e-04 + 8.703431108522175e-04 + 8.678848955854838e-04 + 8.654317562117358e-04 + 8.629836782258149e-04 + 8.605406473168484e-04 + 8.581026490766350e-04 + 8.556696689660546e-04 + 8.532416927295966e-04 + 8.508187061075220e-04 + 8.484006947265324e-04 + 8.459876443379812e-04 + 8.435795407180884e-04 + 8.411763696276702e-04 + 8.387781169670006e-04 + 8.363847686436173e-04 + 8.339963104952486e-04 + 8.316127284703911e-04 + 8.292340085393665e-04 + 8.268601366269382e-04 + 8.244910987818749e-04 + 8.221268810991789e-04 + 8.197674696351961e-04 + 8.174128504795555e-04 + 8.150630097863373e-04 + 8.127179337987174e-04 + 8.103776086785577e-04 + 8.080420206116735e-04 + 8.057111559859151e-04 + 8.033850010442538e-04 + 8.010635420033148e-04 + 7.987467653572807e-04 + 7.964346575211490e-04 + 7.941272048473775e-04 + 7.918243938145987e-04 + 7.895262109139402e-04 + 7.872326426395918e-04 + 7.849436755404454e-04 + 7.826592961986925e-04 + 7.803794912312032e-04 + 7.781042473133076e-04 + 7.758335511109105e-04 + 7.735673892815593e-04 + 7.713057485643111e-04 + 7.690486157536814e-04 + 7.667959776727836e-04 + 7.645478211004869e-04 + 7.623041328912646e-04 + 7.600649000053854e-04 + 7.578301093264215e-04 + 7.555997477583347e-04 + 7.533738022910566e-04 + 7.511522599897612e-04 + 7.489351079282785e-04 + 7.467223331578882e-04 + 7.445139227962291e-04 + 7.423098639775247e-04 + 7.401101438263767e-04 + 7.379147496119391e-04 + 7.357236686066337e-04 + 7.335368879604144e-04 + 7.313543950177981e-04 + 7.291761771981462e-04 + 7.270022218267318e-04 + 7.248325162821272e-04 + 7.226670479949025e-04 + 7.205058044241288e-04 + 7.183487730730944e-04 + 7.161959414766152e-04 + 7.140472971809699e-04 + 7.119028277561113e-04 + 7.097625208231147e-04 + 7.076263641063239e-04 + 7.054943452576004e-04 + 7.033664519218169e-04 + 7.012426719956601e-04 + 6.991229932510109e-04 + 6.970074033540157e-04 + 6.948958902400331e-04 + 6.927884418689321e-04 + 6.906850461485944e-04 + 6.885856909689247e-04 + 6.864903643067344e-04 + 6.843990542306164e-04 + 6.823117487354689e-04 + 6.802284359113009e-04 + 6.781491039842488e-04 + 6.760737410664455e-04 + 6.740023352773578e-04 + 6.719348748282754e-04 + 6.698713480434448e-04 + 6.678117432465053e-04 + 6.657560487034911e-04 + 6.637042527655789e-04 + 6.616563438432147e-04 + 6.596123103782944e-04 + 6.575721408025738e-04 + 6.555358236048545e-04 + 6.535033473936704e-04 + 6.514747007198682e-04 + 6.494498721244171e-04 + 6.474288502266283e-04 + 6.454116237722280e-04 + 6.433981815196293e-04 + 6.413885120963034e-04 + 6.393826043225120e-04 + 6.373804471016686e-04 + 6.353820291728463e-04 + 6.333873394523912e-04 + 6.313963669651093e-04 + 6.294091005879305e-04 + 6.274255292885601e-04 + 6.254456421348147e-04 + 6.234694281911996e-04 + 6.214968766042923e-04 + 6.195279765738893e-04 + 6.175627172048796e-04 + 6.156010877068991e-04 + 6.136430774102346e-04 + 6.116886755862946e-04 + 6.097378715435904e-04 + 6.077906546697107e-04 + 6.058470144286753e-04 + 6.039069402835228e-04 + 6.019704216802507e-04 + 6.000374481409778e-04 + 5.981080092530985e-04 + 5.961820946588096e-04 + 5.942596940083503e-04 + 5.923407969721301e-04 + 5.904253932616448e-04 + 5.885134726736782e-04 + 5.866050250202793e-04 + 5.847000400980857e-04 + 5.827985078791194e-04 + 5.809004183470366e-04 + 5.790057613579966e-04 + 5.771145269606196e-04 + 5.752267052829598e-04 + 5.733422863841403e-04 + 5.714612604000550e-04 + 5.695836175305099e-04 + 5.677093480037018e-04 + 5.658384420678564e-04 + 5.639708900266142e-04 + 5.621066823022896e-04 + 5.602458092869138e-04 + 5.583882613744714e-04 + 5.565340290827777e-04 + 5.546831029664671e-04 + 5.528354736013446e-04 + 5.509911316015094e-04 + 5.491500676402182e-04 + 5.473122724461832e-04 + 5.454777367810608e-04 + 5.436464514412940e-04 + 5.418184072613321e-04 + 5.399935951226492e-04 + 5.381720060069473e-04 + 5.363536309736959e-04 + 5.345384609776805e-04 + 5.327264870731968e-04 + 5.309177004752346e-04 + 5.291120923593077e-04 + 5.273096539494221e-04 + 5.255103765544823e-04 + 5.237142514407170e-04 + 5.219212699733492e-04 + 5.201314236865307e-04 + 5.183447040146093e-04 + 5.165611024246747e-04 + 5.147806105331874e-04 + 5.130032200114061e-04 + 5.112289225530487e-04 + 5.094577098620987e-04 + 5.076895736854020e-04 + 5.059245058583464e-04 + 5.041624983517820e-04 + 5.024035430839573e-04 + 5.006476319990773e-04 + 4.988947572035529e-04 + 4.971449107942239e-04 + 4.953980848818115e-04 + 4.936542716805855e-04 + 4.919134634956966e-04 + 4.901756526846954e-04 + 4.884408316006080e-04 + 4.867089926403434e-04 + 4.849801282731475e-04 + 4.832542310892493e-04 + 4.815312937309208e-04 + 4.798113088560132e-04 + 4.780942690954599e-04 + 4.763801672259719e-04 + 4.746689961731605e-04 + 4.729607488395965e-04 + 4.712554181333483e-04 + 4.695529970142063e-04 + 4.678534786182833e-04 + 4.661568561056742e-04 + 4.644631226216267e-04 + 4.627722714285577e-04 + 4.610842958399921e-04 + 4.593991892076899e-04 + 4.577169450300335e-04 + 4.560375568336446e-04 + 4.543610181249096e-04 + 4.526873225594637e-04 + 4.510164638467393e-04 + 4.493484356950898e-04 + 4.476832319569833e-04 + 4.460208465382815e-04 + 4.443612733350173e-04 + 4.427045064285867e-04 + 4.410505399510805e-04 + 4.393993679504339e-04 + 4.377509846774214e-04 + 4.361053844834223e-04 + 4.344625616662297e-04 + 4.328225106365193e-04 + 4.311852259097922e-04 + 4.295507020701359e-04 + 4.279189337521271e-04 + 4.262899156467061e-04 + 4.246636425301464e-04 + 4.230401092565931e-04 + 4.214193107478168e-04 + 4.198012419839599e-04 + 4.181858979868060e-04 + 4.165732738692198e-04 + 4.149633649793705e-04 + 4.133561665739344e-04 + 4.117516738453971e-04 + 4.101498823326724e-04 + 4.085507876181731e-04 + 4.069543852399522e-04 + 4.053606707914893e-04 + 4.037696400086752e-04 + 4.021812887746215e-04 + 4.005956129436248e-04 + 3.990126084760660e-04 + 3.974322714846641e-04 + 3.958545980293214e-04 + 3.942795842513983e-04 + 3.927072264688036e-04 + 3.911375211195356e-04 + 3.895704646516968e-04 + 3.880060534869329e-04 + 3.864442842922655e-04 + 3.848851538066537e-04 + 3.833286587063809e-04 + 3.817747958651508e-04 + 3.802235622593628e-04 + 3.786749548688228e-04 + 3.771289708205180e-04 + 3.755856073392733e-04 + 3.740448616816441e-04 + 3.725067311649692e-04 + 3.709712132043737e-04 + 3.694383053790672e-04 + 3.679080053579325e-04 + 3.663803108503057e-04 + 3.648552195606523e-04 + 3.633327293646475e-04 + 3.618128382932607e-04 + 3.602955444344858e-04 + 3.587808459078391e-04 + 3.572687408998931e-04 + 3.557592277792442e-04 + 3.542523049927096e-04 + 3.527479710408145e-04 + 3.512462245019904e-04 + 3.497470640500036e-04 + 3.482504884770882e-04 + 3.467564967437580e-04 + 3.452650877985240e-04 + 3.437762605764859e-04 + 3.422900143250168e-04 + 3.408063483640538e-04 + 3.393252619896345e-04 + 3.378467546103840e-04 + 3.363708257653936e-04 + 3.348974751341996e-04 + 3.334267024660963e-04 + 3.319585075646510e-04 + 3.304928903038992e-04 + 3.290298507212495e-04 + 3.275693889604049e-04 + 3.261115052402690e-04 + 3.246561999078441e-04 + 3.232034733603981e-04 + 3.217533259984899e-04 + 3.203057585048498e-04 + 3.188607716794152e-04 + 3.174183662346357e-04 + 3.159785430822741e-04 + 3.145413032833238e-04 + 3.131066479369465e-04 + 3.116745782136557e-04 + 3.102450954058189e-04 + 3.088182010197684e-04 + 3.073938965650610e-04 + 3.059721835802734e-04 + 3.045530637916020e-04 + 3.031365390386899e-04 + 3.017226112552691e-04 + 3.003112824793533e-04 + 2.989025548234045e-04 + 2.974964304955006e-04 + 2.960929118913988e-04 + 2.946920014422941e-04 + 2.932937016150965e-04 + 2.918980151023185e-04 + 2.905049446767149e-04 + 2.891144931550728e-04 + 2.877266634923031e-04 + 2.863414587568181e-04 + 2.849588821248430e-04 + 2.835789368717610e-04 + 2.822016263526234e-04 + 2.808269540128114e-04 + 2.794549234525530e-04 + 2.780855383858511e-04 + 2.767188026160350e-04 + 2.753547199733251e-04 + 2.739932944362576e-04 + 2.726345302253411e-04 + 2.712784314751994e-04 + 2.699250024118205e-04 + 2.685742475595017e-04 + 2.672261714415124e-04 + 2.658807786371782e-04 + 2.645380739000219e-04 + 2.631980619957597e-04 + 2.618607478073471e-04 + 2.605261365239288e-04 + 2.591942332540738e-04 + 2.578650431168888e-04 + 2.565385715181067e-04 + 2.552148239188000e-04 + 2.538938058269625e-04 + 2.525755228940477e-04 + 2.512599808919393e-04 + 2.499471856899045e-04 + 2.486371432244064e-04 + 2.473298595107682e-04 + 2.460253406646765e-04 + 2.447235929605613e-04 + 2.434246227773826e-04 + 2.421284365723855e-04 + 2.408350408491505e-04 + 2.395444422583646e-04 + 2.382566475978551e-04 + 2.369716636126905e-04 + 2.356894972066169e-04 + 2.344101555067014e-04 + 2.331336455922197e-04 + 2.318599746393059e-04 + 2.305891500038487e-04 + 2.293211790903458e-04 + 2.280560693749351e-04 + 2.267938284406659e-04 + 2.255344639796373e-04 + 2.242779837481179e-04 + 2.230243955584571e-04 + 2.217737074088738e-04 + 2.205259273750273e-04 + 2.192810635194828e-04 + 2.180391240377155e-04 + 2.168001172473318e-04 + 2.155640515683406e-04 + 2.143309354202995e-04 + 2.131007772856537e-04 + 2.118735858306198e-04 + 2.106493698084787e-04 + 2.094281380089542e-04 + 2.082098992208740e-04 + 2.069946624015073e-04 + 2.057824366286796e-04 + 2.045732309577213e-04 + 2.033670545521807e-04 + 2.021639166762670e-04 + 2.009638266139778e-04 + 1.997667937615743e-04 + 1.985728276099416e-04 + 1.973819376249420e-04 + 1.961941334186818e-04 + 1.950094247288422e-04 + 1.938278211445808e-04 + 1.926493324216112e-04 + 1.914739685443153e-04 + 1.903017394244089e-04 + 1.891326549660665e-04 + 1.879667251315557e-04 + 1.868039600691931e-04 + 1.856443699312781e-04 + 1.844879648170397e-04 + 1.833347550402995e-04 + 1.821847509305592e-04 + 1.810379627269367e-04 + 1.798944008330572e-04 + 1.787540757117569e-04 + 1.776169978023503e-04 + 1.764831776083264e-04 + 1.753526257075039e-04 + 1.742253527510645e-04 + 1.731013693254543e-04 + 1.719806860311682e-04 + 1.708633135911149e-04 + 1.697492627671219e-04 + 1.686385443331759e-04 + 1.675311690584120e-04 + 1.664271477052953e-04 + 1.653264910845736e-04 + 1.642292101438652e-04 + 1.631353157524277e-04 + 1.620448187436031e-04 + 1.609577300754615e-04 + 1.598740606888992e-04 + 1.587938214812250e-04 + 1.577170233258566e-04 + 1.566436772168694e-04 + 1.555737942159133e-04 + 1.545073851968179e-04 + 1.534444610998845e-04 + 1.523850329588589e-04 + 1.513291117056277e-04 + 1.502767082718022e-04 + 1.492278336091406e-04 + 1.481824986061289e-04 + 1.471407142013266e-04 + 1.461024913960903e-04 + 1.450678410475480e-04 + 1.440367739871709e-04 + 1.430093010797408e-04 + 1.419854331666308e-04 + 1.409651810501559e-04 + 1.399485554884026e-04 + 1.389355672434459e-04 + 1.379262270311977e-04 + 1.369205454852166e-04 + 1.359185332657421e-04 + 1.349202010239166e-04 + 1.339255593445051e-04 + 1.329346186642912e-04 + 1.319473893926901e-04 + 1.309638820305585e-04 + 1.299841069283569e-04 + 1.290080743627276e-04 + 1.280357946577360e-04 + 1.270672779908064e-04 + 1.261025344295187e-04 + 1.251415740258344e-04 + 1.241844068414831e-04 + 1.232310428585276e-04 + 1.222814917974582e-04 + 1.213357634573326e-04 + 1.203938676644921e-04 + 1.194558139769228e-04 + 1.185216119071049e-04 + 1.175912709437735e-04 + 1.166648004395201e-04 + 1.157422096820540e-04 + 1.148235078941047e-04 + 1.139087041565015e-04 + 1.129978074560730e-04 + 1.120908266963163e-04 + 1.111877706801771e-04 + 1.102886481165339e-04 + 1.093934676008822e-04 + 1.085022375478793e-04 + 1.076149663238430e-04 + 1.067316622735248e-04 + 1.058523335072715e-04 + 1.049769880021785e-04 + 1.041056336506420e-04 + 1.032382782134568e-04 + 1.023749293560659e-04 + 1.015155946454073e-04 + 1.006602814254268e-04 + 9.980899691604087e-05 + 9.896174827822380e-05 + 9.811854249648769e-05 + 9.727938640362722e-05 + 9.644428669167931e-05 + 9.561324990949941e-05 + 9.478628247235907e-05 + 9.396339064407664e-05 + 9.314458052650697e-05 + 9.232985807344924e-05 + 9.151922908266137e-05 + 9.071269919463392e-05 + 8.991027389908417e-05 + 8.911195851702552e-05 + 8.831775819982511e-05 + 8.752767793866873e-05 + 8.674172256023528e-05 + 8.595989672073575e-05 + 8.518220490590348e-05 + 8.440865141920385e-05 + 8.363924039733809e-05 + 8.287397580722622e-05 + 8.211286142104977e-05 + 8.135590084051729e-05 + 8.060309749193613e-05 + 7.985445459658365e-05 + 7.910997520302950e-05 + 7.836966218298747e-05 + 7.763351820320998e-05 + 7.690154574559998e-05 + 7.617374709914362e-05 + 7.545012436145964e-05 + 7.473067944430294e-05 + 7.401541405207780e-05 + 7.330432968888297e-05 + 7.259742767376570e-05 + 7.189470911942744e-05 + 7.119617492698978e-05 + 7.050182580949816e-05 + 6.981166227552030e-05 + 6.912568461899398e-05 + 6.844389294104750e-05 + 6.776628713300132e-05 + 6.709286687130013e-05 + 6.642363163388969e-05 + 6.575858067871575e-05 + 6.509771305493562e-05 + 6.444102761503343e-05 + 6.378852298738557e-05 + 6.314019758583242e-05 + 6.249604962295056e-05 + 6.185607708979005e-05 + 6.122027775903875e-05 + 6.058864919749181e-05 + 5.996118875684596e-05 + 5.933789356597989e-05 + 5.871876053888907e-05 + 5.810378637807827e-05 + 5.749296757228591e-05 + 5.688630039241354e-05 + 5.628378087841946e-05 + 5.568540486517338e-05 + 5.509116797959273e-05 + 5.450106561933422e-05 + 5.391509296529475e-05 + 5.333324497665832e-05 + 5.275551640753726e-05 + 5.218190179953343e-05 + 5.161239545977182e-05 + 5.104699148146081e-05 + 5.048568375913861e-05 + 4.992846596763329e-05 + 4.937533154817934e-05 + 4.882627374147808e-05 + 4.828128557792042e-05 + 4.774035986717692e-05 + 4.720348921279551e-05 + 4.667066599018748e-05 + 4.614188237688400e-05 + 4.561713035194944e-05 + 4.509640165438987e-05 + 4.457968783032909e-05 + 4.406698023190481e-05 + 4.355826996891768e-05 + 4.305354796420506e-05 + 4.255280495560636e-05 + 4.205603144616347e-05 + 4.156321774032957e-05 + 4.107435395613752e-05 + 4.058942999665175e-05 + 4.010843556869507e-05 + 3.963136019085871e-05 + 3.915819316952545e-05 + 3.868892361719641e-05 + 3.822354046226270e-05 + 3.776203244031561e-05 + 3.730438809005640e-05 + 3.685059575689182e-05 + 3.640064360230198e-05 + 3.595451960373741e-05 + 3.551221155134442e-05 + 3.507370705450421e-05 + 3.463899354241650e-05 + 3.420805825588243e-05 + 3.378088825986720e-05 + 3.335747045272196e-05 + 3.293779155463552e-05 + 3.252183811026268e-05 + 3.210959649877579e-05 + 3.170105292073684e-05 + 3.129619341367949e-05 + 3.089500386055728e-05 + 3.049746997239612e-05 + 3.010357730585059e-05 + 2.971331126487755e-05 + 2.932665707619367e-05 + 2.894359982344077e-05 + 2.856412445132613e-05 + 2.818821573877814e-05 + 2.781585831987375e-05 + 2.744703669454755e-05 + 2.708173520948728e-05 + 2.671993807124836e-05 + 2.636162935767626e-05 + 2.600679300301263e-05 + 2.565541280257304e-05 + 2.530747242805460e-05 + 2.496295542635910e-05 + 2.462184521206604e-05 + 2.428412507785163e-05 + 2.394977819771233e-05 + 2.361878762181457e-05 + 2.329113628448998e-05 + 2.296680701293541e-05 + 2.264578251757474e-05 + 2.232804539557172e-05 + 2.201357814530585e-05 + 2.170236315904943e-05 + 2.139438272306276e-05 + 2.108961903561866e-05 + 2.078805419745760e-05 + 2.048967020582218e-05 + 2.019444897050880e-05 + 1.990237231614296e-05 + 1.961342197777572e-05 + 1.932757961462368e-05 + 1.904482680353188e-05 + 1.876514502948172e-05 + 1.848851571515060e-05 + 1.821492021640795e-05 + 1.794433980831370e-05 + 1.767675569461099e-05 + 1.741214902258527e-05 + 1.715050088166273e-05 + 1.689179228451716e-05 + 1.663600419203502e-05 + 1.638311752659296e-05 + 1.613311314360082e-05 + 1.588597184706167e-05 + 1.564167440067926e-05 + 1.540020151829502e-05 + 1.516153387453479e-05 + 1.492565211244357e-05 + 1.469253682958783e-05 + 1.446216858599219e-05 + 1.423452791421707e-05 + 1.400959531801727e-05 + 1.378735127470914e-05 + 1.356777623611474e-05 + 1.335085063337062e-05 + 1.313655487876277e-05 + 1.292486936494610e-05 + 1.271577446639333e-05 + 1.250925054517824e-05 + 1.230527796605363e-05 + 1.210383708165599e-05 + 1.190490823144681e-05 + 1.170847175649016e-05 + 1.151450799717159e-05 + 1.132299729345123e-05 + 1.113391999104486e-05 + 1.094725644586371e-05 + 1.076298702423590e-05 + 1.058109209097755e-05 + 1.040155202830912e-05 + 1.022434724511555e-05 + 1.004945815519835e-05 + 9.876865191434875e-06 + 9.706548817576976e-06 + 9.538489515829050e-06 + 9.372667791395497e-06 + 9.209064181113714e-06 + 9.047659251528417e-06 + 8.888433598428113e-06 + 8.731367852817854e-06 + 8.576442682615395e-06 + 8.423638791088146e-06 + 8.272936922653283e-06 + 8.124317863654203e-06 + 7.977762440936675e-06 + 7.833251528895281e-06 + 7.690766049713759e-06 + 7.550286971453687e-06 + 7.411795314787993e-06 + 7.275272152381047e-06 + 7.140698608042407e-06 + 7.008055865069554e-06 + 6.877325162999093e-06 + 6.748487797069182e-06 + 6.621525126924672e-06 + 6.496418573295794e-06 + 6.373149617974847e-06 + 6.251699810845691e-06 + 6.132050767139283e-06 + 6.014184168887862e-06 + 5.898081770597224e-06 + 5.783725394973654e-06 + 5.671096936050662e-06 + 5.560178365641361e-06 + 5.450951726966341e-06 + 5.343399138917566e-06 + 5.237502802602254e-06 + 5.133244993336976e-06 + 5.030608065950253e-06 + 4.929574461382145e-06 + 4.830126698086031e-06 + 4.732247378331277e-06 + 4.635919192753656e-06 + 4.541124912789582e-06 + 4.447847398958468e-06 + 4.356069602328335e-06 + 4.265774557056776e-06 + 4.176945389932598e-06 + 4.089565320281935e-06 + 4.003617653697645e-06 + 3.919085793399028e-06 + 3.835953236362346e-06 + 3.754203567594348e-06 + 3.673820472678765e-06 + 3.594787733829004e-06 + 3.517089224074456e-06 + 3.440708918621393e-06 + 3.365630891050476e-06 + 3.291839308866544e-06 + 3.219318444550688e-06 + 3.148052669614368e-06 + 3.078026451876596e-06 + 3.009224366838337e-06 + 2.941631088840265e-06 + 2.875231390541817e-06 + 2.810010156134118e-06 + 2.745952369256776e-06 + 2.683043113446972e-06 + 2.621267584374269e-06 + 2.560611078236376e-06 + 2.501058994098514e-06 + 2.442596843868032e-06 + 2.385210240033010e-06 + 2.328884899906223e-06 + 2.273606655048426e-06 + 2.219361438179688e-06 + 2.166135288678882e-06 + 2.113914360085187e-06 + 2.062684907096663e-06 + 2.012433292648363e-06 + 1.963145994708785e-06 + 1.914809593193956e-06 + 1.867410777738679e-06 + 1.820936351687259e-06 + 1.775373220609589e-06 + 1.730708401732302e-06 + 1.686929024947410e-06 + 1.644022323067757e-06 + 1.601975642837270e-06 + 1.560776441656341e-06 + 1.520412279298016e-06 + 1.480870830315665e-06 + 1.442139880698395e-06 + 1.404207319625165e-06 + 1.367061149777226e-06 + 1.330689483155573e-06 + 1.295080535963615e-06 + 1.260222640873472e-06 + 1.226104237141460e-06 + 1.192713867093549e-06 + 1.160040189500492e-06 + 1.128071969424555e-06 + 1.096798076008267e-06 + 1.066207493417804e-06 + 1.036289310659725e-06 + 1.007032721493895e-06 + 9.784270338592895e-07 + 9.504616588874799e-07 + 9.231261128777845e-07 + 8.964100256766087e-07 + 8.703031286825232e-07 + 8.447952585150839e-07 + 8.198763641874936e-07 + 7.955364948286083e-07 + 7.717658049346632e-07 + 7.485545598914059e-07 + 7.258931238786546e-07 + 7.037719665410901e-07 + 6.821816664988581e-07 + 6.611128998791421e-07 + 6.405564483055948e-07 + 6.205032003278225e-07 + 6.009441408121614e-07 + 5.818703600152187e-07 + 5.632730528626383e-07 + 5.451435094097432e-07 + 5.274731247677268e-07 + 5.102533965000488e-07 + 4.934759162480692e-07 + 4.771323800991179e-07 + 4.612145841662162e-07 + 4.457144176881476e-07 + 4.306238737382497e-07 + 4.159350428538716e-07 + 4.016401077051372e-07 + 3.877313539402903e-07 + 3.742011622647174e-07 + 3.610420046499709e-07 + 3.482464547968944e-07 + 3.358071790150475e-07 + 3.237169340794661e-07 + 3.119685770955318e-07 + 3.005550554375413e-07 + 2.894694062090586e-07 + 2.787047650882795e-07 + 2.682543557659438e-07 + 2.581114909968296e-07 + 2.482695801165890e-07 + 2.387221182727572e-07 + 2.294626890008751e-07 + 2.204849703818711e-07 + 2.117827241548630e-07 + 2.033497996067267e-07 + 1.951801384624956e-07 + 1.872677641420447e-07 + 1.796067867093452e-07 + 1.721914062527642e-07 + 1.650159027896018e-07 + 1.580746421942365e-07 + 1.513620776690125e-07 + 1.448727406655565e-07 + 1.386012477684331e-07 + 1.325423000867771e-07 + 1.266906752868051e-07 + 1.210412352836648e-07 + 1.155889239131470e-07 + 1.103287600176973e-07 + 1.052558455547364e-07 + 1.003653617115032e-07 + 9.565256321109978e-08 + 9.111278671292873e-08 + 8.674144556415159e-08 + 8.253402527034383e-08 + 7.848609177536523e-08 + 7.459328520671134e-08 + 7.085131661573615e-08 + 6.725597593242166e-08 + 6.380312478868793e-08 + 6.048869454868003e-08 + 5.730869376312298e-08 + 5.425920030057914e-08 + 5.133636064443935e-08 + 4.853639660958214e-08 + 4.585559704969457e-08 + 4.329031836991616e-08 + 4.083699031045398e-08 + 3.849210751108346e-08 + 3.625223113812888e-08 + 3.411399355038575e-08 + 3.207408998824199e-08 + 3.012928121511996e-08 + 2.827639696078167e-08 + 2.651232793166280e-08 + 2.483402934338396e-08 + 2.323852311474514e-08 + 2.172289034680298e-08 + 2.028427559044146e-08 + 1.891988778877689e-08 + 1.762699339728036e-08 + 1.640292125402531e-08 + 1.524506224991720e-08 + 1.415086320949184e-08 + 1.311783222446413e-08 + 1.214353712508316e-08 + 1.122560019257377e-08 + 1.036170378687137e-08 + 9.549587722603005e-09 + 8.787044877843281e-09 + 8.071926961605364e-09 + 7.402140903161946e-09 + 6.775645388472587e-09 + 6.190456604129221e-09 + 5.644643792323680e-09 + 5.136326726146161e-09 + 4.663681262833891e-09 + 4.224934232596794e-09 + 3.818361845837701e-09 + 3.442294891302550e-09 + 3.095113141246544e-09 + 2.775244665814773e-09 + 2.481170514027403e-09 + 2.211418834557044e-09 + 1.964565051498559e-09 + 1.739235878927056e-09 + 1.534103359980556e-09 + 1.347885831336700e-09 + 1.179351163642234e-09 + 1.027310894648576e-09 + 8.906218977827446e-10 + 7.681887799049897e-10 + 6.589582548472836e-10 + 5.619214238374888e-10 + 4.761152985830568e-10 + 4.006175343477256e-10 + 3.345492227248809e-10 + 2.770755362128679e-10 + 2.274009179016229e-10 + 1.847722840116674e-10 + 1.484788129161787e-10 + 1.178476648704315e-10 + 9.224748788184573e-11 + 7.108739902161543e-11 + 5.381328706422649e-11 + 3.991151299277479e-11 + 2.890715266694438e-11 + 2.036091648096713e-11 + 1.387293445657948e-11 + 9.080348320515963e-12 + 5.654863833225597e-12 + 3.306507674167889e-12 + 1.780689492983021e-12 + 8.563646514161430e-13 + 3.496037276049943e-13 + 1.104943293034196e-13 + 2.161027650041889e-14 + 5.028049336448720e-16 + 0.000000000000000e+00 + 0.000000000000000e+00 + 7.485272593961791e+04 + 1.493519318697679e+05 + 2.234970831294996e+05 + 2.972876533345640e+05 + 3.707231243774131e+05 + 4.438029864272003e+05 + 5.165267379297798e+05 + 5.888938856077072e+05 + 6.609039444602393e+05 + 7.325564377633340e+05 + 8.038508970696502e+05 + 8.747868622085482e+05 + 9.453638812860892e+05 + 1.015581510685036e+06 + 1.085439315064852e+06 + 1.154936867361703e+06 + 1.224073748788454e+06 + 1.292849548834672e+06 + 1.361263865266626e+06 + 1.429316304127287e+06 + 1.497006479736322e+06 + 1.564334014690106e+06 + 1.631298539861710e+06 + 1.697899694400910e+06 + 1.764137125734180e+06 + 1.830010489564698e+06 + 1.895519449872339e+06 + 1.960663678913685e+06 + 2.025442857222013e+06 + 2.089856673607306e+06 + 2.153904825156247e+06 + 2.217587017232217e+06 + 2.280902963475303e+06 + 2.343852385802290e+06 + 2.406435014406665e+06 + 2.468650587758616e+06 + 2.530498852605031e+06 + 2.591979563969503e+06 + 2.653092485152322e+06 + 2.713837387730482e+06 + 2.774214051557675e+06 + 2.834222264764299e+06 + 2.893861823757447e+06 + 2.953132533220919e+06 + 3.012034206115212e+06 + 3.070566663677527e+06 + 3.128729735421764e+06 + 3.186523259138526e+06 + 3.243947080895115e+06 + 3.301001055035538e+06 + 3.357685044180497e+06 + 3.413998919227402e+06 + 3.469942559350358e+06 + 3.525515852000176e+06 + 3.580718692904367e+06 + 3.635550986067140e+06 + 3.690012643769410e+06 + 3.744103586568788e+06 + 3.797823743299592e+06 + 3.851173051072835e+06 + 3.904151455276238e+06 + 3.956758909574216e+06 + 4.008995375907890e+06 + 4.060860824495079e+06 + 4.112355233830309e+06 + 4.163478590684799e+06 + 4.214230890106475e+06 + 4.264612135419963e+06 + 4.314622338226588e+06 + 4.364261518404378e+06 + 4.413529704108060e+06 + 4.462426931769069e+06 + 4.510953246095533e+06 + 4.559108700072286e+06 + 4.606893354960858e+06 + 4.654307280299488e+06 + 4.701350553903108e+06 + 4.748023261863358e+06 + 4.794325498548575e+06 + 4.840257366603798e+06 + 4.885818976950768e+06 + 4.931010448787929e+06 + 4.975831909590418e+06 + 5.020283495110085e+06 + 5.064365349375471e+06 + 5.108077624691823e+06 + 5.151420481641092e+06 + 5.194394089081923e+06 + 5.236998624149668e+06 + 5.279234272256376e+06 + 5.321101227090800e+06 + 5.362599690618395e+06 + 5.403729873081312e+06 + 5.444491992998408e+06 + 5.484886277165242e+06 + 5.524912960654069e+06 + 5.564572286813851e+06 + 5.603864507270245e+06 + 5.642789881925615e+06 + 5.681348678959022e+06 + 5.719541174826232e+06 + 5.757367654259708e+06 + 5.794828410268617e+06 + 5.831923744138825e+06 + 5.868653965432902e+06 + 5.905019391990117e+06 + 5.941020349926441e+06 + 5.976657173634544e+06 + 6.011930205783804e+06 + 6.046839797320290e+06 + 6.081386307466779e+06 + 6.115570103722748e+06 + 6.149391561864376e+06 + 6.182851065944540e+06 + 6.215949008292822e+06 + 6.248685789515501e+06 + 6.281061818495561e+06 + 6.313077512392686e+06 + 6.344733296643257e+06 + 6.376029604960365e+06 + 6.406966879333793e+06 + 6.437545570030033e+06 + 6.467766135592271e+06 + 6.497629042840399e+06 + 6.527134766871008e+06 + 6.556283791057392e+06 + 6.585076607049545e+06 + 6.613513714774162e+06 + 6.641595622434638e+06 + 6.669322846511072e+06 + 6.696695911760264e+06 + 6.723715351215711e+06 + 6.750381706187615e+06 + 6.776695526262878e+06 + 6.802657369305105e+06 + 6.828267801454599e+06 + 6.853527397128366e+06 + 6.878436739020114e+06 + 6.902996418100249e+06 + 6.927207033615881e+06 + 6.951069193090819e+06 + 6.974583512325577e+06 + 6.997750615397365e+06 + 7.020571134660101e+06 + 7.043045710744397e+06 + 7.065174992557568e+06 + 7.086959637283633e+06 + 7.108400310383311e+06 + 7.129497685594021e+06 + 7.150252444929884e+06 + 7.170665278681723e+06 + 7.190736885417059e+06 + 7.210467971980117e+06 + 7.229859253491821e+06 + 7.248911453349800e+06 + 7.267625303228384e+06 + 7.286001543078595e+06 + 7.304040921128170e+06 + 7.321744193881537e+06 + 7.339112126119829e+06 + 7.356145490900878e+06 + 7.372845069559221e+06 + 7.389211651706094e+06 + 7.405246035229432e+06 + 7.420949026293878e+06 + 7.436321439340766e+06 + 7.451364097088138e+06 + 7.466077830530737e+06 + 7.480463478940005e+06 + 7.494521889864086e+06 + 7.508253919127826e+06 + 7.521660430832772e+06 + 7.534742297357169e+06 + 7.547500399355968e+06 + 7.559935625760819e+06 + 7.572048873780073e+06 + 7.583841048898782e+06 + 7.595313064878696e+06 + 7.606465843758276e+06 + 7.617300315852673e+06 + 7.627817419753745e+06 + 7.638018102330050e+06 + 7.647903318726848e+06 + 7.657474032366097e+06 + 7.666731214946462e+06 + 7.675675846443305e+06 + 7.684308915108687e+06 + 7.692631417471377e+06 + 7.700644358336839e+06 + 7.708348750787240e+06 + 7.715745616181449e+06 + 7.722835984155037e+06 + 7.729620892620270e+06 + 7.736101387766127e+06 + 7.742278524058277e+06 + 7.748153364239094e+06 + 7.753726979327655e+06 + 7.759000448619737e+06 + 7.763974859687817e+06 + 7.768651308381077e+06 + 7.773030898825390e+06 + 7.777114743423343e+06 + 7.780903962854218e+06 + 7.784399686073998e+06 + 7.787603050315368e+06 + 7.790515201087713e+06 + 7.793137292177119e+06 + 7.795470485646380e+06 + 7.797515951834979e+06 + 7.799274869359110e+06 + 7.800748425111664e+06 + 7.801937814262233e+06 + 7.802844240257111e+06 + 7.803468914819297e+06 + 7.803813057948484e+06 + 7.803877897921070e+06 + 7.803664671290156e+06 + 7.803174622885538e+06 + 7.802409005813722e+06 + 7.801369081457905e+06 + 7.800056119477995e+06 + 7.798471397810592e+06 + 7.796616202669007e+06 + 7.794491828543244e+06 + 7.792099578200010e+06 + 7.789440762682715e+06 + 7.786516701311473e+06 + 7.783328721683091e+06 + 7.779878159671083e+06 + 7.776166359425665e+06 + 7.772194673373749e+06 + 7.767964462218953e+06 + 7.763477094941595e+06 + 7.758733948798692e+06 + 7.753736409323964e+06 + 7.748485870327833e+06 + 7.742983733897421e+06 + 7.737231410396548e+06 + 7.731230318465743e+06 + 7.724981885022229e+06 + 7.718487545259933e+06 + 7.711748742649483e+06 + 7.704766928938209e+06 + 7.697543564150140e+06 + 7.690080116586007e+06 + 7.682378062823243e+06 + 7.674438887715982e+06 + 7.666264084395060e+06 + 7.657855154268012e+06 + 7.649213607019073e+06 + 7.640340960609185e+06 + 7.631238741275986e+06 + 7.621908483533815e+06 + 7.612351730173716e+06 + 7.602570032263434e+06 + 7.592564949147409e+06 + 7.582338048446788e+06 + 7.571890906059416e+06 + 7.561225106159844e+06 + 7.550342241199316e+06 + 7.539243911905785e+06 + 7.527931727283904e+06 + 7.516407304615024e+06 + 7.504672269457198e+06 + 7.492728241314434e+06 + 7.480576638153253e+06 + 7.468218723741564e+06 + 7.455655760425863e+06 + 7.442889013018662e+06 + 7.429919748798485e+06 + 7.416749237509867e+06 + 7.403378751363355e+06 + 7.389809565035504e+06 + 7.376042955668893e+06 + 7.362080202872100e+06 + 7.347922588719724e+06 + 7.333571397752370e+06 + 7.319027916976659e+06 + 7.304293435865226e+06 + 7.289369246356713e+06 + 7.274256642855777e+06 + 7.258956922233087e+06 + 7.243471383825325e+06 + 7.227801329435183e+06 + 7.211948063331366e+06 + 7.195912892248592e+06 + 7.179697125387593e+06 + 7.163302074415106e+06 + 7.146729053463890e+06 + 7.129979379132709e+06 + 7.113054370486340e+06 + 7.095955349055575e+06 + 7.078683638837219e+06 + 7.061240566294081e+06 + 7.043627460354994e+06 + 7.025845652414794e+06 + 7.007896476334333e+06 + 6.989781268440476e+06 + 6.971501367526096e+06 + 6.953058114850082e+06 + 6.934452854137335e+06 + 6.915686931578767e+06 + 6.896761695831302e+06 + 6.877678498017876e+06 + 6.858438691727439e+06 + 6.839043633014950e+06 + 6.819494680401385e+06 + 6.799793194873727e+06 + 6.779940539884974e+06 + 6.759938081354136e+06 + 6.739787187666233e+06 + 6.719489229672301e+06 + 6.699045580689385e+06 + 6.678457616500544e+06 + 6.657726715354848e+06 + 6.636854257967380e+06 + 6.615841627519235e+06 + 6.594690209657516e+06 + 6.573401392495350e+06 + 6.551976566611862e+06 + 6.530417125052197e+06 + 6.508724463327511e+06 + 6.486899979414972e+06 + 6.464945073757762e+06 + 6.442861149265066e+06 + 6.420649611312096e+06 + 6.398311867740063e+06 + 6.375849328856199e+06 + 6.353263407433745e+06 + 6.330555518711952e+06 + 6.307727080396090e+06 + 6.284779512657427e+06 + 6.261714238133261e+06 + 6.238532681926889e+06 + 6.215236271607627e+06 + 6.191826437210800e+06 + 6.168304611237747e+06 + 6.144672228655818e+06 + 6.120930726898375e+06 + 6.097081545864792e+06 + 6.073126127920459e+06 + 6.049065917896771e+06 + 6.024902363091142e+06 + 6.000636913266994e+06 + 5.976271020653763e+06 + 5.951806139946896e+06 + 5.927243728307854e+06 + 5.902585245364108e+06 + 5.877832153209144e+06 + 5.852985916402456e+06 + 5.828048001969552e+06 + 5.803019879401957e+06 + 5.777903020657199e+06 + 5.752698900158828e+06 + 5.727408994796395e+06 + 5.702034783925475e+06 + 5.676577749367647e+06 + 5.651039375410506e+06 + 5.625421148807654e+06 + 5.599724558778713e+06 + 5.573951097009313e+06 + 5.548102257651095e+06 + 5.522179537321717e+06 + 5.496184435104840e+06 + 5.470118452550147e+06 + 5.443983093673326e+06 + 5.417779864956085e+06 + 5.391510275346133e+06 + 5.365175836257203e+06 + 5.338778061569033e+06 + 5.312318467627374e+06 + 5.285798573243992e+06 + 5.259219899696662e+06 + 5.232583970729172e+06 + 5.205892312551324e+06 + 5.179146453838928e+06 + 5.152347925733812e+06 + 5.125498261843814e+06 + 5.098598998242779e+06 + 5.071651673470572e+06 + 5.044657828533064e+06 + 5.017619006902143e+06 + 4.990536754515706e+06 + 4.963412619777663e+06 + 4.936248153557934e+06 + 4.909044909192459e+06 + 4.881804442483181e+06 + 4.854528311698059e+06 + 4.827218077571064e+06 + 4.799875303302180e+06 + 4.772501554557402e+06 + 4.745098399468737e+06 + 4.717667408634203e+06 + 4.690210155117837e+06 + 4.662728214449679e+06 + 4.635223164625784e+06 + 4.607696586108224e+06 + 4.580150061825077e+06 + 4.552585177170437e+06 + 4.525003520004408e+06 + 4.497406680653105e+06 + 4.469796251908661e+06 + 4.442173829029216e+06 + 4.414541009738924e+06 + 4.386899394227949e+06 + 4.359250585152469e+06 + 4.331596187634676e+06 + 4.303937809262771e+06 + 4.276277060090966e+06 + 4.248615552639492e+06 + 4.220954901894585e+06 + 4.193296725308497e+06 + 4.165642642799490e+06 + 4.137994276751838e+06 + 4.110353252015833e+06 + 4.082721195907769e+06 + 4.055099738209962e+06 + 4.027490511170732e+06 + 3.999895149504419e+06 + 3.972315290391369e+06 + 3.944752573477943e+06 + 3.917208640876513e+06 + 3.889685137165464e+06 + 3.862183709389194e+06 + 3.834706007058111e+06 + 3.807253682148634e+06 + 3.779828389103201e+06 + 3.752431784830255e+06 + 3.725065528704253e+06 + 3.697731282565667e+06 + 3.670430710720977e+06 + 3.643165479942679e+06 + 3.615937259469279e+06 + 3.588747721005293e+06 + 3.561598538721256e+06 + 3.534491389253709e+06 + 3.507427951705208e+06 + 3.480409907644318e+06 + 3.453438941105620e+06 + 3.426516738589706e+06 + 3.399644989063177e+06 + 3.372825383958653e+06 + 3.346070405300663e+06 + 3.319465764034668e+06 + 3.293075510459727e+06 + 3.266878224028187e+06 + 3.240869303162930e+06 + 3.215052515525467e+06 + 3.189425288821125e+06 + 3.163986116962672e+06 + 3.138734196986282e+06 + 3.113668261011591e+06 + 3.088787111422079e+06 + 3.064089610843659e+06 + 3.039574592322599e+06 + 3.015240898358055e+06 + 2.991087379108502e+06 + 2.967112887579310e+06 + 2.943316282301805e+06 + 2.919696425519270e+06 + 2.896252185185970e+06 + 2.872982434032476e+06 + 2.849886048305633e+06 + 2.826961910405915e+06 + 2.804208907130155e+06 + 2.781625929123051e+06 + 2.759211873644148e+06 + 2.736965641930875e+06 + 2.714886139213983e+06 + 2.692972277509980e+06 + 2.671222972611625e+06 + 2.649637144535781e+06 + 2.628213719738566e+06 + 2.606951628696007e+06 + 2.585849806893141e+06 + 2.564907195391811e+06 + 2.544122739173453e+06 + 2.523495389008342e+06 + 2.503024101022326e+06 + 2.482707834844927e+06 + 2.462545556054412e+06 + 2.442536236120740e+06 + 2.422678849863131e+06 + 2.402972377820649e+06 + 2.383415806334849e+06 + 2.364008125120872e+06 + 2.344748329835404e+06 + 2.325635421794777e+06 + 2.306668405813749e+06 + 2.287846292982958e+06 + 2.269168099757709e+06 + 2.250632846050586e+06 + 2.232239558087812e+06 + 2.213987266957964e+06 + 2.195875007435864e+06 + 2.177901821473463e+06 + 2.160066755143500e+06 + 2.142368857948063e+06 + 2.124807187017495e+06 + 2.107380803567688e+06 + 2.090088772295509e+06 + 2.072930165358991e+06 + 2.055904058912681e+06 + 2.039009532618940e+06 + 2.022245673041180e+06 + 2.005611571515091e+06 + 1.989106323649552e+06 + 1.972729030298793e+06 + 1.956478797095825e+06 + 1.940354734917566e+06 + 1.924355960016194e+06 + 1.908481592593613e+06 + 1.892730757829583e+06 + 1.877102587255698e+06 + 1.861596215902473e+06 + 1.846210783186639e+06 + 1.830945435012332e+06 + 1.815799321323061e+06 + 1.800771597138300e+06 + 1.785861422681660e+06 + 1.771067961851037e+06 + 1.756390384114008e+06 + 1.741827864373811e+06 + 1.727379581118251e+06 + 1.713044718232857e+06 + 1.698822464839031e+06 + 1.684712013805151e+06 + 1.670712563808684e+06 + 1.656823318006428e+06 + 1.643043483055245e+06 + 1.629372271790869e+06 + 1.615808901794204e+06 + 1.602352594044968e+06 + 1.589002574975609e+06 + 1.575758076322615e+06 + 1.562618333634041e+06 + 1.549582587186907e+06 + 1.536650081999105e+06 + 1.523820067124634e+06 + 1.511091797064626e+06 + 1.498464530780005e+06 + 1.485937531189585e+06 + 1.473510066471004e+06 + 1.461181408848471e+06 + 1.448950834625962e+06 + 1.436817625926470e+06 + 1.424781068397768e+06 + 1.412840451407005e+06 + 1.400995070309470e+06 + 1.389244224245752e+06 + 1.377587216393601e+06 + 1.366023355231714e+06 + 1.354551952598923e+06 + 1.343172324427429e+06 + 1.331883792566280e+06 + 1.320685682335073e+06 + 1.309577322956444e+06 + 1.298558048406515e+06 + 1.287627196362487e+06 + 1.276784109496439e+06 + 1.266028134784103e+06 + 1.255358622320226e+06 + 1.244774926866806e+06 + 1.234276408467881e+06 + 1.223862430240984e+06 + 1.213532358913532e+06 + 1.203285566621101e+06 + 1.193121428991948e+06 + 1.183039325468658e+06 + 1.173038640304339e+06 + 1.163118761212139e+06 + 1.153279079987504e+06 + 1.143518992564451e+06 + 1.133837898317260e+06 + 1.124235201394960e+06 + 1.114710310121780e+06 + 1.105262636013577e+06 + 1.095891594215875e+06 + 1.086596604396163e+06 + 1.077377090233802e+06 + 1.068232479242110e+06 + 1.059162202266971e+06 + 1.050165693700918e+06 + 1.041242393088349e+06 + 1.032391743039430e+06 + 1.023613189178869e+06 + 1.014906181731556e+06 + 1.006270174355871e+06 + 9.977046244036794e+05 + 9.892089935677024e+05 + 9.807827465832273e+05 + 9.724253516436865e+05 + 9.641362810046984e+05 + 9.559150102252770e+05 + 9.477610188719215e+05 + 9.396737899139688e+05 + 9.316528091743080e+05 + 9.236975665277143e+05 + 9.158075561735997e+05 + 9.079822751134991e+05 + 9.002212236533747e+05 + 8.925239057014534e+05 + 8.848898281604102e+05 + 8.773185021612709e+05 + 8.698094424648809e+05 + 8.623621663528095e+05 + 8.549761944053203e+05 + 8.476510510446843e+05 + 8.403862637513808e+05 + 8.331813638606276e+05 + 8.260358860837971e+05 + 8.189493675787508e+05 + 8.119213484350282e+05 + 8.049513730504208e+05 + 7.980389891667470e+05 + 7.911837471192866e+05 + 7.843852003599521e+05 + 7.776429053419405e+05 + 7.709564221446526e+05 + 7.643253140173181e+05 + 7.577491471803443e+05 + 7.512274911598503e+05 + 7.447599180231568e+05 + 7.383460026539576e+05 + 7.319853241781880e+05 + 7.256774641850799e+05 + 7.194220067146936e+05 + 7.132185395309449e+05 + 7.070666529520636e+05 + 7.009659400728574e+05 + 6.949159975085063e+05 + 6.889164244086053e+05 + 6.829668228095188e+05 + 6.770667979456050e+05 + 6.712159574726904e+05 + 6.654139120281750e+05 + 6.596602754201142e+05 + 6.539546638052445e+05 + 6.482966962660485e+05 + 6.426859949531512e+05 + 6.371221843431445e+05 + 6.316048918617199e+05 + 6.261337478197705e+05 + 6.207083848228143e+05 + 6.153284385055641e+05 + 6.099935472682050e+05 + 6.047033517354903e+05 + 5.994574954765684e+05 + 5.942556247644721e+05 + 5.890973881315375e+05 + 5.839824370501742e+05 + 5.789104255366600e+05 + 5.738810098259439e+05 + 5.688938490610282e+05 + 5.639486048510919e+05 + 5.590449410545636e+05 + 5.541825244030216e+05 + 5.493610239467514e+05 + 5.445801109611018e+05 + 5.398394595653637e+05 + 5.351387461239359e+05 + 5.304776492529869e+05 + 5.258558503354813e+05 + 5.212730328842620e+05 + 5.167288826663374e+05 + 5.122230881980868e+05 + 5.077553400369428e+05 + 5.033253309731948e+05 + 4.989327564601310e+05 + 4.945773139811803e+05 + 4.902587033143605e+05 + 4.859766266490913e+05 + 4.817307881770549e+05 + 4.775208945372272e+05 + 4.733466546121029e+05 + 4.692077791818720e+05 + 4.651039814988278e+05 + 4.610349770333872e+05 + 4.570004830895544e+05 + 4.530002193970305e+05 + 4.490339078641191e+05 + 4.451012721716975e+05 + 4.412020383482487e+05 + 4.373359346346440e+05 + 4.335026910461138e+05 + 4.297020398291695e+05 + 4.259337153289251e+05 + 4.221974536699117e+05 + 4.184929932504728e+05 + 4.148200744355357e+05 + 4.111784393468680e+05 + 4.075678324287459e+05 + 4.039879999559653e+05 + 4.004386899065928e+05 + 3.969196525836200e+05 + 3.934306400492930e+05 + 3.899714060581240e+05 + 3.865417066557175e+05 + 3.831412995849459e+05 + 3.797699442729113e+05 + 3.764274023566474e+05 + 3.731134371233769e+05 + 3.698278135622827e+05 + 3.665702987827645e+05 + 3.633406615280775e+05 + 3.601386722825769e+05 + 3.569641034535824e+05 + 3.538167290782596e+05 + 3.506963250581825e+05 + 3.476026690576235e+05 + 3.445355402744773e+05 + 3.414947198014101e+05 + 3.384799905288533e+05 + 3.354911367821696e+05 + 3.325279446663284e+05 + 3.295902021531574e+05 + 3.266776986085196e+05 + 3.237902250531212e+05 + 3.209275743946921e+05 + 3.180895409646722e+05 + 3.152759207164211e+05 + 3.124865112943467e+05 + 3.097211117598721e+05 + 3.069795229448014e+05 + 3.042615472339510e+05 + 3.015669883675390e+05 + 2.988956518738279e+05 + 2.962473447339852e+05 + 2.936218752484120e+05 + 2.910190535228683e+05 + 2.884386911235514e+05 + 2.858806009067059e+05 + 2.833445973150213e+05 + 2.808304963224871e+05 + 2.783381153152796e+05 + 2.758672731341289e+05 + 2.734177900017136e+05 + 2.709894875428715e+05 + 2.685821889451269e+05 + 2.661957187314447e+05 + 2.638299027837526e+05 + 2.614845683836235e+05 + 2.591595441417535e+05 + 2.568546601323142e+05 + 2.545697477628685e+05 + 2.523046396706559e+05 + 2.500591699434964e+05 + 2.478331740644226e+05 + 2.456264886409644e+05 + 2.434389516128850e+05 + 2.412704023657298e+05 + 2.391206814330098e+05 + 2.369896306904384e+05 + 2.348770933197599e+05 + 2.327829136173563e+05 + 2.307069372775990e+05 + 2.286490111745211e+05 + 2.266089832623643e+05 + 2.245867029748616e+05 + 2.225820209237310e+05 + 2.205947887183862e+05 + 2.186248592921233e+05 + 2.166720868039439e+05 + 2.147363264605261e+05 + 2.128174347706827e+05 + 2.109152693889452e+05 + 2.090296889843596e+05 + 2.071605535077064e+05 + 2.053077240108720e+05 + 2.034710625599661e+05 + 2.016504325007521e+05 + 1.998456982030229e+05 + 1.980567250370223e+05 + 1.962833797138016e+05 + 1.945255298870997e+05 + 1.927830441323482e+05 + 1.910557922964590e+05 + 1.893436452442803e+05 + 1.876464748512476e+05 + 1.859641540986015e+05 + 1.842965568967865e+05 + 1.826435581738182e+05 + 1.810050340354470e+05 + 1.793808614993985e+05 + 1.777709185575675e+05 + 1.761750842223128e+05 + 1.745932384546507e+05 + 1.730252623268235e+05 + 1.714710377603550e+05 + 1.699304475698536e+05 + 1.684033757560533e+05 + 1.668897071565261e+05 + 1.653893274220018e+05 + 1.639021233234161e+05 + 1.624279825159149e+05 + 1.609667934564489e+05 + 1.595184456811041e+05 + 1.580828296015031e+05 + 1.566598363968120e+05 + 1.552493582649543e+05 + 1.538512882702573e+05 + 1.524655202573447e+05 + 1.510919491064028e+05 + 1.497304705096718e+05 + 1.483809809029638e+05 + 1.470433777326744e+05 + 1.457175592117312e+05 + 1.444034242942686e+05 + 1.431008729933990e+05 + 1.418098060421684e+05 + 1.405301248671864e+05 + 1.392617319094920e+05 + 1.380045303259556e+05 + 1.367584239714278e+05 + 1.355233176631985e+05 + 1.342991169575939e+05 + 1.330857281367400e+05 + 1.318830583220329e+05 + 1.306910153813551e+05 + 1.295095079605118e+05 + 1.283384454277201e+05 + 1.271777378619642e+05 + 1.260272961737778e+05 + 1.248870320778846e+05 + 1.237568578844075e+05 + 1.226366865814357e+05 + 1.215264320654815e+05 + 1.204260088572700e+05 + 1.193353321256488e+05 + 1.182543179024696e+05 + 1.171828828329888e+05 + 1.161209441971572e+05 + 1.150684200368906e+05 + 1.140252290172446e+05 + 1.129912905290012e+05 + 1.119665246989244e+05 + 1.109508522377596e+05 + 1.099441945145749e+05 + 1.089464735474612e+05 + 1.079576119759672e+05 + 1.069775332658437e+05 + 1.060061613940063e+05 + 1.050434208615313e+05 + 1.040892370245828e+05 + 1.031435357647047e+05 + 1.022062434696232e+05 + 1.012772873208916e+05 + 1.003565950684523e+05 + 9.944409498683986e+04 + 9.853971604124359e+04 + 9.764338777276211e+04 + 9.675504027159151e+04 + 9.587460425195689e+04 + 9.500201099602232e+04 + 9.413719236230559e+04 + 9.328008081668093e+04 + 9.243060935253548e+04 + 9.158871152497102e+04 + 9.075432150057999e+04 + 8.992737394814150e+04 + 8.910780408381460e+04 + 8.829554772812854e+04 + 8.749054118346877e+04 + 8.669272128864085e+04 + 8.590202548133289e+04 + 8.511839166524833e+04 + 8.434175826486869e+04 + 8.357206428147563e+04 + 8.280924917713979e+04 + 8.205325293940101e+04 + 8.130401609766987e+04 + 8.056147962909786e+04 + 7.982558503959214e+04 + 7.909627435336930e+04 + 7.837349003289423e+04 + 7.765717506702039e+04 + 7.694727293156023e+04 + 7.624372753229675e+04 + 7.554648331079309e+04 + 7.485548516457589e+04 + 7.417067840840557e+04 + 7.349200889763620e+04 + 7.281942291879337e+04 + 7.215286716185752e+04 + 7.149228885123666e+04 + 7.083763563584826e+04 + 7.018885556144921e+04 + 6.954589717270633e+04 + 6.890870943981725e+04 + 6.827724174191867e+04 + 6.765144392787226e+04 + 6.703126624692857e+04 + 6.641665935636370e+04 + 6.580757439267423e+04 + 6.520396286738058e+04 + 6.460577668030654e+04 + 6.401296820028803e+04 + 6.342549017134596e+04 + 6.284329573351656e+04 + 6.226633845994343e+04 + 6.169457228450359e+04 + 6.112795154311772e+04 + 6.056643099463883e+04 + 6.000996574136701e+04 + 5.945851127624574e+04 + 5.891202351267684e+04 + 5.837045869755570e+04 + 5.783377345558033e+04 + 5.730192482224794e+04 + 5.677487016169035e+04 + 5.625256720967715e+04 + 5.573497408983271e+04 + 5.522204924870504e+04 + 5.471375150828325e+04 + 5.421004006036229e+04 + 5.371087441354901e+04 + 5.321621444975662e+04 + 5.272602039469258e+04 + 5.224025278120609e+04 + 5.175887251825204e+04 + 5.128184085536604e+04 + 5.080911934597862e+04 + 5.034066989608183e+04 + 4.987645473684455e+04 + 4.941643640415772e+04 + 4.896057778639881e+04 + 4.850884207769758e+04 + 4.806119277190277e+04 + 4.761759372195903e+04 + 4.717800906318111e+04 + 4.674240321587012e+04 + 4.631074096205200e+04 + 4.588298735942246e+04 + 4.545910774537743e+04 + 4.503906779825089e+04 + 4.462283347373459e+04 + 4.421037101686000e+04 + 4.380164697651120e+04 + 4.339662816996042e+04 + 4.299528171729217e+04 + 4.259757504220949e+04 + 4.220347581706527e+04 + 4.181295199605238e+04 + 4.142597183433527e+04 + 4.104250383855765e+04 + 4.066251680332360e+04 + 4.028597979815155e+04 + 3.991286213370509e+04 + 3.954313341370015e+04 + 3.917676350485435e+04 + 3.881372250661203e+04 + 3.845398080843414e+04 + 3.809750905736740e+04 + 3.774427812448402e+04 + 3.739425915595785e+04 + 3.704742356207799e+04 + 3.670374297805164e+04 + 3.636318929176292e+04 + 3.602573463927982e+04 + 3.569135138416072e+04 + 3.536001215672983e+04 + 3.503168981923036e+04 + 3.470635744840465e+04 + 3.438398837321134e+04 + 3.406455615393208e+04 + 3.374803457309094e+04 + 3.343439765983423e+04 + 3.312361965186892e+04 + 3.281567499925836e+04 + 3.251053841177231e+04 + 3.220818480242078e+04 + 3.190858928930939e+04 + 3.161172723179306e+04 + 3.131757418548949e+04 + 3.102610591483113e+04 + 3.073729842710719e+04 + 3.045112791454117e+04 + 3.016757076691534e+04 + 2.988660361584138e+04 + 2.960820327321778e+04 + 2.933234674204399e+04 + 2.905901125674980e+04 + 2.878817423326317e+04 + 2.851981328372868e+04 + 2.825390622682930e+04 + 2.799043106146120e+04 + 2.772936599846356e+04 + 2.747068942978120e+04 + 2.721437991599866e+04 + 2.696041623625263e+04 + 2.670877735469368e+04 + 2.645944239579323e+04 + 2.621239068106394e+04 + 2.596760172331024e+04 + 2.572505519859007e+04 + 2.548473096648493e+04 + 2.524660907253227e+04 + 2.501066972455740e+04 + 2.477689330466365e+04 + 2.454526037995217e+04 + 2.431575168300119e+04 + 2.408834810912627e+04 + 2.386303072521571e+04 + 2.363978076395976e+04 + 2.341857962988761e+04 + 2.319940888531031e+04 + 2.298225025100848e+04 + 2.276708562349589e+04 + 2.255389704384009e+04 + 2.234266670372097e+04 + 2.213337697918792e+04 + 2.192601038403303e+04 + 2.172054957464747e+04 + 2.151697738916041e+04 + 2.131527680276242e+04 + 2.111543093252480e+04 + 2.091742306098243e+04 + 2.072123660422193e+04 + 2.052685512482519e+04 + 2.033426234716574e+04 + 2.014344212098682e+04 + 1.995437843426629e+04 + 1.976705544479930e+04 + 1.958145743480243e+04 + 1.939756881213320e+04 + 1.921537413684756e+04 + 1.903485810052154e+04 + 1.885600554387774e+04 + 1.867880143037429e+04 + 1.850323084111479e+04 + 1.832927901452323e+04 + 1.815693131178678e+04 + 1.798617320994377e+04 + 1.781699034075772e+04 + 1.764936845332217e+04 + 1.748329340510924e+04 + 1.731875119617678e+04 + 1.715572795110660e+04 + 1.699420990919751e+04 + 1.683418344358223e+04 + 1.667563503873475e+04 + 1.651855128954979e+04 + 1.636291893365341e+04 + 1.620872481606836e+04 + 1.605595588699932e+04 + 1.590459923143625e+04 + 1.575464203906202e+04 + 1.560607160533683e+04 + 1.545887535681969e+04 + 1.531304082443131e+04 + 1.516855564487381e+04 + 1.502540756956110e+04 + 1.488358445432479e+04 + 1.474307426938193e+04 + 1.460386509288249e+04 + 1.446594509814283e+04 + 1.432930256776952e+04 + 1.419392589719183e+04 + 1.405980357729324e+04 + 1.392692420658459e+04 + 1.379527648729671e+04 + 1.366484921045415e+04 + 1.353563127160237e+04 + 1.340761167272577e+04 + 1.328077950662285e+04 + 1.315512397004589e+04 + 1.303063434970357e+04 + 1.290730001970431e+04 + 1.278511047006237e+04 + 1.266405528001385e+04 + 1.254412410683634e+04 + 1.242530670538782e+04 + 1.230759293022181e+04 + 1.219097272173875e+04 + 1.207543611008671e+04 + 1.196097321598039e+04 + 1.184757424417080e+04 + 1.173522949287449e+04 + 1.162392934478714e+04 + 1.151366426384512e+04 + 1.140442480834694e+04 + 1.129620161634352e+04 + 1.118898540470730e+04 + 1.108276698378667e+04 + 1.097753724013271e+04 + 1.087328713748510e+04 + 1.077000773143687e+04 + 1.066769015148828e+04 + 1.056632560317495e+04 + 1.046590537899618e+04 + 1.036642084361778e+04 + 1.026786343864257e+04 + 1.017022468912003e+04 + 1.007349618928313e+04 + 9.977669609110566e+03 + 9.882736699245763e+03 + 9.788689277747368e+03 + 9.695519238085319e+03 + 9.603218551128393e+03 + 9.511779252802526e+03 + 9.421193453258842e+03 + 9.331453338088948e+03 + 9.242551155902798e+03 + 9.154479227014926e+03 + 9.067229945268125e+03 + 8.980795767109534e+03 + 8.895169220035050e+03 + 8.810342900070031e+03 + 8.726309463870590e+03 + 8.643061639313548e+03 + 8.560592220958239e+03 + 8.478894062727115e+03 + 8.397960086705119e+03 + 8.317783279679375e+03 + 8.238356687487571e+03 + 8.159673421912185e+03 + 8.081726656780256e+03 + 8.004509624589081e+03 + 7.928015622553179e+03 + 7.852238006535134e+03 + 7.777170190090504e+03 + 7.702805651284038e+03 + 7.629137923269694e+03 + 7.556160595291118e+03 + 7.483867321684783e+03 + 7.412251809739373e+03 + 7.341307820993124e+03 + 7.271029178292266e+03 + 7.201409756944273e+03 + 7.132443488123264e+03 + 7.064124361053455e+03 + 6.996446414793790e+03 + 6.929403743407001e+03 + 6.862990497342477e+03 + 6.797200875139910e+03 + 6.732029129721631e+03 + 6.667469570161823e+03 + 6.603516551523545e+03 + 6.540164480186692e+03 + 6.477407817185619e+03 + 6.415241069687017e+03 + 6.353658796883785e+03 + 6.292655608307650e+03 + 6.232226157518487e+03 + 6.172365149982773e+03 + 6.113067340658933e+03 + 6.054327527853383e+03 + 5.996140560115221e+03 + 5.938501333076975e+03 + 5.881404784939355e+03 + 5.824845903828315e+03 + 5.768819722461626e+03 + 5.713321315085574e+03 + 5.658345805472132e+03 + 5.603888359764917e+03 + 5.549944184846402e+03 + 5.496508537213697e+03 + 5.443576713947663e+03 + 5.391144051368102e+03 + 5.339205932358427e+03 + 5.287757780134069e+03 + 5.236795058291703e+03 + 5.186313275265001e+03 + 5.136307977685731e+03 + 5.086774751617681e+03 + 5.037709226449485e+03 + 4.989107068721900e+03 + 4.940963984504954e+03 + 4.893275721702213e+03 + 4.846038063285039e+03 + 4.799246830729600e+03 + 4.752897888312231e+03 + 4.706987133608333e+03 + 4.661510499922751e+03 + 4.616463963202395e+03 + 4.571843532909213e+03 + 4.527645253893021e+03 + 4.483865209302220e+03 + 4.440499515324517e+03 + 4.397544325863069e+03 + 4.354995830474305e+03 + 4.312850249817291e+03 + 4.271103841242844e+03 + 4.229752898292512e+03 + 4.188793745275323e+03 + 4.148222740634546e+03 + 4.108036277499050e+03 + 4.068230779797937e+03 + 4.028802705831530e+03 + 3.989748546349355e+03 + 3.951064821861195e+03 + 3.912748086813885e+03 + 3.874794926585441e+03 + 3.837201956255947e+03 + 3.799965825835211e+03 + 3.763083213042953e+03 + 3.726550823283946e+03 + 3.690365398229224e+03 + 3.654523706502605e+03 + 3.619022543185401e+03 + 3.583858737044133e+03 + 3.549029144619817e+03 + 3.514530650270483e+03 + 3.480360168271582e+03 + 3.446514639490202e+03 + 3.412991033253400e+03 + 3.379786348393921e+03 + 3.346897609451590e+03 + 3.314321868815865e+03 + 3.282056205761040e+03 + 3.250097724763853e+03 + 3.218443559530467e+03 + 3.187090870156345e+03 + 3.156036839918450e+03 + 3.125278679259401e+03 + 3.094813626126841e+03 + 3.064638941053712e+03 + 3.034751909739014e+03 + 3.005149845581890e+03 + 2.975830084543172e+03 + 2.946789986568892e+03 + 2.918026936797055e+03 + 2.889538343339952e+03 + 2.861321641252218e+03 + 2.833374286700636e+03 + 2.805693756659566e+03 + 2.778277556469224e+03 + 2.751123212016022e+03 + 2.724228269242043e+03 + 2.697590301914011e+03 + 2.671206904109025e+03 + 2.645075688971241e+03 + 2.619194294405700e+03 + 2.593560380141823e+03 + 2.568171626715687e+03 + 2.543025737542386e+03 + 2.518120435211838e+03 + 2.493453462030127e+03 + 2.469022585172818e+03 + 2.444825590018873e+03 + 2.420860280374604e+03 + 2.397124484732366e+03 + 2.373616049394018e+03 + 2.350332838436762e+03 + 2.327272738609171e+03 + 2.304433655218562e+03 + 2.281813512846715e+03 + 2.259410255601743e+03 + 2.237221844685107e+03 + 2.215246260811306e+03 + 2.193481506123082e+03 + 2.171925598942706e+03 + 2.150576574458654e+03 + 2.129432488764117e+03 + 2.108491414442484e+03 + 2.087751441300839e+03 + 2.067210678851041e+03 + 2.046867252353801e+03 + 2.026719303565292e+03 + 2.006764993817073e+03 + 1.987002500493615e+03 + 1.967430017215378e+03 + 1.948045754608514e+03 + 1.928847939062915e+03 + 1.909834815497017e+03 + 1.891004644375446e+03 + 1.872355700452609e+03 + 1.853886275802303e+03 + 1.835594678424827e+03 + 1.817479231062563e+03 + 1.799538272810955e+03 + 1.781770158220284e+03 + 1.764173256646331e+03 + 1.746745953278825e+03 + 1.729486647489427e+03 + 1.712393752933919e+03 + 1.695465699568615e+03 + 1.678700930817119e+03 + 1.662097903850073e+03 + 1.645655091966725e+03 + 1.629370981458450e+03 + 1.613244072202941e+03 + 1.597272880432907e+03 + 1.581455933682248e+03 + 1.565791771294578e+03 + 1.550278951289465e+03 + 1.534916043199639e+03 + 1.519701626827566e+03 + 1.504634298568118e+03 + 1.489712666891078e+03 + 1.474935351996483e+03 + 1.460300988758884e+03 + 1.445808223853118e+03 + 1.431455716350942e+03 + 1.417242138357507e+03 + 1.403166173386480e+03 + 1.389226518010158e+03 + 1.375421881007569e+03 + 1.361750982089843e+03 + 1.348212553926219e+03 + 1.334805341072190e+03 + 1.321528098720046e+03 + 1.308379594477261e+03 + 1.295358607457036e+03 + 1.282463927300085e+03 + 1.269694355823267e+03 + 1.257048705895313e+03 + 1.244525800699936e+03 + 1.232124475203576e+03 + 1.219843575042076e+03 + 1.207681956049559e+03 + 1.195638485404256e+03 + 1.183712040621380e+03 + 1.171901509388141e+03 + 1.160205790292893e+03 + 1.148623791762007e+03 + 1.137154432301071e+03 + 1.125796641257775e+03 + 1.114549357263016e+03 + 1.103411528660030e+03 + 1.092382114454342e+03 + 1.081460082700689e+03 + 1.070644411095300e+03 + 1.059934087579793e+03 + 1.049328108720195e+03 + 1.038825480494848e+03 + 1.028425218971371e+03 + 1.018126348582361e+03 + 1.007927902927668e+03 + 9.978289254104482e+02 + 9.878284675679866e+02 + 9.779255898738762e+02 + 9.681193621623704e+02 + 9.584088623012988e+02 + 9.487931771658360e+02 + 9.392714022763922e+02 + 9.298426409505287e+02 + 9.205060057185552e+02 + 9.112606172437117e+02 + 9.021056036681564e+02 + 8.930401022331439e+02 + 8.840632580637879e+02 + 8.751742235821227e+02 + 8.663721600224155e+02 + 8.576562363107178e+02 + 8.490256285516970e+02 + 8.404795212594149e+02 + 8.320171064683727e+02 + 8.236375833541338e+02 + 8.153401590664150e+02 + 8.071240480244585e+02 + 7.989884718119321e+02 + 7.909326598382663e+02 + 7.829558483527250e+02 + 7.750572805483002e+02 + 7.672362073200991e+02 + 7.594918862313154e+02 + 7.518235817392584e+02 + 7.442305655888715e+02 + 7.367121158939042e+02 + 7.292675176279554e+02 + 7.218960630476328e+02 + 7.145970504767349e+02 + 7.073697847820883e+02 + 7.002135778527235e+02 + 6.931277475695276e+02 + 6.861116184218383e+02 + 6.791645215373005e+02 + 6.722857937677298e+02 + 6.654747784472139e+02 + 6.587308253877042e+02 + 6.520532900189463e+02 + 6.454415341582659e+02 + 6.388949257382737e+02 + 6.324128381494658e+02 + 6.259946511571761e+02 + 6.196397502963263e+02 + 6.133475264197438e+02 + 6.071173768094372e+02 + 6.009487042352257e+02 + 5.948409165626937e+02 + 5.887934278801442e+02 + 5.828056577149104e+02 + 5.768770306335107e+02 + 5.710069770500405e+02 + 5.651949326339243e+02 + 5.594403381224611e+02 + 5.537426401234757e+02 + 5.481012901152631e+02 + 5.425157443662293e+02 + 5.369854648776029e+02 + 5.315099185870243e+02 + 5.260885773011631e+02 + 5.207209179649268e+02 + 5.154064223119497e+02 + 5.101445770921985e+02 + 5.049348740937783e+02 + 4.997768095346806e+02 + 4.946698844046663e+02 + 4.896136047862775e+02 + 4.846074811232588e+02 + 4.796510285618583e+02 + 4.747437670096575e+02 + 4.698852205388910e+02 + 4.650749179071526e+02 + 4.603123926691712e+02 + 4.555971823563725e+02 + 4.509288288727789e+02 + 4.463068788207030e+02 + 4.417308827523721e+02 + 4.372003955530256e+02 + 4.327149765501697e+02 + 4.282741889383891e+02 + 4.238776002586995e+02 + 4.195247821802792e+02 + 4.152153101301976e+02 + 4.109487639564401e+02 + 4.067247273397822e+02 + 4.025427875685178e+02 + 3.984025363858318e+02 + 3.943035692477232e+02 + 3.902454851077117e+02 + 3.862278871965347e+02 + 3.822503823673904e+02 + 3.783125809462488e+02 + 3.744140974100913e+02 + 3.705545496319825e+02 + 3.667335588488600e+02 + 3.629507504654170e+02 + 3.592057531920941e+02 + 3.554981990087823e+02 + 3.518277237997781e+02 + 3.481939667588409e+02 + 3.445965704231110e+02 + 3.410351808614133e+02 + 3.375094474036331e+02 + 3.340190228447264e+02 + 3.305635632260179e+02 + 3.271427276872922e+02 + 3.237561788604219e+02 + 3.204035826343936e+02 + 3.170846077594078e+02 + 3.137989262426811e+02 + 3.105462135925898e+02 + 3.073261480789523e+02 + 3.041384108878384e+02 + 3.009826866110996e+02 + 2.978586626197819e+02 + 2.947660292278245e+02 + 2.917044799774912e+02 + 2.886737110898206e+02 + 2.856734216347172e+02 + 2.827033138205502e+02 + 2.797630925185924e+02 + 2.768524653868921e+02 + 2.739711430162750e+02 + 2.711188386172004e+02 + 2.682952682417776e+02 + 2.655001507075566e+02 + 2.627332073441469e+02 + 2.599941621974095e+02 + 2.572827420191948e+02 + 2.545986760992428e+02 + 2.519416963867954e+02 + 2.493115374004747e+02 + 2.467079361520411e+02 + 2.441306322519956e+02 + 2.415793676749632e+02 + 2.390538868162835e+02 + 2.365539368283623e+02 + 2.340792670618049e+02 + 2.316296291092408e+02 + 2.292047772816256e+02 + 2.268044680583002e+02 + 2.244284601550858e+02 + 2.220765149177526e+02 + 2.197483957886078e+02 + 2.174438683808943e+02 + 2.151627007150871e+02 + 2.129046628991759e+02 + 2.106695273325222e+02 + 2.084570687245048e+02 + 2.062670637405841e+02 + 2.040992912166423e+02 + 2.019535322675416e+02 + 1.998295699356443e+02 + 1.977271893831886e+02 + 1.956461779038128e+02 + 1.935863247355415e+02 + 1.915474213798307e+02 + 1.895292611678483e+02 + 1.875316391958052e+02 + 1.855543528482985e+02 + 1.835972014670904e+02 + 1.816599861324650e+02 + 1.797425099308505e+02 + 1.778445779263947e+02 + 1.759659969484732e+02 + 1.741065756739388e+02 + 1.722661246609050e+02 + 1.704444562587039e+02 + 1.686413847511611e+02 + 1.668567261386198e+02 + 1.650902980928235e+02 + 1.633419201885993e+02 + 1.616114136777071e+02 + 1.598986014805750e+02 + 1.582033083619683e+02 + 1.565253606828442e+02 + 1.548645864376032e+02 + 1.532208154238845e+02 + 1.515938789750561e+02 + 1.499836100268175e+02 + 1.483898432531407e+02 + 1.468124148175615e+02 + 1.452511624739094e+02 + 1.437059256375598e+02 + 1.421765451600804e+02 + 1.406628634671130e+02 + 1.391647245894945e+02 + 1.376819739463268e+02 + 1.362144585029153e+02 + 1.347620267668397e+02 + 1.333245285902542e+02 + 1.319018153476916e+02 + 1.304937399101563e+02 + 1.291001564496927e+02 + 1.277209206089626e+02 + 1.263558894821456e+02 + 1.250049214462411e+02 + 1.236678763269726e+02 + 1.223446153162142e+02 + 1.210350008534558e+02 + 1.197388968258891e+02 + 1.184561684202564e+02 + 1.171866820332312e+02 + 1.159303054814129e+02 + 1.146869078350233e+02 + 1.134563593521917e+02 + 1.122385316718855e+02 + 1.110332976232454e+02 + 1.098405311979204e+02 + 1.086601077608261e+02 + 1.074919038117468e+02 + 1.063357969778381e+02 + 1.051916662270612e+02 + 1.040593916259906e+02 + 1.029388543466298e+02 + 1.018299368341955e+02 + 1.007325226151947e+02 + 9.964649633160360e+01 + 9.857174381861689e+01 + 9.750815195741964e+01 + 9.645560874783912e+01 + 9.541400332430685e+01 + 9.438322582683587e+01 + 9.336316750654088e+01 + 9.235372071959520e+01 + 9.135477878367931e+01 + 9.036623609174318e+01 + 8.938798812110527e+01 + 8.841993128541375e+01 + 8.746196304638978e+01 + 8.651398190997742e+01 + 8.557588728936869e+01 + 8.464757961904559e+01 + 8.372896034435935e+01 + 8.281993180539014e+01 + 8.192039735296085e+01 + 8.103026128111058e+01 + 8.014942874495698e+01 + 7.927780590401203e+01 + 7.841529983858037e+01 + 7.756181847617736e+01 + 7.671727071446733e+01 + 7.588156631674119e+01 + 7.505461587372024e+01 + 7.423633095310755e+01 + 7.342662394299153e+01 + 7.262540803009301e+01 + 7.183259736215271e+01 + 7.104810687905130e+01 + 7.027185230277652e+01 + 6.950375028269488e+01 + 6.874371822566854e+01 + 6.799167430460268e+01 + 6.724753760248480e+01 + 6.651122793902957e+01 + 6.578266588744034e+01 + 6.506177289160765e+01 + 6.434847109859102e+01 + 6.364268339115934e+01 + 6.294433351272004e+01 + 6.225334589309644e+01 + 6.156964567793161e+01 + 6.089315882540817e+01 + 6.022381196253272e+01 + 5.956153243192431e+01 + 5.890624835784802e+01 + 5.825788851805937e+01 + 5.761638239540161e+01 + 5.698166019812880e+01 + 5.635365277455941e+01 + 5.573229169643452e+01 + 5.511750921575248e+01 + 5.450923819422894e+01 + 5.390741220139490e+01 + 5.331196548214332e+01 + 5.272283288224639e+01 + 5.213994992103501e+01 + 5.156325275682114e+01 + 5.099267813956413e+01 + 5.042816349744366e+01 + 4.986964685712531e+01 + 4.931706681520628e+01 + 4.877036264381848e+01 + 4.822947419115650e+01 + 4.769434186145065e+01 + 4.716490671386568e+01 + 4.664111036072763e+01 + 4.612289496053356e+01 + 4.561020331071316e+01 + 4.510297873475514e+01 + 4.460116508782800e+01 + 4.410470685618211e+01 + 4.361354903662819e+01 + 4.312763714409483e+01 + 4.264691728541290e+01 + 4.217133607138089e+01 + 4.170084064118655e+01 + 4.123537869187841e+01 + 4.077489840121474e+01 + 4.031934846724150e+01 + 3.986867813088396e+01 + 3.942283710103097e+01 + 3.898177559914080e+01 + 3.854544435810492e+01 + 3.811379455962712e+01 + 3.768677789368684e+01 + 3.726434656726136e+01 + 3.684645322013298e+01 + 3.643305096011807e+01 + 3.602409338288954e+01 + 3.561953452360934e+01 + 3.521932891649128e+01 + 3.482343153306697e+01 + 3.443179775085747e+01 + 3.404438343901749e+01 + 3.366114490993698e+01 + 3.328203887889210e+01 + 3.290702252020257e+01 + 3.253605343510379e+01 + 3.216908962260722e+01 + 3.180608952699355e+01 + 3.144701200592500e+01 + 3.109181631097627e+01 + 3.074046212448961e+01 + 3.039290952642646e+01 + 3.004911898312090e+01 + 2.970905136572861e+01 + 2.937266792505025e+01 + 2.903993030352619e+01 + 2.871080057194543e+01 + 2.838524113862799e+01 + 2.806321475945346e+01 + 2.774468462701056e+01 + 2.742961427426368e+01 + 2.711796757973106e+01 + 2.680970882898042e+01 + 2.650480264209962e+01 + 2.620321398820359e+01 + 2.590490820737408e+01 + 2.560985096325826e+01 + 2.531800827674159e+01 + 2.502934653543958e+01 + 2.474383242809609e+01 + 2.446143296972707e+01 + 2.418211555345684e+01 + 2.390584787621480e+01 + 2.363259794654911e+01 + 2.336233413255323e+01 + 2.309502509558490e+01 + 2.283063979379989e+01 + 2.256914753657127e+01 + 2.231051792982671e+01 + 2.205472087402378e+01 + 2.180172658443576e+01 + 2.155150556919213e+01 + 2.130402866286656e+01 + 2.105926696778129e+01 + 2.081719185319289e+01 + 2.057777503962135e+01 + 2.034098850331043e+01 + 2.010680447006149e+01 + 1.987519550063876e+01 + 1.964613442528577e+01 + 1.941959432580914e+01 + 1.919554855827313e+01 + 1.897397075515799e+01 + 1.875483482712853e+01 + 1.853811494931854e+01 + 1.832378554313712e+01 + 1.811182128985819e+01 + 1.790219714756028e+01 + 1.769488831219516e+01 + 1.748987022919803e+01 + 1.728711861052799e+01 + 1.708660939866258e+01 + 1.688831878332332e+01 + 1.669222321036695e+01 + 1.649829934690638e+01 + 1.630652410217020e+01 + 1.611687463464447e+01 + 1.592932831698975e+01 + 1.574386275792588e+01 + 1.556045580679505e+01 + 1.537908552081595e+01 + 1.519973018767385e+01 + 1.502236832516821e+01 + 1.484697865494502e+01 + 1.467354012841447e+01 + 1.450203191274907e+01 + 1.433243337158686e+01 + 1.416472409674524e+01 + 1.399888388846149e+01 + 1.383489273805109e+01 + 1.367273085728561e+01 + 1.351237865788487e+01 + 1.335381673920349e+01 + 1.319702591751814e+01 + 1.304198719993403e+01 + 1.288868177614133e+01 + 1.273709104714046e+01 + 1.258719659672914e+01 + 1.243898018736778e+01 + 1.229242378719869e+01 + 1.214750954184370e+01 + 1.200421977308728e+01 + 1.186253699764556e+01 + 1.172244390266849e+01 + 1.158392335040491e+01 + 1.144695839482221e+01 + 1.131153225250660e+01 + 1.117762830926809e+01 + 1.104523013501555e+01 + 1.091432145806843e+01 + 1.078488617566414e+01 + 1.065690836231677e+01 + 1.053037224372639e+01 + 1.040526220941685e+01 + 1.028156282144551e+01 + 1.015925878941852e+01 + 1.003833498402321e+01 + 9.918776439618298e+00 + 9.800568334039683e+00 + 9.683696005431957e+00 + 9.568144946174611e+00 + 9.453900787919052e+00 + 9.340949322860192e+00 + 9.229276489267212e+00 + 9.118868359446189e+00 + 9.009711163642070e+00 + 8.901791273796370e+00 + 8.795095191785947e+00 + 8.689609570648638e+00 + 8.585321201853549e+00 + 8.482217005296002e+00 + 8.380284046581689e+00 + 8.279509523576925e+00 + 8.179880760330461e+00 + 8.081385224417300e+00 + 7.984010508389771e+00 + 7.887744326863491e+00 + 7.792574535206823e+00 + 7.698489109519010e+00 + 7.605476146133772e+00 + 7.513523878187234e+00 + 7.422620653280592e+00 + 7.332754935473541e+00 + 7.243915322674125e+00 + 7.156090524556396e+00 + 7.069269365741123e+00 + 6.983440797017198e+00 + 6.898593877074525e+00 + 6.814717779007339e+00 + 6.731801797516630e+00 + 6.649835331100769e+00 + 6.568807890289103e+00 + 6.488709103685009e+00 + 6.409528699865692e+00 + 6.331256515961359e+00 + 6.253882503643640e+00 + 6.177396712290002e+00 + 6.101789297547966e+00 + 6.027050522250886e+00 + 5.953170745278158e+00 + 5.880140434362498e+00 + 5.807950156612827e+00 + 5.736590570385212e+00 + 5.666052441485230e+00 + 5.596326632559173e+00 + 5.527404095182261e+00 + 5.459275884510260e+00 + 5.391933149962507e+00 + 5.325367127621260e+00 + 5.259569151260657e+00 + 5.194530647433734e+00 + 5.130243129823101e+00 + 5.066698206416531e+00 + 5.003887570881058e+00 + 4.941803001453549e+00 + 4.880436373900456e+00 + 4.819779643419130e+00 + 4.759824844738533e+00 + 4.700564110200850e+00 + 4.641989649988083e+00 + 4.584093751736625e+00 + 4.526868793565781e+00 + 4.470307230074105e+00 + 4.414401594729965e+00 + 4.359144507919768e+00 + 4.304528662004969e+00 + 4.250546825086277e+00 + 4.197191851627622e+00 + 4.144456666159979e+00 + 4.092334266478478e+00 + 4.040817732600400e+00 + 3.989900213340690e+00 + 3.939574930757155e+00 + 3.889835183170760e+00 + 3.840674335613269e+00 + 3.792085827136757e+00 + 3.744063169627271e+00 + 3.696599938809659e+00 + 3.649689781941428e+00 + 3.603326418080199e+00 + 3.557503628382808e+00 + 3.512215261374414e+00 + 3.467455236465362e+00 + 3.423217534816642e+00 + 3.379496201686086e+00 + 3.336285348654454e+00 + 3.293579148649655e+00 + 3.251371841226642e+00 + 3.209657725464606e+00 + 3.168431157939940e+00 + 3.127686563504146e+00 + 3.087418425005168e+00 + 3.047621281181530e+00 + 3.008289735457784e+00 + 2.969418446796439e+00 + 2.931002129285814e+00 + 2.893035560502260e+00 + 2.855513571053400e+00 + 2.818431044951714e+00 + 2.781782927380857e+00 + 2.745564215550934e+00 + 2.709769959926351e+00 + 2.674395268211684e+00 + 2.639435297946605e+00 + 2.604885259637663e+00 + 2.570740420206636e+00 + 2.536996093812299e+00 + 2.503647644969884e+00 + 2.470690493614489e+00 + 2.438120105903952e+00 + 2.405931997235836e+00 + 2.374121735683807e+00 + 2.342684934097179e+00 + 2.311617253784086e+00 + 2.280914406216957e+00 + 2.250572146781167e+00 + 2.220586279273893e+00 + 2.190952653478714e+00 + 2.161667162009881e+00 + 2.132725747880213e+00 + 2.104124396422602e+00 + 2.075859132902099e+00 + 2.047926031580743e+00 + 2.020321209266692e+00 + 1.993040822266800e+00 + 1.966081073072867e+00 + 1.939438205166541e+00 + 1.913108500802827e+00 + 1.887088286918805e+00 + 1.861373929947225e+00 + 1.835961834345823e+00 + 1.810848447621857e+00 + 1.786030255213551e+00 + 1.761503779943143e+00 + 1.737265586321442e+00 + 1.713312275115213e+00 + 1.689640483715834e+00 + 1.666246890168493e+00 + 1.643128207113473e+00 + 1.620281182788458e+00 + 1.597702604935450e+00 + 1.575389294699183e+00 + 1.553338108095691e+00 + 1.531545938928566e+00 + 1.510009713407906e+00 + 1.488726392261371e+00 + 1.467692971937850e+00 + 1.446906480207705e+00 + 1.426363979153670e+00 + 1.406062564959602e+00 + 1.385999363940103e+00 + 1.366171536058216e+00 + 1.346576274130787e+00 + 1.327210800124654e+00 + 1.308072368828510e+00 + 1.289158266486426e+00 + 1.270465807665891e+00 + 1.251992339187792e+00 + 1.233735237889272e+00 + 1.215691907984316e+00 + 1.197859785096899e+00 + 1.180236333919823e+00 + 1.162819045897185e+00 + 1.145605442741095e+00 + 1.128593073973588e+00 + 1.111779515284567e+00 + 1.095162371880828e+00 + 1.078739275475473e+00 + 1.062507883334588e+00 + 1.046465881646835e+00 + 1.030610981874106e+00 + 1.014940920295882e+00 + 9.994534611152829e-01 + 9.841463930086627e-01 + 9.690175291593270e-01 + 9.540647093618980e-01 + 9.392857969051628e-01 + 9.246786792729085e-01 + 9.102412696338000e-01 + 8.959715035501604e-01 + 8.818673402027871e-01 + 8.679267639726700e-01 + 8.541477806828601e-01 + 8.405284189410931e-01 + 8.270667316623297e-01 + 8.137607925445277e-01 + 8.006086976874616e-01 + 7.876085666023992e-01 + 7.747585387876328e-01 + 7.620567754307064e-01 + 7.495014602973783e-01 + 7.370907966216457e-01 + 7.248230088485250e-01 + 7.126963429559050e-01 + 7.007090638413013e-01 + 6.888594572109231e-01 + 6.771458292779966e-01 + 6.655665045807759e-01 + 6.541198279036498e-01 + 6.428041637825399e-01 + 6.316178947210964e-01 + 6.205594229105623e-01 + 6.096271692194244e-01 + 5.988195719936049e-01 + 5.881350889221284e-01 + 5.775721954595900e-01 + 5.671293841176136e-01 + 5.568051664208875e-01 + 5.465980707194322e-01 + 5.365066418737608e-01 + 5.265294432641988e-01 + 5.166650544231339e-01 + 5.069120710261563e-01 + 4.972691065208522e-01 + 4.877347899800689e-01 + 4.783077664492044e-01 + 4.689866979017357e-01 + 4.597702612073233e-01 + 4.506571489021905e-01 + 4.416460700276524e-01 + 4.327357479310139e-01 + 4.239249212044141e-01 + 4.152123442913231e-01 + 4.065967853823845e-01 + 3.980770275573000e-01 + 3.896518691214699e-01 + 3.813201217302061e-01 + 3.730806117264635e-01 + 3.649321798861945e-01 + 3.568736798624809e-01 + 3.489039797704173e-01 + 3.410219613244947e-01 + 3.332265187013593e-01 + 3.255165605086018e-01 + 3.178910083898543e-01 + 3.103487958586359e-01 + 3.028888701360887e-01 + 2.955101913786422e-01 + 2.882117315106021e-01 + 2.809924755137371e-01 + 2.738514204626846e-01 + 2.667875748867065e-01 + 2.597999604522775e-01 + 2.528876103054370e-01 + 2.460495685714950e-01 + 2.392848919788540e-01 + 2.325926484518769e-01 + 2.259719167857841e-01 + 2.194217877354759e-01 + 2.129413627175208e-01 + 2.065297538274627e-01 + 2.001860848470379e-01 + 1.939094897086202e-01 + 1.876991127051828e-01 + 1.815541094417414e-01 + 1.754736452756007e-01 + 1.694568956974621e-01 + 1.635030470498157e-01 + 1.576112950508609e-01 + 1.517808453564031e-01 + 1.460109140364644e-01 + 1.403007261766342e-01 + 1.346495165784953e-01 + 1.290565300180611e-01 + 1.235210199704221e-01 + 1.180422494287421e-01 + 1.126194909047226e-01 + 1.072520253105725e-01 + 1.019391428882170e-01 + 9.668014295417104e-02 + 9.147433293860986e-02 + 8.632102939670111e-02 + 8.121955756354367e-02 + 7.616925052308399e-02 + 7.116945022739722e-02 + 6.621950693894064e-02 + 6.131877854698166e-02 + 5.646663155698649e-02 + 5.166244039926220e-02 + 4.690558691983024e-02 + 4.219546135291571e-02 + 3.753146148005052e-02 + 3.291299228995687e-02 + 2.833946692296159e-02 + 2.381030573044091e-02 + 1.932493608562982e-02 + 1.488279323951757e-02 + 1.048331933378767e-02 + 6.125963364112487e-03 + 1.810181928441051e-03 + -2.464561780353438e-03 + -6.698797872311066e-03 + -1.089304914227914e-02 + -1.504783206514415e-02 + -1.916365656771068e-02 + -2.324102555134537e-02 + -2.728043585421590e-02 + -3.128237791440187e-02 + -3.524733543518016e-02 + -3.917578628623598e-02 + -4.306820207473552e-02 + -4.692504798332473e-02 + -5.074678357205704e-02 + -5.453386225593208e-02 + -5.828673132987599e-02 + -6.200583267980977e-02 + -6.569160218738558e-02 + -6.934446987831328e-02 + -7.296486055963237e-02 + -7.655319319650564e-02 + -8.010988114185758e-02 + -8.363533270338043e-02 + -8.712995053780122e-02 + -9.059413193193132e-02 + -9.402826926020177e-02 + -9.743274938790861e-02 + -1.008079541064505e-01 + -1.041542604608216e-01 + -1.074720401131623e-01 + -1.107616598682374e-01 + -1.140234819193764e-01 + -1.172578632830942e-01 + -1.204651563472630e-01 + -1.236457089966172e-01 + -1.267998640735528e-01 + -1.299279600346222e-01 + -1.330303309819349e-01 + -1.361073061299021e-01 + -1.391592104828469e-01 + -1.421863647988072e-01 + -1.451890851855495e-01 + -1.481676836992942e-01 + -1.511224682100277e-01 + -1.540537421399553e-01 + -1.569618050599837e-01 + -1.598469524764272e-01 + -1.627094755828420e-01 + -1.655496618778017e-01 + -1.683677949236447e-01 + -1.711641541517163e-01 + -1.739390154343296e-01 + -1.766926508045859e-01 + -1.794253283544359e-01 + -1.821373127460387e-01 + -1.848288648981078e-01 + -1.875002419786971e-01 + -1.901516978443653e-01 + -1.927834826976386e-01 + -1.953958431846420e-01 + -1.979890227559544e-01 + -2.005632612952037e-01 + -2.031187952951354e-01 + -2.056558581586019e-01 + -2.081746798356756e-01 + -2.106754870586183e-01 + -2.131585035595214e-01 + -2.156239496932611e-01 + -2.180720427808593e-01 + -2.205029972492180e-01 + -2.229170242226580e-01 + -2.253143319488933e-01 + -2.276951258807381e-01 + -2.300596082676421e-01 + -2.324079786235529e-01 + -2.347404337552020e-01 + -2.370571673694448e-01 + -2.393583705568364e-01 + -2.416442317867498e-01 + -2.439149365965413e-01 + -2.461706679522762e-01 + -2.484116061949506e-01 + -2.506379289887288e-01 + -2.528498115997045e-01 + -2.550474267213886e-01 + -2.572309443123204e-01 + -2.594005320626763e-01 + -2.615563552809436e-01 + -2.636985766193694e-01 + -2.658273564814242e-01 + -2.679428529152917e-01 + -2.700452216727706e-01 + -2.721346163277996e-01 + -2.742111879420810e-01 + -2.762750854486131e-01 + -2.783264557785196e-01 + -2.803654434689624e-01 + -2.823921908087395e-01 + -2.844068381756654e-01 + -2.864095239109061e-01 + -2.884003839950212e-01 + -2.903795524955394e-01 + -2.923471617904532e-01 + -2.943033420191942e-01 + -2.962482212763220e-01 + -2.981819256807660e-01 + -3.001045795717013e-01 + -3.020163055275332e-01 + -3.039172239885258e-01 + -3.058074538261057e-01 + -3.076871122705734e-01 + -3.095563141516905e-01 + -3.114151727559051e-01 + -3.132637999546681e-01 + -3.151023057635463e-01 + -3.169307984585886e-01 + -3.187493845723606e-01 + -3.205581689117248e-01 + -3.223572548745565e-01 + -3.241467443749414e-01 + -3.259267374142799e-01 + -3.276973325691862e-01 + -3.294586270053045e-01 + -3.312107160890871e-01 + -3.329536937962613e-01 + -3.346876527428858e-01 + -3.364126840777460e-01 + -3.381288775295205e-01 + -3.398363212268976e-01 + -3.415351019703767e-01 + -3.432253052941175e-01 + -3.449070152459444e-01 + -3.465803145779997e-01 + -3.482452848007116e-01 + -3.499020059501182e-01 + -3.515505568826250e-01 + -3.531910152859590e-01 + -3.548234573752725e-01 + -3.564479582224311e-01 + -3.580645917875246e-01 + -3.596734306319520e-01 + -3.612745463110025e-01 + -3.628680093075062e-01 + -3.644538885605036e-01 + -3.660322520458910e-01 + -3.676031668332329e-01 + -3.691666987516095e-01 + -3.707229124779602e-01 + -3.722718716670178e-01 + -3.738136390608672e-01 + -3.753482761467254e-01 + -3.768758432749062e-01 + -3.783964002465877e-01 + -3.799100056353694e-01 + -3.814167166997948e-01 + -3.829165900177774e-01 + -3.844096813827866e-01 + -3.858960456354959e-01 + -3.873757364088854e-01 + -3.888488064600710e-01 + -3.903153077375676e-01 + -3.917752913452379e-01 + -3.932288075353338e-01 + -3.946759055987133e-01 + -3.961166339473677e-01 + -3.975510402779339e-01 + -3.989791715122148e-01 + -4.004010735521537e-01 + -4.018167915706618e-01 + -4.032263700983349e-01 + -4.046298528561717e-01 + -4.060272827301947e-01 + -4.074187016384220e-01 + -4.088041511045764e-01 + -4.101836719923956e-01 + -4.115573039941210e-01 + -4.129250863351419e-01 + -4.142870577221169e-01 + -4.156432559911067e-01 + -4.169937183587125e-01 + -4.183384813924751e-01 + -4.196775807600287e-01 + -4.210110517890521e-01 + -4.223389293056927e-01 + -4.236612469859990e-01 + -4.249780381358575e-01 + -4.262893357128523e-01 + -4.275951718561576e-01 + -4.288955781548201e-01 + -4.301905856512787e-01 + -4.314802245948859e-01 + -4.327645248125306e-01 + -4.340435157401011e-01 + -4.353172261319986e-01 + -4.365856843065750e-01 + -4.378489181140523e-01 + -4.391069545111200e-01 + -4.403598201489721e-01 + -4.416075414900063e-01 + -4.428501441617211e-01 + -4.440876533447125e-01 + -4.453200938994337e-01 + -4.465474900663732e-01 + -4.477698656375739e-01 + -4.489872440132049e-01 + -4.501996482091757e-01 + -4.514071007293397e-01 + -4.526096234423685e-01 + -4.538072381973440e-01 + -4.549999664346564e-01 + -4.561878287239103e-01 + -4.573708454564815e-01 + -4.585490367840940e-01 + -4.597224223838171e-01 + -4.608910215483196e-01 + -4.620548532229073e-01 + -4.632139359470852e-01 + -4.643682878385313e-01 + -4.655179266829866e-01 + -4.666628700026897e-01 + -4.678031349656974e-01 + -4.689387384105899e-01 + -4.700696968538801e-01 + -4.711960263436956e-01 + -4.723177425902391e-01 + -4.734348612979066e-01 + -4.745473978039702e-01 + -4.756553669920032e-01 + -4.767587833133353e-01 + -4.778576611599107e-01 + -4.789520147944609e-01 + -4.800418577229591e-01 + -4.811272034289876e-01 + -4.822080654661853e-01 + -4.832844566828151e-01 + -4.843563896207272e-01 + -4.854238767122197e-01 + -4.864869303811440e-01 + -4.875455627166793e-01 + -4.885997853037442e-01 + -4.896496095260600e-01 + -4.906950466868944e-01 + -4.917361080077966e-01 + -4.927728043326091e-01 + -4.938051461993963e-01 + -4.948331438840640e-01 + -4.958568075736277e-01 + -4.968761473512240e-01 + -4.978911731225704e-01 + -4.989018945002663e-01 + -4.999083208482176e-01 + -5.009104613021941e-01 + -5.019083248972503e-01 + -5.029019205602130e-01 + -5.038912570358333e-01 + -5.048763428126108e-01 + -5.058571862004539e-01 + -5.068337954285524e-01 + -5.078061785541023e-01 + -5.087743434557772e-01 + -5.097382978215317e-01 + -5.106980492175714e-01 + -5.116536051046018e-01 + -5.126049728172039e-01 + -5.135521595357905e-01 + -5.144951722594635e-01 + -5.154340177423219e-01 + -5.163687027408718e-01 + -5.172992340140365e-01 + -5.182256180631845e-01 + -5.191478612397169e-01 + -5.200659697981208e-01 + -5.209799499565910e-01 + -5.218898077706294e-01 + -5.227955490891694e-01 + -5.236971796844924e-01 + -5.245947053253370e-01 + -5.254881317688337e-01 + -5.263774645512288e-01 + -5.272627090126604e-01 + -5.281438703531137e-01 + -5.290209539977515e-01 + -5.298939652703512e-01 + -5.307629091340913e-01 + -5.316277904778723e-01 + -5.324886142110474e-01 + -5.333453852986519e-01 + -5.341981085648844e-01 + -5.350467886587517e-01 + -5.358914300418783e-01 + -5.367320373706596e-01 + -5.375686152841432e-01 + -5.384011680734792e-01 + -5.392297001047128e-01 + -5.400542157871245e-01 + -5.408747194069761e-01 + -5.416912150785628e-01 + -5.425037068232484e-01 + -5.433121987186342e-01 + -5.441166948984431e-01 + -5.449171994793097e-01 + -5.457137163885653e-01 + -5.465062494211190e-01 + -5.472948023341125e-01 + -5.480793790518950e-01 + -5.488599834110802e-01 + -5.496366191273470e-01 + -5.504092899443518e-01 + -5.511779994928784e-01 + -5.519427513029639e-01 + -5.527035490710489e-01 + -5.534603964642170e-01 + -5.542132970155166e-01 + -5.549622540841348e-01 + -5.557072711574547e-01 + -5.564483519201564e-01 + -5.571854997347668e-01 + -5.579187179919134e-01 + -5.586480102698358e-01 + -5.593733798207278e-01 + -5.600948299343745e-01 + -5.608123641702291e-01 + -5.615259858006734e-01 + -5.622356980349319e-01 + -5.629415042452564e-01 + -5.636434078437447e-01 + -5.643414121719204e-01 + -5.650355204128609e-01 + -5.657257359067179e-01 + -5.664120620074502e-01 + -5.670945018655422e-01 + -5.677730587569899e-01 + -5.684477360210265e-01 + -5.691185368946378e-01 + -5.697854646230838e-01 + -5.704485224954847e-01 + -5.711077138615709e-01 + -5.717630418949810e-01 + -5.724145097236072e-01 + -5.730621208149326e-01 + -5.737058785857441e-01 + -5.743457862811681e-01 + -5.749818469287452e-01 + -5.756140639044424e-01 + -5.762424408221788e-01 + -5.768669806578469e-01 + -5.774876866716016e-01 + -5.781045625480883e-01 + -5.787176116189466e-01 + -5.793268371095267e-01 + -5.799322422747063e-01 + -5.805338304985933e-01 + -5.811316053198213e-01 + -5.817255703521417e-01 + -5.823157287697720e-01 + -5.829020838792335e-01 + -5.834846393919914e-01 + -5.840633987477251e-01 + -5.846383653791880e-01 + -5.852095428966003e-01 + -5.857769347455314e-01 + -5.863405444497650e-01 + -5.869003757905312e-01 + -5.874564321893059e-01 + -5.880087171065591e-01 + -5.885572344676900e-01 + -5.891019879107314e-01 + -5.896429809482692e-01 + -5.901802172691951e-01 + -5.907137006344829e-01 + -5.912434348345698e-01 + -5.917694236578408e-01 + -5.922916709366750e-01 + -5.928101804947136e-01 + -5.933249560457028e-01 + -5.938360014505492e-01 + -5.943433207019595e-01 + -5.948469177828256e-01 + -5.953467966083078e-01 + -5.958429610716830e-01 + -5.963354151930935e-01 + -5.968241629967538e-01 + -5.973092085065336e-01 + -5.977905558475627e-01 + -5.982682091919279e-01 + -5.987421727060626e-01 + -5.992124504335189e-01 + -5.996790465513929e-01 + -6.001419654039183e-01 + -6.006012111567195e-01 + -6.010567880969444e-01 + -6.015087007096906e-01 + -6.019569532209035e-01 + -6.024015499278780e-01 + -6.028424953542454e-01 + -6.032797938560218e-01 + -6.037134498528102e-01 + -6.041434679778884e-01 + -6.045698527611498e-01 + -6.049926086966888e-01 + -6.054117403213235e-01 + -6.058272522301629e-01 + -6.062391491063152e-01 + -6.066474357377917e-01 + -6.070521167750024e-01 + -6.074531969194140e-01 + -6.078506811508277e-01 + -6.082445741313003e-01 + -6.086348804777613e-01 + -6.090216052590219e-01 + -6.094047534940796e-01 + -6.097843300758768e-01 + -6.101603397998798e-01 + -6.105327877567767e-01 + -6.109016791586246e-01 + -6.112670188349342e-01 + -6.116288118244032e-01 + -6.119870634033990e-01 + -6.123417788015699e-01 + -6.126929631816058e-01 + -6.130406216857481e-01 + -6.133847595846266e-01 + -6.137253821544012e-01 + -6.140624946688724e-01 + -6.143961025253998e-01 + -6.147262111225434e-01 + -6.150528258526081e-01 + -6.153759522847098e-01 + -6.156955958775930e-01 + -6.160117619293205e-01 + -6.163244560848270e-01 + -6.166336840291214e-01 + -6.169394513190288e-01 + -6.172417635406579e-01 + -6.175406263458638e-01 + -6.178360454670117e-01 + -6.181280266570051e-01 + -6.184165756494230e-01 + -6.187016981615432e-01 + -6.189834001775048e-01 + -6.192616876391890e-01 + -6.195365661632112e-01 + -6.198080416901866e-01 + -6.200761202902517e-01 + -6.203408078682074e-01 + -6.206021103982113e-01 + -6.208600339819080e-01 + -6.211145848662764e-01 + -6.213657689929696e-01 + -6.216135922609831e-01 + -6.218580610691606e-01 + -6.220991816188346e-01 + -6.223369599996067e-01 + -6.225714026287344e-01 + -6.228025157602012e-01 + -6.230303055354752e-01 + -6.232547783843585e-01 + -6.234759406837302e-01 + -6.236937987287140e-01 + -6.239083589372102e-01 + -6.241196278170006e-01 + -6.243276119301496e-01 + -6.245323178052680e-01 + -6.247337518435782e-01 + -6.249319204108740e-01 + -6.251268303788342e-01 + -6.253184884524475e-01 + -6.255069009766967e-01 + -6.256920746682559e-01 + -6.258740163701917e-01 + -6.260527328726909e-01 + -6.262282307801417e-01 + -6.264005167589047e-01 + -6.265695976839164e-01 + -6.267354804070123e-01 + -6.268981717782514e-01 + -6.270576786775590e-01 + -6.272140080071444e-01 + -6.273671666729116e-01 + -6.275171615794166e-01 + -6.276639997579259e-01 + -6.278076882572314e-01 + -6.279482340166599e-01 + -6.280856440870369e-01 + -6.282199255881125e-01 + -6.283510856176625e-01 + -6.284791313265480e-01 + -6.286040698559974e-01 + -6.287259082244099e-01 + -6.288446536460758e-01 + -6.289603134448759e-01 + -6.290728947784819e-01 + -6.291824049507972e-01 + -6.292888513285516e-01 + -6.293922409846477e-01 + -6.294925812787570e-01 + -6.295898797737625e-01 + -6.296841435561055e-01 + -6.297753800606519e-01 + -6.298635970427606e-01 + -6.299488015475858e-01 + -6.300310009429952e-01 + -6.301102030719377e-01 + -6.301864151575183e-01 + -6.302596446383568e-01 + -6.303298993873763e-01 + -6.303971867419816e-01 + -6.304615141097702e-01 + -6.305228892569528e-01 + -6.305813198299395e-01 + -6.306368133767261e-01 + -6.306893774095105e-01 + -6.307390196514921e-01 + -6.307857478830511e-01 + -6.308295697989852e-01 + -6.308704929504492e-01 + -6.309085250312678e-01 + -6.309436740902202e-01 + -6.309759477467396e-01 + -6.310053535533821e-01 + -6.310318994644450e-01 + -6.310555933584764e-01 + -6.310764430051494e-01 + -6.310944561130111e-01 + -6.311096405770181e-01 + -6.311220043955681e-01 + -6.311315554778594e-01 + -6.311383016234806e-01 + -6.311422506666780e-01 + -6.311434107412031e-01 + -6.311417897164122e-01 + -6.311373953205278e-01 + -6.311302357049288e-01 + -6.311203189243589e-01 + -6.311076528624781e-01 + -6.310922454230109e-01 + -6.310741047155799e-01 + -6.310532389686786e-01 + -6.310296560496979e-01 + -6.310033639455465e-01 + -6.309743708604478e-01 + -6.309426848479184e-01 + -6.309083139781310e-01 + -6.308712663782245e-01 + -6.308315500367087e-01 + -6.307891730465176e-01 + -6.307441436976022e-01 + -6.306964701860184e-01 + -6.306461606431758e-01 + -6.305932231701467e-01 + -6.305376659121696e-01 + -6.304794970062023e-01 + -6.304187245700347e-01 + -6.303553569805046e-01 + -6.302894025962669e-01 + -6.302208694979176e-01 + -6.301497657748981e-01 + -6.300760996409697e-01 + -6.299998795100692e-01 + -6.299211136514613e-01 + -6.298398102240708e-01 + -6.297559773915322e-01 + -6.296696236381925e-01 + -6.295807574013027e-01 + -6.294893864698483e-01 + -6.293955192818200e-01 + -6.292991645521353e-01 + -6.292003301703828e-01 + -6.290990243894826e-01 + -6.289952557946986e-01 + -6.288890326523825e-01 + -6.287803632353285e-01 + -6.286692558944412e-01 + -6.285557190083507e-01 + -6.284397608763040e-01 + -6.283213897612988e-01 + -6.282006141317734e-01 + -6.280774423035318e-01 + -6.279518824496017e-01 + -6.278239431715447e-01 + -6.276936329659792e-01 + -6.275609600561086e-01 + -6.274259327786519e-01 + -6.272885594754783e-01 + -6.271488484695774e-01 + -6.270068083043618e-01 + -6.268624473656752e-01 + -6.267157737459064e-01 + -6.265667960682418e-01 + -6.264155229504741e-01 + -6.262619625962428e-01 + -6.261061232036680e-01 + -6.259480131935671e-01 + -6.257876413121770e-01 + -6.256250157840939e-01 + -6.254601447340036e-01 + -6.252930366972961e-01 + -6.251237002207461e-01 + -6.249521437614608e-01 + -6.247783756272571e-01 + -6.246024042025771e-01 + -6.244242378628240e-01 + -6.242438848129136e-01 + -6.240613535991787e-01 + -6.238766528797192e-01 + -6.236897908519111e-01 + -6.235007757894835e-01 + -6.233096161276966e-01 + -6.231163203935827e-01 + -6.229208970207127e-01 + -6.227233543185018e-01 + -6.225237005131427e-01 + -6.223219439896936e-01 + -6.221180932821515e-01 + -6.219121568092019e-01 + -6.217041429518334e-01 + -6.214940600654550e-01 + -6.212819164164759e-01 + -6.210677203231969e-01 + -6.208514802076882e-01 + -6.206332045396580e-01 + -6.204129016762088e-01 + -6.201905798335827e-01 + -6.199662474146647e-01 + -6.197399127976954e-01 + -6.195115842446862e-01 + -6.192812702436963e-01 + -6.190489791767655e-01 + -6.188147191241502e-01 + -6.185784984898128e-01 + -6.183403256742428e-01 + -6.181002088023494e-01 + -6.178581563548247e-01 + -6.176141767575239e-01 + -6.173682779492388e-01 + -6.171204683326561e-01 + -6.168707564823446e-01 + -6.166191506410910e-01 + -6.163656588303806e-01 + -6.161102891648960e-01 + -6.158530502595377e-01 + -6.155939503803246e-01 + -6.153329975648972e-01 + -6.150702000825419e-01 + -6.148055662334609e-01 + -6.145391042808102e-01 + -6.142708224010276e-01 + -6.140007288452335e-01 + -6.137288318637003e-01 + -6.134551394181680e-01 + -6.131796597624336e-01 + -6.129024013716257e-01 + -6.126233721919256e-01 + -6.123425803085120e-01 + -6.120600340492245e-01 + -6.117757414641359e-01 + -6.114897106263320e-01 + -6.112019497520672e-01 + -6.109124671249474e-01 + -6.106212708396386e-01 + -6.103283687668304e-01 + -6.100337690977098e-01 + -6.097374799911853e-01 + -6.094395093910078e-01 + -6.091398653642833e-01 + -6.088385561024087e-01 + -6.085355898356886e-01 + -6.082309742898534e-01 + -6.079247173286892e-01 + -6.076168274335947e-01 + -6.073073125569597e-01 + -6.069961804304546e-01 + -6.066834389911383e-01 + -6.063690964791788e-01 + -6.060531610171422e-01 + -6.057356401425160e-01 + -6.054165419100932e-01 + -6.050958745176729e-01 + -6.047736455801236e-01 + -6.044498630713394e-01 + -6.041245351487873e-01 + -6.037976695799242e-01 + -6.034692741523835e-01 + -6.031393567367498e-01 + -6.028079252312161e-01 + -6.024749874110954e-01 + -6.021405510415967e-01 + -6.018046242357692e-01 + -6.014672148046522e-01 + -6.011283302871711e-01 + -6.007879786053585e-01 + -6.004461675660895e-01 + -6.001029047914931e-01 + -5.997581981158361e-01 + -5.994120553308485e-01 + -5.990644841299763e-01 + -5.987154923482985e-01 + -5.983650876334137e-01 + -5.980132773866208e-01 + -5.976600693793478e-01 + -5.973054714798371e-01 + -5.969494914365839e-01 + -5.965921366499265e-01 + -5.962334146660923e-01 + -5.958733334161985e-01 + -5.955119002909084e-01 + -5.951491227180785e-01 + -5.947850085981863e-01 + -5.944195652847630e-01 + -5.940528001366523e-01 + -5.936847210858313e-01 + -5.933153356463837e-01 + -5.929446511281147e-01 + -5.925726749957585e-01 + -5.921994146776245e-01 + -5.918248776317639e-01 + -5.914490714685531e-01 + -5.910720036664838e-01 + -5.906936815737919e-01 + -5.903141124838686e-01 + -5.899333037986367e-01 + -5.895512629817115e-01 + -5.891679973989596e-01 + -5.887835143677140e-01 + -5.883978211708401e-01 + -5.880109250637769e-01 + -5.876228334774148e-01 + -5.872335538851382e-01 + -5.868430932520374e-01 + -5.864514587624869e-01 + -5.860586578978172e-01 + -5.856646978273519e-01 + -5.852695857497323e-01 + -5.848733289413443e-01 + -5.844759343592291e-01 + -5.840774091773326e-01 + -5.836777608879199e-01 + -5.832769964405170e-01 + -5.828751228494087e-01 + -5.824721474461222e-01 + -5.820680771414760e-01 + -5.816629188869962e-01 + -5.812566799399100e-01 + -5.808493672421570e-01 + -5.804409877894702e-01 + -5.800315488931496e-01 + -5.796210574409676e-01 + -5.792095201878247e-01 + -5.787969440874288e-01 + -5.783833362006023e-01 + -5.779687035684036e-01 + -5.775530530727403e-01 + -5.771363914738108e-01 + -5.767187255974381e-01 + -5.763000625573806e-01 + -5.758804091701334e-01 + -5.754597721176316e-01 + -5.750381583420940e-01 + -5.746155745735496e-01 + -5.741920274506414e-01 + -5.737675239291555e-01 + -5.733420708457294e-01 + -5.729156748671344e-01 + -5.724883426083383e-01 + -5.720600807791615e-01 + -5.716308961653045e-01 + -5.712007954599810e-01 + -5.707697852850102e-01 + -5.703378722272431e-01 + -5.699050629355835e-01 + -5.694713639828742e-01 + -5.690367818720566e-01 + -5.686013232951076e-01 + -5.681649948578290e-01 + -5.677278029894381e-01 + -5.672897541841998e-01 + -5.668508549514105e-01 + -5.664111117923113e-01 + -5.659705312703284e-01 + -5.655291198708162e-01 + -5.650868839171556e-01 + -5.646438297396282e-01 + -5.641999637601537e-01 + -5.637552925169101e-01 + -5.633098223391846e-01 + -5.628635595080524e-01 + -5.624165104013904e-01 + -5.619686812129524e-01 + -5.615200782186582e-01 + -5.610707080464411e-01 + -5.606205767194389e-01 + -5.601696901727680e-01 + -5.597180550627928e-01 + -5.592656777039943e-01 + -5.588125640834879e-01 + -5.583587201544192e-01 + -5.579041522213387e-01 + -5.574488666805684e-01 + -5.569928694014434e-01 + -5.565361664370690e-01 + -5.560787640439252e-01 + -5.556206683493946e-01 + -5.551618853326125e-01 + -5.547024208857518e-01 + -5.542422809960397e-01 + -5.537814718214656e-01 + -5.533199995563840e-01 + -5.528578699065892e-01 + -5.523950887012330e-01 + -5.519316620139326e-01 + -5.514675956847120e-01 + -5.510028956792654e-01 + -5.505375681122012e-01 + -5.500716185256428e-01 + -5.496050526609592e-01 + -5.491378767346244e-01 + -5.486700964364043e-01 + -5.482017173715276e-01 + -5.477327453596050e-01 + -5.472631862648399e-01 + -5.467930458019109e-01 + -5.463223294669347e-01 + -5.458510432287255e-01 + -5.453791929736209e-01 + -5.449067840000759e-01 + -5.444338218966174e-01 + -5.439603123952597e-01 + -5.434862611282417e-01 + -5.430116738240560e-01 + -5.425365561436022e-01 + -5.420609134355956e-01 + -5.415847511352451e-01 + -5.411080748136494e-01 + -5.406308901525899e-01 + -5.401532026526268e-01 + -5.396750177177082e-01 + -5.391963408895222e-01 + -5.387171774649071e-01 + -5.382375326730170e-01 + -5.377574122460893e-01 + -5.372768215524759e-01 + -5.367957656717409e-01 + -5.363142502106338e-01 + -5.358322806031541e-01 + -5.353498619928253e-01 + -5.348669995911720e-01 + -5.343836987381033e-01 + -5.338999648334884e-01 + -5.334158029976483e-01 + -5.329312184607295e-01 + -5.324462166033185e-01 + -5.319608023124877e-01 + -5.314749807283198e-01 + -5.309887574568507e-01 + -5.305021373834015e-01 + -5.300151254489873e-01 + -5.295277270550899e-01 + -5.290399471460621e-01 + -5.285517906644440e-01 + -5.280632628772480e-01 + -5.275743687930865e-01 + -5.270851133485194e-01 + -5.265955015863814e-01 + -5.261055383771222e-01 + -5.256152286457003e-01 + -5.251245776074701e-01 + -5.246335901741112e-01 + -5.241422711040373e-01 + -5.236506252667474e-01 + -5.231586574906982e-01 + -5.226663726352896e-01 + -5.221737757253558e-01 + -5.216808715970676e-01 + -5.211876649797802e-01 + -5.206941607382560e-01 + -5.202003635069621e-01 + -5.197062778404161e-01 + -5.192119087096243e-01 + -5.187172609937223e-01 + -5.182223393579939e-01 + -5.177271482807863e-01 + -5.172316923981860e-01 + -5.167359765148800e-01 + -5.162400053033168e-01 + -5.157437833544083e-01 + -5.152473152222008e-01 + -5.147506055161414e-01 + -5.142536587718484e-01 + -5.137564794429633e-01 + -5.132590721404230e-01 + -5.127614414610688e-01 + -5.122635918987255e-01 + -5.117655278912282e-01 + -5.112672538418732e-01 + -5.107687741570007e-01 + -5.102700934318621e-01 + -5.097712161511054e-01 + -5.092721464952894e-01 + -5.087728887586368e-01 + -5.082734473837237e-01 + -5.077738269224511e-01 + -5.072740315773641e-01 + -5.067740655070005e-01 + -5.062739331645890e-01 + -5.057736389339202e-01 + -5.052731870571522e-01 + -5.047725816027976e-01 + -5.042718267556677e-01 + -5.037709267723895e-01 + -5.032698858620083e-01 + -5.027687082997551e-01 + -5.022673983228978e-01 + -5.017659599138892e-01 + -5.012643972224838e-01 + -5.007627144504877e-01 + -5.002609154317549e-01 + -4.997590043686060e-01 + -4.992569856498065e-01 + -4.987548629163329e-01 + -4.982526401066572e-01 + -4.977503215591422e-01 + -4.972479113357710e-01 + -4.967454132608579e-01 + -4.962428310876830e-01 + -4.957401690448965e-01 + -4.952374311333114e-01 + -4.947346209581790e-01 + -4.942317423902789e-01 + -4.937287995168010e-01 + -4.932257964768885e-01 + -4.927227367709692e-01 + -4.922196239985463e-01 + -4.917164622446805e-01 + -4.912132554897460e-01 + -4.907100074652099e-01 + -4.902067216434476e-01 + -4.897034018910860e-01 + -4.892000521396699e-01 + -4.886966760659724e-01 + -4.881932772071136e-01 + -4.876898592264887e-01 + -4.871864261030248e-01 + -4.866829812293292e-01 + -4.861795280147214e-01 + -4.856760706607693e-01 + -4.851726126266216e-01 + -4.846691571189501e-01 + -4.841657080652781e-01 + -4.836622690168214e-01 + -4.831588433295234e-01 + -4.826554347812969e-01 + -4.821520468729375e-01 + -4.816486828820545e-01 + -4.811453462713459e-01 + -4.806420406166251e-01 + -4.801387695127853e-01 + -4.796355363816449e-01 + -4.791323444815042e-01 + -4.786291970695780e-01 + -4.781260978727254e-01 + -4.776230503252043e-01 + -4.771200574973888e-01 + -4.766171228253449e-01 + -4.761142497486388e-01 + -4.756114415594656e-01 + -4.751087014547341e-01 + -4.746060327035595e-01 + -4.741034386828772e-01 + -4.736009226286348e-01 + -4.730984877717556e-01 + -4.725961373913972e-01 + -4.720938746127822e-01 + -4.715917026117559e-01 + -4.710896247171748e-01 + -4.705876440070008e-01 + -4.700857635192827e-01 + -4.695839864443119e-01 + -4.690823160197067e-01 + -4.685807553687465e-01 + -4.680793073815264e-01 + -4.675779752270572e-01 + -4.670767621237579e-01 + -4.665756710130068e-01 + -4.660747048828000e-01 + -4.655738667469987e-01 + -4.650731595624170e-01 + -4.645725864569897e-01 + -4.640721505529091e-01 + -4.635718546060059e-01 + -4.630717014390969e-01 + -4.625716940204364e-01 + -4.620718354412200e-01 + -4.615721285948061e-01 + -4.610725762550177e-01 + -4.605731814392849e-01 + -4.600739469669758e-01 + -4.595748755034438e-01 + -4.590759700613133e-01 + -4.585772335293120e-01 + -4.580786685902926e-01 + -4.575802780165928e-01 + -4.570820645542124e-01 + -4.565840309327809e-01 + -4.560861801183225e-01 + -4.555885148408127e-01 + -4.550910375242214e-01 + -4.545937511574892e-01 + -4.540966585092119e-01 + -4.535997618400510e-01 + -4.531030641427475e-01 + -4.526065682230286e-01 + -4.521102761991050e-01 + -4.516141909960451e-01 + -4.511183155232459e-01 + -4.506226519976986e-01 + -4.501272029129367e-01 + -4.496319709853875e-01 + -4.491369589904647e-01 + -4.486421693813144e-01 + -4.481476045149588e-01 + -4.476532669406306e-01 + -4.471591591812127e-01 + -4.466652837454304e-01 + -4.461716431800687e-01 + -4.456782399407630e-01 + -4.451850764283579e-01 + -4.446921550770006e-01 + -4.441994783152980e-01 + -4.437070485743053e-01 + -4.432148683148817e-01 + -4.427229399164072e-01 + -4.422312656936392e-01 + -4.417398479943503e-01 + -4.412486891611525e-01 + -4.407577915365379e-01 + -4.402671575060104e-01 + -4.397767893532367e-01 + -4.392866892910770e-01 + -4.387968597340107e-01 + -4.383073029681190e-01 + -4.378180210879689e-01 + -4.373290163087382e-01 + -4.368402909924953e-01 + -4.363518475687816e-01 + -4.358636880897754e-01 + -4.353758146064535e-01 + -4.348882293452391e-01 + -4.344009344375495e-01 + -4.339139321041776e-01 + -4.334272247184532e-01 + -4.329408142509990e-01 + -4.324547026649819e-01 + -4.319688922210647e-01 + -4.314833850702666e-01 + -4.309981832317592e-01 + -4.305132886299618e-01 + -4.300287034407881e-01 + -4.295444298858639e-01 + -4.290604699557538e-01 + -4.285768254794737e-01 + -4.280934983815344e-01 + -4.276104909880938e-01 + -4.271278053251724e-01 + -4.266454431999359e-01 + -4.261634065006114e-01 + -4.256816972235624e-01 + -4.252003173934318e-01 + -4.247192689049444e-01 + -4.242385536499166e-01 + -4.237581735472888e-01 + -4.232781305399061e-01 + -4.227984264773803e-01 + -4.223190631457318e-01 + -4.218400424415704e-01 + -4.213613663598799e-01 + -4.208830368540804e-01 + -4.204050553934426e-01 + -4.199274237890383e-01 + -4.194501442515862e-01 + -4.189732183034799e-01 + -4.184966476302857e-01 + -4.180204343198091e-01 + -4.175445800025869e-01 + -4.170690862458790e-01 + -4.165939547960728e-01 + -4.161191875920802e-01 + -4.156447863485650e-01 + -4.151707524172177e-01 + -4.146970878143395e-01 + -4.142237944432959e-01 + -4.137508735370036e-01 + -4.132783267358912e-01 + -4.128061559031381e-01 + -4.123343628419361e-01 + -4.118629489726368e-01 + -4.113919157251955e-01 + -4.109212649651476e-01 + -4.104509983224866e-01 + -4.099811172669397e-01 + -4.095116233202941e-01 + -4.090425181490535e-01 + -4.085738034100841e-01 + -4.081054804950061e-01 + -4.076375508831032e-01 + -4.071700161671795e-01 + -4.067028779692610e-01 + -4.062361376606066e-01 + -4.057697965353959e-01 + -4.053038563335837e-01 + -4.048383186645677e-01 + -4.043731849141807e-01 + -4.039084564103662e-01 + -4.034441345521896e-01 + -4.029802208230617e-01 + -4.025167167043049e-01 + -4.020536236099443e-01 + -4.015909428779320e-01 + -4.011286758375757e-01 + -4.006668238996902e-01 + -4.002053885607212e-01 + -3.997443712200008e-01 + -3.992837731698375e-01 + -3.988235956310492e-01 + -3.983638399923307e-01 + -3.979045076012857e-01 + -3.974455996644362e-01 + -3.969871175556028e-01 + -3.965290626455606e-01 + -3.960714361699136e-01 + -3.956142393772600e-01 + -3.951574735515524e-01 + -3.947011399985758e-01 + -3.942452398354794e-01 + -3.937897742524872e-01 + -3.933347447660869e-01 + -3.928801524442024e-01 + -3.924259983101159e-01 + -3.919722839283850e-01 + -3.915190103656777e-01 + -3.910661784756895e-01 + -3.906137895655892e-01 + -3.901618450499643e-01 + -3.897103461604648e-01 + -3.892592935529919e-01 + -3.888086884121152e-01 + -3.883585322771145e-01 + -3.879088261485191e-01 + -3.874595710701922e-01 + -3.870107681556456e-01 + -3.865624182556531e-01 + -3.861145225228043e-01 + -3.856670823544472e-01 + -3.852200986473037e-01 + -3.847735723987576e-01 + -3.843275047747315e-01 + -3.838818965026468e-01 + -3.834367486164244e-01 + -3.829920625791138e-01 + -3.825478392535004e-01 + -3.821040794322394e-01 + -3.816607841213338e-01 + -3.812179544462010e-01 + -3.807755914230820e-01 + -3.803336958584798e-01 + -3.798922686597118e-01 + -3.794513108818827e-01 + -3.790108236907996e-01 + -3.785708078449803e-01 + -3.781312640889848e-01 + -3.776921934895628e-01 + -3.772535969260590e-01 + -3.768154752452403e-01 + -3.763778294637203e-01 + -3.759406605249527e-01 + -3.755039692439986e-01 + -3.750677562951532e-01 + -3.746320226741715e-01 + -3.741967693787305e-01 + -3.737619968727901e-01 + -3.733277061470673e-01 + -3.728938983949728e-01 + -3.724605741247380e-01 + -3.720277341192338e-01 + -3.715953793429491e-01 + -3.711635103186269e-01 + -3.707321278978636e-01 + -3.703012331728173e-01 + -3.698708267439747e-01 + -3.694409092657609e-01 + -3.690114815572269e-01 + -3.685825444178367e-01 + -3.681540985405766e-01 + -3.677261445488677e-01 + -3.672986832474148e-01 + -3.668717154245260e-01 + -3.664452417818624e-01 + -3.660192630178876e-01 + -3.655937798167319e-01 + -3.651687928442244e-01 + -3.647443027770453e-01 + -3.643203103057334e-01 + -3.638968161236534e-01 + -3.634738208501973e-01 + -3.630513251204181e-01 + -3.626293296340570e-01 + -3.622078349959966e-01 + -3.617868418102546e-01 + -3.613663507559756e-01 + -3.609463624258993e-01 + -3.605268773947004e-01 + -3.601078962972172e-01 + -3.596894197319259e-01 + -3.592714482724653e-01 + -3.588539824938156e-01 + -3.584370229705016e-01 + -3.580205702765974e-01 + -3.576046249835235e-01 + -3.571891876270012e-01 + -3.567742587321382e-01 + -3.563598388637925e-01 + -3.559459285557617e-01 + -3.555325283179373e-01 + -3.551196386794110e-01 + -3.547072601423791e-01 + -3.542953931967586e-01 + -3.538840383835105e-01 + -3.534731961849680e-01 + -3.530628670336623e-01 + -3.526530514374770e-01 + -3.522437498876279e-01 + -3.518349628366106e-01 + -3.514266907403692e-01 + -3.510189340474827e-01 + -3.506116931966672e-01 + -3.502049686305037e-01 + -3.497987607886671e-01 + -3.493930701015490e-01 + -3.489878969787067e-01 + -3.485832418243141e-01 + -3.481791050461803e-01 + -3.477754870557839e-01 + -3.473723882547367e-01 + -3.469698090253131e-01 + -3.465677497415546e-01 + -3.461662107762984e-01 + -3.457651925045610e-01 + -3.453646952920019e-01 + -3.449647194961470e-01 + -3.445652654674211e-01 + -3.441663335489293e-01 + -3.437679240760669e-01 + -3.433700373751777e-01 + -3.429726737709668e-01 + -3.425758335909973e-01 + -3.421795171685511e-01 + -3.417837248082679e-01 + -3.413884567949557e-01 + -3.409937134173290e-01 + -3.405994949798623e-01 + -3.402058017865278e-01 + -3.398126341027504e-01 + -3.394199921921239e-01 + -3.390278763247643e-01 + -3.386362867748747e-01 + -3.382452237948665e-01 + -3.378546876227794e-01 + -3.374646785249709e-01 + -3.370751967392704e-01 + -3.366862424749215e-01 + -3.362978159833941e-01 + -3.359099174981644e-01 + -3.355225472182573e-01 + -3.351357053498810e-01 + -3.347493921057998e-01 + -3.343636076973308e-01 + -3.339783523006465e-01 + -3.335936261089775e-01 + -3.332094293460974e-01 + -3.328257621547543e-01 + -3.324426246881435e-01 + -3.320600171619409e-01 + -3.316779397280902e-01 + -3.312963925265998e-01 + -3.309153757279716e-01 + -3.305348894653861e-01 + -3.301549338737799e-01 + -3.297755091265357e-01 + -3.293966153413448e-01 + -3.290182526215280e-01 + -3.286404211114115e-01 + -3.282631209285871e-01 + -3.278863521751846e-01 + -3.275101149642331e-01 + -3.271344093953196e-01 + -3.267592355617253e-01 + -3.263845935667566e-01 + -3.260104834866909e-01 + -3.256369053882101e-01 + -3.252638593756906e-01 + -3.248913455268053e-01 + -3.245193638907094e-01 + -3.241479145199382e-01 + -3.237769974872735e-01 + -3.234066128661293e-01 + -3.230367606622389e-01 + -3.226674409186036e-01 + -3.222986537192250e-01 + -3.219303990783447e-01 + -3.215626770146125e-01 + -3.211954875680534e-01 + -3.208288307426270e-01 + -3.204627065455645e-01 + -3.200971150001409e-01 + -3.197320561103191e-01 + -3.193675298795836e-01 + -3.190035363166080e-01 + -3.186400754009213e-01 + -3.182771471114187e-01 + -3.179147514432106e-01 + -3.175528883799994e-01 + -3.171915578921511e-01 + -3.168307599394173e-01 + -3.164704945026028e-01 + -3.161107615528752e-01 + -3.157515610223091e-01 + -3.153928928534279e-01 + -3.150347570035595e-01 + -3.146771534412893e-01 + -3.143200821026652e-01 + -3.139635429048980e-01 + -3.136075357713601e-01 + -3.132520606345795e-01 + -3.128971174301287e-01 + -3.125427060850263e-01 + -3.121888265003789e-01 + -3.118354785699559e-01 + -3.114826622209773e-01 + -3.111303773636806e-01 + -3.107786238863177e-01 + -3.104274016699458e-01 + -3.100767106006651e-01 + -3.097265505731036e-01 + -3.093769214901858e-01 + -3.090278232197667e-01 + -3.086792556057311e-01 + -3.083312185528201e-01 + -3.079837119302323e-01 + -3.076367355606988e-01 + -3.072902893338402e-01 + -3.069443731087555e-01 + -3.065989866918071e-01 + -3.062541299629242e-01 + -3.059098027895394e-01 + -3.055660049841727e-01 + -3.052227363624680e-01 + -3.048799967605036e-01 + -3.045377860349815e-01 + -3.041961039937703e-01 + -3.038549504461246e-01 + -3.035143252370935e-01 + -3.031742281687436e-01 + -3.028346590287212e-01 + -3.024956176228422e-01 + -3.021571037727226e-01 + -3.018191172891879e-01 + -3.014816579447884e-01 + -3.011447255196590e-01 + -3.008083198102758e-01 + -3.004724406299011e-01 + -3.001370877400161e-01 + -2.998022608844934e-01 + -2.994679598576119e-01 + -2.991341844426006e-01 + -2.988009344054493e-01 + -2.984682095047237e-01 + -2.981360094999961e-01 + -2.978043341494513e-01 + -2.974731831997393e-01 + -2.971425563888731e-01 + -2.968124534589884e-01 + -2.964828741880918e-01 + -2.961538183196636e-01 + -2.958252855581221e-01 + -2.954972756291568e-01 + -2.951697882690829e-01 + -2.948428232168150e-01 + -2.945163801997789e-01 + -2.941904589349781e-01 + -2.938650591314966e-01 + -2.935401805013078e-01 + -2.932158227514122e-01 + -2.928919855851135e-01 + -2.925686687341852e-01 + -2.922458719001491e-01 + -2.919235947316832e-01 + -2.916018369508684e-01 + -2.912805982685433e-01 + -2.909598783275820e-01 + -2.906396768390501e-01 + -2.903199935181080e-01 + -2.900008280188033e-01 + -2.896821799916231e-01 + -2.893640491090957e-01 + -2.890464350819901e-01 + -2.887293375718574e-01 + -2.884127562142981e-01 + -2.880966906629407e-01 + -2.877811406014185e-01 + -2.874661057081589e-01 + -2.871515856017081e-01 + -2.868375799224188e-01 + -2.865240883310016e-01 + -2.862111104788733e-01 + -2.858986460098822e-01 + -2.855866945593247e-01 + -2.852752557499685e-01 + -2.849643292069009e-01 + -2.846539145659956e-01 + -2.843440114851778e-01 + -2.840346195818731e-01 + -2.837257384382189e-01 + -2.834173676812886e-01 + -2.831095069534547e-01 + -2.828021558855677e-01 + -2.824953140398479e-01 + -2.821889810170294e-01 + -2.818831564761255e-01 + -2.815778399984394e-01 + -2.812730311718642e-01 + -2.809687296216254e-01 + -2.806649349155874e-01 + -2.803616466281044e-01 + -2.800588643758425e-01 + -2.797565877594176e-01 + -2.794548163592918e-01 + -2.791535497376408e-01 + -2.788527874670974e-01 + -2.785525291296920e-01 + -2.782527743138438e-01 + -2.779535226034689e-01 + -2.776547735631448e-01 + -2.773565267276963e-01 + -2.770587816777879e-01 + -2.767615379983304e-01 + -2.764647952191276e-01 + -2.761685529026718e-01 + -2.758728106207560e-01 + -2.755775679024411e-01 + -2.752828243050798e-01 + -2.749885793975552e-01 + -2.746948327088052e-01 + -2.744015837838452e-01 + -2.741088321749212e-01 + -2.738165773969221e-01 + -2.735248189792139e-01 + -2.732335564721672e-01 + -2.729427894286903e-01 + -2.726525173657923e-01 + -2.723627397775934e-01 + -2.720734562117226e-01 + -2.717846661993187e-01 + -2.714963692431568e-01 + -2.712085648712142e-01 + -2.709212526021956e-01 + -2.706344319386959e-01 + -2.703481024077038e-01 + -2.700622635195931e-01 + -2.697769147573882e-01 + -2.694920556414950e-01 + -2.692076856844960e-01 + -2.689238043678598e-01 + -2.686404111894522e-01 + -2.683575056539274e-01 + -2.680750872618156e-01 + -2.677931554956217e-01 + -2.675117098420171e-01 + -2.672307498062658e-01 + -2.669502748541435e-01 + -2.666702844564393e-01 + -2.663907781339158e-01 + -2.661117553567487e-01 + -2.658332155842312e-01 + -2.655551583220328e-01 + -2.652775830228958e-01 + -2.650004891310402e-01 + -2.647238761635896e-01 + -2.644477435849875e-01 + -2.641720908275026e-01 + -2.638969173638899e-01 + -2.636222226679477e-01 + -2.633480062028563e-01 + -2.630742674151484e-01 + -2.628010057675256e-01 + -2.625282207270022e-01 + -2.622559117157733e-01 + -2.619840781930577e-01 + -2.617127196438671e-01 + -2.614418354712815e-01 + -2.611714251171109e-01 + -2.609014880694051e-01 + -2.606320237300626e-01 + -2.603630315312080e-01 + -2.600945109576975e-01 + -2.598264614192900e-01 + -2.595588823272051e-01 + -2.592917731268140e-01 + -2.590251332515323e-01 + -2.587589621335352e-01 + -2.584932592080277e-01 + -2.582280238899204e-01 + -2.579632555996659e-01 + -2.576989537753064e-01 + -2.574351178135512e-01 + -2.571717471205571e-01 + -2.569088411498421e-01 + -2.566463993032694e-01 + -2.563844209736399e-01 + -2.561229055922761e-01 + -2.558618525664796e-01 + -2.556012613001373e-01 + -2.553411312228936e-01 + -2.550814617199114e-01 + -2.548222521633088e-01 + -2.545635019702024e-01 + -2.543052105636615e-01 + -2.540473773489130e-01 + -2.537900016856341e-01 + -2.535330829668063e-01 + -2.532766206092137e-01 + -2.530206140025920e-01 + -2.527650625373169e-01 + -2.525099656058339e-01 + -2.522553225882911e-01 + -2.520011328519368e-01 + -2.517473957689335e-01 + -2.514941107682424e-01 + -2.512412772287477e-01 + -2.509888944763962e-01 + -2.507369619085763e-01 + -2.504854789235316e-01 + -2.502344448942996e-01 + -2.499838591881081e-01 + -2.497337211675947e-01 + -2.494840301941274e-01 + -2.492347856449565e-01 + -2.489859868896926e-01 + -2.487376332816026e-01 + -2.484897241934407e-01 + -2.482422589923619e-01 + -2.479952370263670e-01 + -2.477486576676466e-01 + -2.475025202765806e-01 + -2.472568241776227e-01 + -2.470115687463755e-01 + -2.467667533575219e-01 + -2.465223773319763e-01 + -2.462784400223504e-01 + -2.460349407926405e-01 + -2.457918789841980e-01 + -2.455492539346369e-01 + -2.453070649885519e-01 + -2.450653115068263e-01 + -2.448239928212929e-01 + -2.445831082525867e-01 + -2.443426571517872e-01 + -2.441026388729581e-01 + -2.438630527595843e-01 + -2.436238981283992e-01 + -2.433851743000419e-01 + -2.431468806082007e-01 + -2.429090164035606e-01 + -2.426715810065056e-01 + -2.424345737214870e-01 + -2.421979939156839e-01 + -2.419618409122474e-01 + -2.417261139893861e-01 + -2.414908125047008e-01 + -2.412559357926221e-01 + -2.410214831428485e-01 + -2.407874538901436e-01 + -2.405538473594189e-01 + -2.403206628448788e-01 + -2.400878996494245e-01 + -2.398555571012901e-01 + -2.396236345489386e-01 + -2.393921312730178e-01 + -2.391610465589564e-01 + -2.389303797400675e-01 + -2.387001301206918e-01 + -2.384702970040557e-01 + -2.382408797142012e-01 + -2.380118775445049e-01 + -2.377832897784107e-01 + -2.375551157141706e-01 + -2.373273546676602e-01 + -2.371000059435911e-01 + -2.368730688080822e-01 + -2.366465425799626e-01 + -2.364204265795865e-01 + -2.361947200488833e-01 + -2.359694222928865e-01 + -2.357445326362645e-01 + -2.355200503199650e-01 + -2.352959746331166e-01 + -2.350723049000333e-01 + -2.348490404054174e-01 + -2.346261804119323e-01 + -2.344037241869703e-01 + -2.341816710532240e-01 + -2.339600202844186e-01 + -2.337387711161095e-01 + -2.335179228570654e-01 + -2.332974748051559e-01 + -2.330774262250223e-01 + -2.328577763695832e-01 + -2.326385245095027e-01 + -2.324196699339893e-01 + -2.322012119071908e-01 + -2.319831496985562e-01 + -2.317654825913850e-01 + -2.315482098501277e-01 + -2.313313307350805e-01 + -2.311148445095759e-01 + -2.308987504304352e-01 + -2.306830477605995e-01 + -2.304677357746768e-01 + -2.302528137313455e-01 + -2.300382808821649e-01 + -2.298241364802842e-01 + -2.296103797849810e-01 + -2.293970100645507e-01 + -2.291840265929282e-01 + -2.289714285853380e-01 + -2.287592152687787e-01 + -2.285473859575702e-01 + -2.283359398767679e-01 + -2.281248762354934e-01 + -2.279141943457151e-01 + -2.277038934347241e-01 + -2.274939727028900e-01 + -2.272844314557521e-01 + -2.270752689260884e-01 + -2.268664843063303e-01 + -2.266580768713468e-01 + -2.264500458690802e-01 + -2.262423905203003e-01 + -2.260351100742427e-01 + -2.258282037663114e-01 + -2.256216708189812e-01 + -2.254155104759721e-01 + -2.252097219699820e-01 + -2.250043045201664e-01 + -2.247992573593203e-01 + -2.245945797283117e-01 + -2.243902708632074e-01 + -2.241863299547213e-01 + -2.239827562297785e-01 + -2.237795489631809e-01 + -2.235767073479459e-01 + -2.233742305913306e-01 + -2.231721179508258e-01 + -2.229703686190215e-01 + -2.227689817967141e-01 + -2.225679567331226e-01 + -2.223672926389324e-01 + -2.221669887137792e-01 + -2.219670441735496e-01 + -2.217674582593449e-01 + -2.215682301863681e-01 + -2.213693591068690e-01 + -2.211708442449674e-01 + -2.209726848493868e-01 + -2.207748801280919e-01 + -2.205774292618204e-01 + -2.203803314334422e-01 + -2.201835858632719e-01 + -2.199871917828554e-01 + -2.197911484006530e-01 + -2.195954548584606e-01 + -2.194001103587555e-01 + -2.192051141421317e-01 + -2.190104654040938e-01 + -2.188161633272981e-01 + -2.186222070944807e-01 + -2.184285959003400e-01 + -2.182353289399314e-01 + -2.180424054040691e-01 + -2.178498244760091e-01 + -2.176575853577229e-01 + -2.174656872563868e-01 + -2.172741293153170e-01 + -2.170829107184905e-01 + -2.168920306960583e-01 + -2.167014883907840e-01 + -2.165112829716549e-01 + -2.163214136638050e-01 + -2.161318796380734e-01 + -2.159426800456078e-01 + -2.157538140457635e-01 + -2.155652808465100e-01 + -2.153770796378261e-01 + -2.151892095617934e-01 + -2.150016698096599e-01 + -2.148144595574814e-01 + -2.146275779264766e-01 + -2.144410241083473e-01 + -2.142547972973645e-01 + -2.140688966286696e-01 + -2.138833212713278e-01 + -2.136980704068839e-01 + -2.135131431962469e-01 + -2.133285387811797e-01 + -2.131442563097479e-01 + -2.129602949644751e-01 + -2.127766538888946e-01 + -2.125933322205115e-01 + -2.124103291538361e-01 + -2.122276438425608e-01 + -2.120452754131807e-01 + -2.118632230196053e-01 + -2.116814858091140e-01 + -2.115000629262062e-01 + -2.113189535366684e-01 + -2.111381567835468e-01 + -2.109576717897184e-01 + -2.107774976942802e-01 + -2.105976336477213e-01 + -2.104180788038142e-01 + -2.102388322993181e-01 + -2.100598932684242e-01 + -2.098812608470688e-01 + -2.097029341689293e-01 + -2.095249123662445e-01 + -2.093471945702010e-01 + -2.091697799107427e-01 + -2.089926675162990e-01 + -2.088158565139439e-01 + -2.086393460309292e-01 + -2.084631351904399e-01 + -2.082872231124364e-01 + -2.081116089361562e-01 + -2.079362917906518e-01 + -2.077612707758083e-01 + -2.075865450013143e-01 + -2.074121135936494e-01 + -2.072379756933602e-01 + -2.070641303864229e-01 + -2.068905767613353e-01 + -2.067173139701657e-01 + -2.065443411403108e-01 + -2.063716573678470e-01 + -2.061992617242399e-01 + -2.060271533191409e-01 + -2.058553312792487e-01 + -2.056837947057731e-01 + -2.055125426973447e-01 + -2.053415743575506e-01 + -2.051708887985432e-01 + -2.050004850901530e-01 + -2.048303622987561e-01 + -2.046605195888223e-01 + -2.044909560501713e-01 + -2.043216707137636e-01 + -2.041526627003059e-01 + -2.039839311053935e-01 + -2.038154749924674e-01 + -2.036472934779214e-01 + -2.034793856435669e-01 + -2.033117505334730e-01 + -2.031443872603400e-01 + -2.029772949053144e-01 + -2.028104725040109e-01 + -2.026439191805672e-01 + -2.024776340229360e-01 + -2.023116160442759e-01 + -2.021458643362796e-01 + -2.019803779931268e-01 + -2.018151560674621e-01 + -2.016501976152565e-01 + -2.014855017030598e-01 + -2.013210674074579e-01 + -2.011568937830132e-01 + -2.009929798869589e-01 + -2.008293247967324e-01 + -2.006659275530022e-01 + -2.005027871933542e-01 + -2.003399027889975e-01 + -2.001772733993469e-01 + -2.000148980648089e-01 + -1.998527758076494e-01 + -1.996909056937942e-01 + -1.995292867991256e-01 + -1.993679181495860e-01 + -1.992067987609143e-01 + -1.990459276660739e-01 + -1.988853039482369e-01 + -1.987249266385540e-01 + -1.985647947339026e-01 + -1.984049072747179e-01 + -1.982452632992336e-01 + -1.980858618372365e-01 + -1.979267019229292e-01 + -1.977677825816225e-01 + -1.976091028269504e-01 + -1.974506616652059e-01 + -1.972924581269546e-01 + -1.971344912595772e-01 + -1.969767600601777e-01 + -1.968192635197257e-01 + -1.966620006473419e-01 + -1.965049704914111e-01 + -1.963481720619957e-01 + -1.961916043159508e-01 + -1.960352662866058e-01 + -1.958791570023236e-01 + -1.957232754408347e-01 + -1.955676205736371e-01 + -1.954121914050787e-01 + -1.952569869811404e-01 + -1.951020062511957e-01 + -1.949472481743376e-01 + -1.947927118000400e-01 + -1.946383960967118e-01 + -1.944843000224015e-01 + -1.943304226010136e-01 + -1.941767627996486e-01 + -1.940233195719773e-01 + -1.938700919198527e-01 + -1.937170788011597e-01 + -1.935642791691659e-01 + -1.934116920404960e-01 + -1.932593163702813e-01 + -1.931071510855982e-01 + -1.929551951739442e-01 + -1.928034476111422e-01 + -1.926519073523142e-01 + -1.925005733427448e-01 + -1.923494445333268e-01 + -1.921985198850023e-01 + -1.920477983694047e-01 + -1.918972789099489e-01 + -1.917469604078139e-01 + -1.915968418647559e-01 + -1.914469222323946e-01 + -1.912972003977130e-01 + -1.911476753153419e-01 + -1.909983459354311e-01 + -1.908492111828614e-01 + -1.907002699990133e-01 + -1.905515213021572e-01 + -1.904029639829450e-01 + -1.902545969863778e-01 + -1.901064192518998e-01 + -1.899584296832387e-01 + -1.898106271909556e-01 + -1.896630106872466e-01 + -1.895155790805884e-01 + -1.893683312769490e-01 + -1.892212661839438e-01 + -1.890743827110565e-01 + -1.889276797429612e-01 + -1.887811561730439e-01 + -1.886348109311961e-01 + -1.884886428944042e-01 + -1.883426509295119e-01 + -1.881968339481026e-01 + -1.880511908244114e-01 + -1.879057204210790e-01 + -1.877604216379617e-01 + -1.876152933660684e-01 + -1.874703344759240e-01 + -1.873255438107255e-01 + -1.871809202288037e-01 + -1.870364626111082e-01 + -1.868921698587295e-01 + -1.867480408184902e-01 + -1.866040743068453e-01 + -1.864602692045282e-01 + -1.863166243684211e-01 + -1.861731386269271e-01 + -1.860298108368848e-01 + -1.858866398430710e-01 + -1.857436244746902e-01 + -1.856007635808946e-01 + -1.854580560069532e-01 + -1.853155005825300e-01 + -1.851730961153894e-01 + -1.850308414342497e-01 + -1.848887353926573e-01 + -1.847467767859671e-01 + -1.846049644137440e-01 + -1.844632971091011e-01 + -1.843217736893654e-01 + -1.841803929497542e-01 + -1.840391536710132e-01 + -1.838980546953601e-01 + -1.837570948428414e-01 + -1.836162728527662e-01 + -1.834755875142904e-01 + -1.833350376404768e-01 + -1.831946220322505e-01 + -1.830543394615971e-01 + -1.829141886951903e-01 + -1.827741685199023e-01 + -1.826342777082210e-01 + -1.824945150204152e-01 + -1.823548792146146e-01 + -1.822153690716325e-01 + -1.820759833652730e-01 + -1.819367208114371e-01 + -1.817975801678411e-01 + -1.816585602158104e-01 + -1.815196596908165e-01 + -1.813808773333636e-01 + -1.812422118861797e-01 + -1.811036620617022e-01 + -1.809652266051679e-01 + -1.808269042761388e-01 + -1.806886937510042e-01 + -1.805505937426575e-01 + -1.804126030135119e-01 + -1.802747202867512e-01 + -1.801369442562599e-01 + -1.799992736029428e-01 + -1.798617070428147e-01 + -1.797242432859927e-01 + -1.795868810229852e-01 + -1.794496189531687e-01 + -1.793124557648680e-01 + -1.791753901302353e-01 + -1.790384207459855e-01 + -1.789015462957270e-01 + -1.787647654308534e-01 + -1.786280768266134e-01 + -1.784914791546937e-01 + -1.783549710639760e-01 + -1.782185512276328e-01 + -1.780822183108111e-01 + -1.779459709418989e-01 + -1.778098077766785e-01 + -1.776737274641330e-01 + -1.775377286080068e-01 + -1.774018098685905e-01 + -1.772659699092733e-01 + -1.771302073149079e-01 + -1.769945207098710e-01 + -1.768589087303928e-01 + -1.767233699571559e-01 + -1.765879030229077e-01 + -1.764525065751527e-01 + -1.763171791703831e-01 + -1.761819193770276e-01 + -1.760467257991546e-01 + -1.759115970764966e-01 + -1.757765317826701e-01 + -1.756415284467932e-01 + -1.755065856823099e-01 + -1.753717020824757e-01 + -1.752368761936218e-01 + -1.751021065398607e-01 + -1.749673916979465e-01 + -1.748327302843573e-01 + -1.746981207950505e-01 + -1.745635617515470e-01 + -1.744290517410085e-01 + -1.742945892830799e-01 + -1.741601728915786e-01 + -1.740258011048612e-01 + -1.738914724532109e-01 + -1.737571854631234e-01 + -1.736229386541751e-01 + -1.734887304969607e-01 + -1.733545594900176e-01 + -1.732204242005106e-01 + -1.730863230847530e-01 + -1.729522545880760e-01 + -1.728182172369972e-01 + -1.726842095169024e-01 + -1.725502298924380e-01 + -1.724162768405715e-01 + -1.722823488297978e-01 + -1.721484443102564e-01 + -1.720145617082366e-01 + -1.718806995062888e-01 + -1.717468561876117e-01 + -1.716130301380271e-01 + -1.714792197950507e-01 + -1.713454236195253e-01 + -1.712116399980747e-01 + -1.710778673588311e-01 + -1.709441041583024e-01 + -1.708103487974718e-01 + -1.706765996579921e-01 + -1.705428551238503e-01 + -1.704091136122197e-01 + -1.702753735243250e-01 + -1.701416332337174e-01 + -1.700078910979654e-01 + -1.698741454939726e-01 + -1.697403948188084e-01 + -1.696066374424440e-01 + -1.694728717024124e-01 + -1.693390959182572e-01 + -1.692053084687103e-01 + -1.690715077127793e-01 + -1.689376919605925e-01 + -1.688038595533757e-01 + -1.686700088249354e-01 + -1.685361380772680e-01 + -1.684022456043037e-01 + -1.682683297240474e-01 + -1.681343887869522e-01 + -1.680004210524648e-01 + -1.678664247764668e-01 + -1.677323982879569e-01 + -1.675983398888628e-01 + -1.674642478421112e-01 + -1.673301203727337e-01 + -1.671959557470875e-01 + -1.670617522514179e-01 + -1.669275081523977e-01 + -1.667932216748827e-01 + -1.666588910302535e-01 + -1.665245144618999e-01 + -1.663900902261115e-01 + -1.662556165573334e-01 + -1.661210916150787e-01 + -1.659865136192500e-01 + -1.658518808205523e-01 + -1.657171913793817e-01 + -1.655824434869752e-01 + -1.654476353589082e-01 + -1.653127651402884e-01 + -1.651778310039089e-01 + -1.650428311475997e-01 + -1.649077636947325e-01 + -1.647726268054878e-01 + -1.646374186795111e-01 + -1.645021374143022e-01 + -1.643667811219276e-01 + -1.642313479645375e-01 + -1.640958360797542e-01 + -1.639602435821208e-01 + -1.638245685678920e-01 + -1.636888091342957e-01 + -1.635529633807896e-01 + -1.634170294056715e-01 + -1.632810052843662e-01 + -1.631448890987297e-01 + -1.630086789505001e-01 + -1.628723728914803e-01 + -1.627359689604526e-01 + -1.625994652175857e-01 + -1.624628597296842e-01 + -1.623261505411777e-01 + -1.621893356498993e-01 + -1.620524130844338e-01 + -1.619153808874262e-01 + -1.617782370829140e-01 + -1.616409796594323e-01 + -1.615036066042981e-01 + -1.613661159511956e-01 + -1.612285056749289e-01 + -1.610907737177844e-01 + -1.609529180644884e-01 + -1.608149367058714e-01 + -1.606768276166446e-01 + -1.605385887249777e-01 + -1.604002179666082e-01 + -1.602617132920370e-01 + -1.601230726507729e-01 + -1.599842939686116e-01 + -1.598453751511857e-01 + -1.597063141136520e-01 + -1.595671087574746e-01 + -1.594277569704320e-01 + -1.592882566576958e-01 + -1.591486057239582e-01 + -1.590088020577491e-01 + -1.588688434990599e-01 + -1.587287278927537e-01 + -1.585884531125515e-01 + -1.584480170419727e-01 + -1.583074175205078e-01 + -1.581666523320080e-01 + -1.580257193426469e-01 + -1.578846164090064e-01 + -1.577433413163997e-01 + -1.576018918624315e-01 + -1.574602658488709e-01 + -1.573184610699215e-01 + -1.571764753211041e-01 + -1.570343063929105e-01 + -1.568919520623428e-01 + -1.567494100904384e-01 + -1.566066782348808e-01 + -1.564637542612438e-01 + -1.563206358983480e-01 + -1.561773208679784e-01 + -1.560338069324230e-01 + -1.558900918260982e-01 + -1.557461732592160e-01 + -1.556020489404925e-01 + -1.554577165574562e-01 + -1.553131737930803e-01 + -1.551684183667380e-01 + -1.550234479797016e-01 + -1.548782603038531e-01 + -1.547328529851514e-01 + -1.545872236759721e-01 + -1.544413700402710e-01 + -1.542952897416497e-01 + -1.541489804232354e-01 + -1.540024397056934e-01 + -1.538556652055174e-01 + -1.537086545589509e-01 + -1.535614054107616e-01 + -1.534139153265976e-01 + -1.532661818894184e-01 + -1.531182027303390e-01 + -1.529699754364569e-01 + -1.528214975805129e-01 + -1.526727667337492e-01 + -1.525237804315604e-01 + -1.523745362382714e-01 + -1.522250317710374e-01 + -1.520752645246755e-01 + -1.519252320010430e-01 + -1.517749318043420e-01 + -1.516243614355856e-01 + -1.514735183783821e-01 + -1.513224001894709e-01 + -1.511710043669709e-01 + -1.510193283881065e-01 + -1.508673697644896e-01 + -1.507151259726486e-01 + -1.505625944726656e-01 + -1.504097727456887e-01 + -1.502566582737337e-01 + -1.501032485283334e-01 + -1.499495409558064e-01 + -1.497955329817901e-01 + -1.496412220312461e-01 + -1.494866055649385e-01 + -1.493316810218445e-01 + -1.491764458131949e-01 + -1.490208973395277e-01 + -1.488650330073115e-01 + -1.487088502321930e-01 + -1.485523464347014e-01 + -1.483955190026080e-01 + -1.482383652905619e-01 + -1.480808826638546e-01 + -1.479230685198502e-01 + -1.477649202713839e-01 + -1.476064352431592e-01 + -1.474476107543291e-01 + -1.472884441602761e-01 + -1.471289328556787e-01 + -1.469690741904771e-01 + -1.468088654425319e-01 + -1.466483039513407e-01 + -1.464873870643751e-01 + -1.463261120992439e-01 + -1.461644763685366e-01 + -1.460024771711030e-01 + -1.458401117873900e-01 + -1.456773775256328e-01 + -1.455142717039091e-01 + -1.453507916253095e-01 + -1.451869345373153e-01 + -1.450226976893540e-01 + -1.448580783981553e-01 + -1.446930739597145e-01 + -1.445276816274984e-01 + -1.443618986028185e-01 + -1.441957221635531e-01 + -1.440291496154044e-01 + -1.438621781832398e-01 + -1.436948050802729e-01 + -1.435270275400491e-01 + -1.433588428498931e-01 + -1.431902482406432e-01 + -1.430212409022274e-01 + -1.428518180756519e-01 + -1.426819769869911e-01 + -1.425117148424586e-01 + -1.423410288662751e-01 + -1.421699162875700e-01 + -1.419983743276740e-01 + -1.418264001727286e-01 + -1.416539910114460e-01 + -1.414811440470098e-01 + -1.413078564852965e-01 + -1.411341255423312e-01 + -1.409599484327010e-01 + -1.407853222965617e-01 + -1.406102443173973e-01 + -1.404347117561954e-01 + -1.402587217519092e-01 + -1.400822714659246e-01 + -1.399053581557436e-01 + -1.397279789611117e-01 + -1.395501310267636e-01 + -1.393718115937786e-01 + -1.391930178402762e-01 + -1.390137469173205e-01 + -1.388339959972602e-01 + -1.386537622953096e-01 + -1.384730430031857e-01 + -1.382918352262208e-01 + -1.381101361784074e-01 + -1.379279430940128e-01 + -1.377452530885734e-01 + -1.375620633453186e-01 + -1.373783710804598e-01 + -1.371941734475252e-01 + -1.370094676273606e-01 + -1.368242508215203e-01 + -1.366385202106050e-01 + -1.364522729969404e-01 + -1.362655063841710e-01 + -1.360782175127518e-01 + -1.358904035964053e-01 + -1.357020618930480e-01 + -1.355131895302153e-01 + -1.353237837054462e-01 + -1.351338416922692e-01 + -1.349433606493001e-01 + -1.347523377766847e-01 + -1.345607703467432e-01 + -1.343686555800670e-01 + -1.341759906744941e-01 + -1.339827728348106e-01 + -1.337889993407819e-01 + -1.335946674413715e-01 + -1.333997743261954e-01 + -1.332043172962129e-01 + -1.330082936252972e-01 + -1.328117004983506e-01 + -1.326145352418741e-01 + -1.324167951779929e-01 + -1.322184775207451e-01 + -1.320195795498869e-01 + -1.318200985870884e-01 + -1.316200319613715e-01 + -1.314193769770063e-01 + -1.312181309395709e-01 + -1.310162911860090e-01 + -1.308138550466099e-01 + -1.306108198680959e-01 + -1.304071830473609e-01 + -1.302029419358534e-01 + -1.299980938748928e-01 + -1.297926362720222e-01 + -1.295865665431496e-01 + -1.293798820977739e-01 + -1.291725803345192e-01 + -1.289646586840201e-01 + -1.287561146028010e-01 + -1.285469455426532e-01 + -1.283371489741077e-01 + -1.281267223855238e-01 + -1.279156632656401e-01 + -1.277039691180485e-01 + -1.274916374652222e-01 + -1.272786658468367e-01 + -1.270650518164191e-01 + -1.268507929437779e-01 + -1.266358868257778e-01 + -1.264203310530910e-01 + -1.262041232080785e-01 + -1.259872609212462e-01 + -1.257697418614985e-01 + -1.255515637216261e-01 + -1.253327241582721e-01 + -1.251132208452764e-01 + -1.248930515084944e-01 + -1.246722139029198e-01 + -1.244507057917270e-01 + -1.242285249387353e-01 + -1.240056691417677e-01 + -1.237821362222557e-01 + -1.235579240163024e-01 + -1.233330303641722e-01 + -1.231074531412701e-01 + -1.228811902909497e-01 + -1.226542397328350e-01 + -1.224265993926709e-01 + -1.221982672554952e-01 + -1.219692413178169e-01 + -1.217395195880366e-01 + -1.215091001037845e-01 + -1.212779809466277e-01 + -1.210461602256936e-01 + -1.208136360512357e-01 + -1.205804065871132e-01 + -1.203464700244819e-01 + -1.201118245085727e-01 + -1.198764682774020e-01 + -1.196403996473917e-01 + -1.194036168896808e-01 + -1.191661183059585e-01 + -1.189279022478277e-01 + -1.186889671039581e-01 + -1.184493112863093e-01 + -1.182089332317922e-01 + -1.179678314226111e-01 + -1.177260043675167e-01 + -1.174834505981953e-01 + -1.172401686820595e-01 + -1.169961572351099e-01 + -1.167514149230550e-01 + -1.165059404117411e-01 + -1.162597324012717e-01 + -1.160127896494247e-01 + -1.157651109471359e-01 + -1.155166951180050e-01 + -1.152675410245315e-01 + -1.150176475647009e-01 + -1.147670136740031e-01 + -1.145156383362403e-01 + -1.142635205942373e-01 + -1.140106595167702e-01 + -1.137570541746248e-01 + -1.135027037005393e-01 + -1.132476072885310e-01 + -1.129917641930528e-01 + -1.127351736954119e-01 + -1.124778350964029e-01 + -1.122197477241451e-01 + -1.119609110122396e-01 + -1.117013244465126e-01 + -1.114409874638995e-01 + -1.111798996306716e-01 + -1.109180606010562e-01 + -1.106554699641756e-01 + -1.103921273860412e-01 + -1.101280326261970e-01 + -1.098631854961093e-01 + -1.095975858385727e-01 + -1.093312335328026e-01 + -1.090641285282797e-01 + -1.087962708201439e-01 + -1.085276604456853e-01 + -1.082582975002112e-01 + -1.079881821487157e-01 + -1.077173146235393e-01 + -1.074456951780328e-01 + -1.071733241222459e-01 + -1.069002018428373e-01 + -1.066263287812705e-01 + -1.063517054311524e-01 + -1.060763323447030e-01 + -1.058002101349244e-01 + -1.055233394662503e-01 + -1.052457210591484e-01 + -1.049673557111256e-01 + -1.046882442747131e-01 + -1.044083876525769e-01 + -1.041277868396362e-01 + -1.038464428925467e-01 + -1.035643569070283e-01 + -1.032815300276830e-01 + -1.029979634724506e-01 + -1.027136585692167e-01 + -1.024286166734134e-01 + -1.021428391952729e-01 + -1.018563276648054e-01 + -1.015690836461885e-01 + -1.012811087487812e-01 + -1.009924046817547e-01 + -1.007029732152899e-01 + -1.004128161832720e-01 + -1.001219355157012e-01 + -9.983033318947014e-02 + -9.953801123789381e-02 + -9.924497181065381e-02 + -9.895121710675164e-02 + -9.865674937610269e-02 + -9.836157098451072e-02 + -9.806568435550449e-02 + -9.776909196067303e-02 + -9.747179635418697e-02 + -9.717380018935569e-02 + -9.687510621844479e-02 + -9.657571724434684e-02 + -9.627563612429342e-02 + -9.597486578738405e-02 + -9.567340928055243e-02 + -9.537126972794066e-02 + -9.506845031433311e-02 + -9.476495428226560e-02 + -9.446078497156121e-02 + -9.415594584893169e-02 + -9.385044042616277e-02 + -9.354427227181475e-02 + -9.323744504129654e-02 + -9.292996250412121e-02 + -9.262182850819620e-02 + -9.231304696021701e-02 + -9.200362185853247e-02 + -9.169355729461479e-02 + -9.138285746240042e-02 + -9.107152658866502e-02 + -9.075956898041553e-02 + -9.044698911371001e-02 + -9.013379148226228e-02 + -8.981998063663986e-02 + -8.950556130228651e-02 + -8.919053824746566e-02 + -8.887491629233073e-02 + -8.855870039130662e-02 + -8.824189556852820e-02 + -8.792450692262105e-02 + -8.760653968435946e-02 + -8.728799913331863e-02 + -8.696889060279582e-02 + -8.664921956673162e-02 + -8.632899159487925e-02 + -8.600821233560418e-02 + -8.568688750540578e-02 + -8.536502290181033e-02 + -8.504262441718942e-02 + -8.471969805751253e-02 + -8.439624991228166e-02 + -8.407228614692197e-02 + -8.374781300392073e-02 + -8.342283682562317e-02 + -8.309736406563741e-02 + -8.277140122021633e-02 + -8.244495489588025e-02 + -8.211803185128563e-02 + -8.179063884910803e-02 + -8.146278271646745e-02 + -8.113447041766966e-02 + -8.080570906396743e-02 + -8.047650581443898e-02 + -8.014686781276095e-02 + -7.981680240792462e-02 + -7.948631704884632e-02 + -7.915541916537581e-02 + -7.882411634928695e-02 + -7.849241630024972e-02 + -7.816032674199919e-02 + -7.782785552351333e-02 + -7.749501058768483e-02 + -7.716179991262426e-02 + -7.682823159405887e-02 + -7.649431382724470e-02 + -7.616005485767063e-02 + -7.582546305035900e-02 + -7.549054686796573e-02 + -7.515531479419155e-02 + -7.481977542018518e-02 + -7.448393744841550e-02 + -7.414780964186224e-02 + -7.381140084542565e-02 + -7.347471999131863e-02 + -7.313777608894645e-02 + -7.280057821330421e-02 + -7.246313551410177e-02 + -7.212545725749871e-02 + -7.178755277686714e-02 + -7.144943146088469e-02 + -7.111110278446059e-02 + -7.077257629326426e-02 + -7.043386160590276e-02 + -7.009496843792977e-02 + -6.975590657859251e-02 + -6.941668588038064e-02 + -6.907731623736572e-02 + -6.873780762436528e-02 + -6.839817013263073e-02 + -6.805841390025939e-02 + -6.771854911509186e-02 + -6.737858603607710e-02 + -6.703853501790490e-02 + -6.669840646610939e-02 + -6.635821080020089e-02 + -6.601795854049251e-02 + -6.567766029345612e-02 + -6.533732672933422e-02 + -6.499696856244534e-02 + -6.465659655384894e-02 + -6.431622152906083e-02 + -6.397585437117004e-02 + -6.363550601749234e-02 + -6.329518746487584e-02 + -6.295490977261127e-02 + -6.261468405232740e-02 + -6.227452144919842e-02 + -6.193443314782195e-02 + -6.159443038613483e-02 + -6.125452449175782e-02 + -6.091472682299209e-02 + -6.057504875634485e-02 + -6.023550170757411e-02 + -5.989609715178513e-02 + -5.955684662483948e-02 + -5.921776169220818e-02 + -5.887885392543541e-02 + -5.854013491232313e-02 + -5.820161634720029e-02 + -5.786330995026137e-02 + -5.752522743033200e-02 + -5.718738052319666e-02 + -5.684978102238282e-02 + -5.651244078415603e-02 + -5.617537160914179e-02 + -5.583858533460867e-02 + -5.550209389764803e-02 + -5.516590918758542e-02 + -5.483004310132478e-02 + -5.449450760284144e-02 + -5.415931467224975e-02 + -5.382447627285256e-02 + -5.349000433169809e-02 + -5.315591089010736e-02 + -5.282220800586091e-02 + -5.248890761304951e-02 + -5.215602173570349e-02 + -5.182356244333215e-02 + -5.149154173522678e-02 + -5.115997163165661e-02 + -5.082886416999516e-02 + -5.049823136495267e-02 + -5.016808523392447e-02 + -4.983843779429676e-02 + -4.950930104255546e-02 + -4.918068697018742e-02 + -4.885260756307724e-02 + -4.852507478656667e-02 + -4.819810059193260e-02 + -4.787169691654435e-02 + -4.754587567726797e-02 + -4.722064876717863e-02 + -4.689602805511645e-02 + -4.657202539266345e-02 + -4.624865260469157e-02 + -4.592592148337583e-02 + -4.560384379218360e-02 + -4.528243126408893e-02 + -4.496169559628109e-02 + -4.464164845065374e-02 + -4.432230145577399e-02 + -4.400366620090135e-02 + -4.368575422918912e-02 + -4.336857704568598e-02 + -4.305214611378011e-02 + -4.273647284209545e-02 + -4.242156859535529e-02 + -4.210744469338823e-02 + -4.179411239819886e-02 + -4.148158291999365e-02 + -4.116986741259326e-02 + -4.085897697584740e-02 + -4.054892265423421e-02 + -4.023971542131075e-02 + -3.993136619382251e-02 + -3.962388583123071e-02 + -3.931728511726146e-02 + -3.901157477126550e-02 + -3.870676544906659e-02 + -3.840286772630143e-02 + -3.809989211312446e-02 + -3.779784905219891e-02 + -3.749674889469364e-02 + -3.719660192057603e-02 + -3.689741833904486e-02 + -3.659920826728452e-02 + -3.630198174507707e-02 + -3.600574873447752e-02 + -3.571051910326670e-02 + -3.541630263528623e-02 + -3.512310902944081e-02 + -3.483094789226868e-02 + -3.453982873906083e-02 + -3.424976098893085e-02 + -3.396075396966352e-02 + -3.367281691959212e-02 + -3.338595897696143e-02 + -3.310018917253339e-02 + -3.281551644593574e-02 + -3.253194963945627e-02 + -3.224949747985180e-02 + -3.196816859917648e-02 + -3.168797153104005e-02 + -3.140891468854379e-02 + -3.113100638362195e-02 + -3.085425482400445e-02 + -3.057866810270934e-02 + -3.030425420554928e-02 + -3.003102099769914e-02 + -2.975897623664901e-02 + -2.948812757423692e-02 + -2.921848253324668e-02 + -2.895004852283206e-02 + -2.868283284398848e-02 + -2.841684266935389e-02 + -2.815208505554401e-02 + -2.788856694668657e-02 + -2.762629515277662e-02 + -2.736527636581735e-02 + -2.710551716378724e-02 + -2.684702399109470e-02 + -2.658980317107453e-02 + -2.633386090723251e-02 + -2.607920326420626e-02 + -2.582583618939347e-02 + -2.557376551170278e-02 + -2.532299691452088e-02 + -2.507353595904179e-02 + -2.482538808499682e-02 + -2.457855859507147e-02 + -2.433305266394129e-02 + -2.408887533435539e-02 + -2.384603151841623e-02 + -2.360452600145211e-02 + -2.336436343201402e-02 + -2.312554832609387e-02 + -2.288808507226996e-02 + -2.265197791904223e-02 + -2.241723098404469e-02 + -2.218384825955722e-02 + -2.195183359578144e-02 + -2.172119070564114e-02 + -2.149192317903017e-02 + -2.126403446983029e-02 + -2.103752788800763e-02 + -2.081240661670658e-02 + -2.058867370373213e-02 + -2.036633206151460e-02 + -2.014538447470562e-02 + -1.992583358220780e-02 + -1.970768189350454e-02 + -1.949093179354288e-02 + -1.927558551890179e-02 + -1.906164517346932e-02 + -1.884911273803516e-02 + -1.863799005561765e-02 + -1.842827883084014e-02 + -1.821998063919436e-02 + -1.801309692347624e-02 + -1.780762899421212e-02 + -1.760357803216378e-02 + -1.740094507853177e-02 + -1.719973104881908e-02 + -1.699993673354589e-02 + -1.680156278321953e-02 + -1.660460972117107e-02 + -1.640907794332531e-02 + -1.621496771146485e-02 + -1.602227916433384e-02 + -1.583101231371427e-02 + -1.564116703862817e-02 + -1.545274309883748e-02 + -1.526574012772238e-02 + -1.508015761914186e-02 + -1.489599495303444e-02 + -1.471325139301425e-02 + -1.453192606709004e-02 + -1.435201798658663e-02 + -1.417352604127694e-02 + -1.399644899327932e-02 + -1.382078549302728e-02 + -1.364653407164367e-02 + -1.347369313289915e-02 + -1.330226097046918e-02 + -1.313223576283522e-02 + -1.296361556501821e-02 + -1.279639832327015e-02 + -1.263058187022890e-02 + -1.246616392163480e-02 + -1.230314208589229e-02 + -1.214151385550256e-02 + -1.198127661271531e-02 + -1.182242763705999e-02 + -1.166496409538715e-02 + -1.150888304589847e-02 + -1.135418144553720e-02 + -1.120085614521435e-02 + -1.104890388968280e-02 + -1.089832132231912e-02 + -1.074910498405016e-02 + -1.060125131793449e-02 + -1.045475666967276e-02 + -1.030961728045836e-02 + -1.016582929759894e-02 + -1.002338877483435e-02 + -9.882291666582974e-03 + -9.742533838318660e-03 + -9.604111064390042e-03 + -9.467019020603698e-03 + -9.331253297647066e-03 + -9.196809399717920e-03 + -9.063682741005771e-03 + -8.931868652392608e-03 + -8.801362376326071e-03 + -8.672159070075210e-03 + -8.544253813557785e-03 + -8.417641603575709e-03 + -8.292317355716863e-03 + -8.168275911736944e-03 + -8.045512032694060e-03 + -7.924020397696789e-03 + -7.803795618017998e-03 + -7.684832236645861e-03 + -7.567124714929550e-03 + -7.450667444285828e-03 + -7.335454748033539e-03 + -7.221480883382279e-03 + -7.108740043647531e-03 + -6.997226348639058e-03 + -6.886933855315346e-03 + -6.777856561420043e-03 + -6.669988397445708e-03 + -6.563323236817328e-03 + -6.457854897474280e-03 + -6.353577131920331e-03 + -6.250483639666226e-03 + -6.148568068731946e-03 + -6.047824005897748e-03 + -5.948244987599076e-03 + -5.849824501347084e-03 + -5.752555978513681e-03 + -5.656432807264865e-03 + -5.561448332218353e-03 + -5.467595840626229e-03 + -5.374868578311233e-03 + -5.283259752176146e-03 + -5.192762521793490e-03 + -5.103370006626977e-03 + -5.015075286629148e-03 + -4.927871401782054e-03 + -4.841751355904269e-03 + -4.756708115524279e-03 + -4.672734611342882e-03 + -4.589823741835811e-03 + -4.507968371897898e-03 + -4.427161334111035e-03 + -4.347395432360992e-03 + -4.268663440821394e-03 + -4.190958105123732e-03 + -4.114272145237943e-03 + -4.038598254543495e-03 + -3.963929102325757e-03 + -3.890257335680627e-03 + -3.817575578228647e-03 + -3.745876433060482e-03 + -3.675152484340320e-03 + -3.605396296371936e-03 + -3.536600416225055e-03 + -3.468757375181376e-03 + -3.401859688514519e-03 + -3.335899857279008e-03 + -3.270870369624132e-03 + -3.206763701552144e-03 + -3.143572317880366e-03 + -3.081288673370167e-03 + -3.019905214426235e-03 + -2.959414379161780e-03 + -2.899808598243805e-03 + -2.841080297295089e-03 + -2.783221897187762e-03 + -2.726225814481816e-03 + -2.670084462721576e-03 + -2.614790253659214e-03 + -2.560335598217736e-03 + -2.506712907773368e-03 + -2.453914594305456e-03 + -2.401933071402304e-03 + -2.350760756079694e-03 + -2.300390068609935e-03 + -2.250813433830373e-03 + -2.202023282926124e-03 + -2.154012052729510e-03 + -2.106772187286276e-03 + -2.060296139752280e-03 + -2.014576371403607e-03 + -1.969605353508076e-03 + -1.925375568654703e-03 + -1.881879509874747e-03 + -1.839109683164610e-03 + -1.797058608044933e-03 + -1.755718816706420e-03 + -1.715082857166331e-03 + -1.675143293280333e-03 + -1.635892703443056e-03 + -1.597323683697497e-03 + -1.559428848712492e-03 + -1.522200830321536e-03 + -1.485632280183934e-03 + -1.449715870092584e-03 + -1.414444291041649e-03 + -1.379810256474448e-03 + -1.345806501791486e-03 + -1.312425783716085e-03 + -1.279660883530597e-03 + -1.247504606026809e-03 + -1.215949779407020e-03 + -1.184989258561609e-03 + -1.154615923413642e-03 + -1.124822679454977e-03 + -1.095602461024569e-03 + -1.066948228352973e-03 + -1.038852968864840e-03 + -1.011309701783447e-03 + -9.843114734261671e-04 + -9.578513585924095e-04 + -9.319224654553773e-04 + -9.065179311555053e-04 + -8.816309237062864e-04 + -8.572546450286300e-04 + -8.333823272254078e-04 + -8.100072355517002e-04 + -7.871226705798197e-04 + -7.647219641139264e-04 + -7.427984826268421e-04 + -7.213456295816899e-04 + -7.003568409622573e-04 + -6.798255887472549e-04 + -6.597453829223955e-04 + -6.401097674232178e-04 + -6.209123240167792e-04 + -6.021466729184949e-04 + -5.838064694784330e-04 + -5.658854088227745e-04 + -5.483772251938055e-04 + -5.312756891491387e-04 + -5.145746125261390e-04 + -4.982678466736966e-04 + -4.823492804447342e-04 + -4.668128456938979e-04 + -4.516525141842837e-04 + -4.368622962674093e-04 + -4.224362466667431e-04 + -4.083684605074392e-04 + -3.946530726767535e-04 + -3.812842634726816e-04 + -3.682562540846601e-04 + -3.555633067280622e-04 + -3.431997298953713e-04 + -3.311598733734528e-04 + -3.194381292167991e-04 + -3.080289365424697e-04 + -2.969267761950205e-04 + -2.861261725181284e-04 + -2.756216974519896e-04 + -2.654079650780054e-04 + -2.554796342192304e-04 + -2.458314117831762e-04 + -2.364580472261255e-04 + -2.273543358783364e-04 + -2.185151214599497e-04 + -2.099352907310207e-04 + -2.016097775283348e-04 + -1.935335642058985e-04 + -1.857016766078902e-04 + -1.781091887621722e-04 + -1.707512232636694e-04 + -1.636229467286454e-04 + -1.567195750846747e-04 + -1.500363727286114e-04 + -1.435686485603325e-04 + -1.373117617901810e-04 + -1.312611200233150e-04 + -1.254121759173047e-04 + -1.197604333131590e-04 + -1.143014443478538e-04 + -1.090308067994204e-04 + -1.039441703359566e-04 + -9.903723276260782e-05 + -9.430573818110019e-05 + -8.974548326970851e-05 + -8.535231263786576e-05 + -8.112211782233505e-05 + -7.705084341917009e-05 + -7.313448187442813e-05 + -6.936907333027644e-05 + -6.575071119249467e-05 + -6.227553658275769e-05 + -5.893973914900695e-05 + -5.573956203853484e-05 + -5.267129595161032e-05 + -4.973128084575291e-05 + -4.691591029739419e-05 + -4.422162539456759e-05 + -4.164491727590835e-05 + -3.918233061143914e-05 + -3.683045757249665e-05 + -3.458594118927672e-05 + -3.244547785270521e-05 + -3.040581149216269e-05 + -2.846373767254703e-05 + -2.661610506551807e-05 + -2.485980997915884e-05 + -2.319180110803831e-05 + -2.160907991664170e-05 + -2.010869565536285e-05 + -1.868775067119526e-05 + -1.734339969669114e-05 + -1.607284546059277e-05 + -1.487334444060605e-05 + -1.374220506933501e-05 + -1.267678404087566e-05 + -1.167449238525668e-05 + -1.073279264625808e-05 + -9.849195952118955e-06 + -9.021268250139948e-06 + -8.246626567968640e-06 + -7.522936912951058e-06 + -6.847920492700095e-06 + -6.219349168072015e-06 + -5.635044236568559e-06 + -5.092882470932048e-06 + -4.590790895044201e-06 + -4.126746487291455e-06 + -3.698781859286729e-06 + -3.304979520120749e-06 + -2.943472506918129e-06 + -2.612449510984778e-06 + -2.310148834845521e-06 + -2.034859929271865e-06 + -1.784927798002038e-06 + -1.558746837246077e-06 + -1.354763241038800e-06 + -1.171478551126456e-06 + -1.007443551831214e-06 + -8.612614901318561e-07 + -7.315906671893518e-07 + -6.171385461507109e-07 + -5.166657190198488e-07 + -4.289874671965282e-07 + -3.529682224298676e-07 + -2.875261969469119e-07 + -2.316338735426513e-07 + -1.843129441850586e-07 + -1.446395043479685e-07 + -1.117434618507405e-07 + -8.480406305106530e-08 + -6.305553831873507e-08 + -4.578544870958251e-08 + -3.233089380928512e-08 + -2.208448022054805e-08 + -1.449165750478591e-08 + -9.047683233008124e-09 + -5.303766739728543e-09 + -2.863504097386866e-09 + -1.380623394721897e-09 + -5.650641971230870e-10 + -1.790372475752245e-10 + -3.511990521548007e-11 + -8.192061809269277e-13 + 0.000000000000000e+00 diff --git a/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve b/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve new file mode 100644 index 0000000000..f00fb1ec12 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/in.spin.iron-nve @@ -0,0 +1,49 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin/random 31 2.2 +velocity all create 100 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.02726 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin/spin 0.0 0.00 21 +fix 3 all nve/spin lattice moving +timestep 0.0001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 100000 diff --git a/examples/SPIN/test_problems/validation_nve/plot_nve.py b/examples/SPIN/test_problems/validation_nve/plot_nve.py new file mode 100755 index 0000000000..ffa7defaed --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/plot_nve.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import numpy as np, pylab, tkinter +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +from decimal import * +import sys, string, os + + +argv = sys.argv +if len(argv) != 3: + print("Syntax: ./plot_precession.py res_lammps.dat") + sys.exit() + +dirname = os.path.join(os.getcwd(), "Feb_07") +lammps_file = sys.argv[1] +llg_file = sys.argv[2] + +t,tmag,temp,e_mag,e_kin,e_pot,e_tot = np.loadtxt(lammps_file,skiprows=0, usecols=(1,2,3,4,5,6,7),unpack=True) + +fig = plt.figure(figsize=(8,8)) +ax1 = plt.subplot(3,1,1) +ax2 = plt.subplot(3,1,2) +ax3 = plt.subplot(3,1,3) + +ax1.plot(t, e_tot, 'k-', label='Total energy') +ax1.plot(t, e_pot, 'r-', label='Potential energy') +ax1.set_ylabel("E (eV)") +ax1.legend(loc=3) + +ax2.plot(t, e_kin, 'b-', label='Kinetic energy') +ax2.plot(t, e_mag, 'g-', label='Magnetic energy') +ax2.set_ylabel("E (eV)") +ax2.legend(loc=3) + +ax3.plot(t, temp, 'b--', label='Latt. temperature') +ax3.plot(t, tmag, 'r--', label='Spin temperature') +ax3.set_ylabel("T (K)") +ax3.legend(loc=3) + +plt.xlabel('Time (in ps)') +plt.legend() +plt.show() + +fig.savefig(os.path.join(os.getcwd(), "nve_spin_lattice.pdf"), bbox_inches="tight") +plt.close(fig) diff --git a/examples/SPIN/test_problems/validation_nve/run-test-nve.sh b/examples/SPIN/test_problems/validation_nve/run-test-nve.sh new file mode 100755 index 0000000000..441e7cf46d --- /dev/null +++ b/examples/SPIN/test_problems/validation_nve/run-test-nve.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# clean old res +rm res_*.dat + +# compute Lammps +./../../../../src/lmp_serial \ + -in in.spin.iron-nve +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_lammps.dat + +# plot results +python3 -m plot_nve.py res_lammps.dat res_llg.dat diff --git a/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy b/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy new file mode 100644 index 0000000000..69231bb7ee --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/Fe_Mishin2006.eam.alloy @@ -0,0 +1,15009 @@ +comment 1 +comment 2 +Converted by Ganga P Purja Pun using C++ code on Mon Nov 3 10:48:17 2014 +1 Fe +5001 2.400000000000000e-03 5001 1.134673400048920e-03 5.673367000244601e+00 +26 5.584700000000000e+01 2.855300000000000e+00 BCC + 0.000000000000000e+00 + -2.338738741480766e-02 + -4.628214468925276e-02 + -6.912258036387915e-02 + -9.175603963501618e-02 + -1.141497214000000e-01 + -1.363388222824136e-01 + -1.583226111166723e-01 + -1.800965752913192e-01 + -2.016645989796093e-01 + -2.230289901000000e-01 + -2.441907391820846e-01 + -2.651515164004249e-01 + -2.859130554412839e-01 + -3.064769176965011e-01 + -3.268446844000000e-01 + -3.470179531091855e-01 + -3.669982968636285e-01 + -3.867872784635140e-01 + -4.063864535839295e-01 + -4.257973671999999e-01 + -4.450215551034667e-01 + -4.640605423684923e-01 + -4.829158454463851e-01 + -5.015889707095635e-01 + -5.200814146000000e-01 + -5.383946650297390e-01 + -5.565301991603658e-01 + -5.744894857268537e-01 + -5.922739837155686e-01 + -6.098851423000000e-01 + -6.273244022645037e-01 + -6.445931939756846e-01 + -6.616929394780281e-01 + -6.786250511352716e-01 + -6.953909318999999e-01 + -7.119919765952718e-01 + -7.284295696432279e-01 + -7.447050873484842e-01 + -7.608198966551055e-01 + -7.767753551999997e-01 + -7.925728126189590e-01 + -8.082136084644670e-01 + -8.236990743647939e-01 + -8.390305327768360e-01 + -8.542092970000001e-01 + -8.692366724924486e-01 + -8.841139549244775e-01 + -8.988424321942350e-01 + -9.134233831702263e-01 + -9.278580778000000e-01 + -9.421477783833053e-01 + -9.562937376266863e-01 + -9.702972006149804e-01 + -9.841594035897939e-01 + -9.978815741999999e-01 + -1.011464932626877e+00 + -1.024910689374227e+00 + -1.038220047492379e+00 + -1.051394201530088e+00 + -1.064434338000000e+00 + -1.077341636148692e+00 + -1.090117264995124e+00 + -1.102762386491716e+00 + -1.115278154872840e+00 + -1.127665716000000e+00 + -1.139926208397942e+00 + -1.152060761127338e+00 + -1.164070496370367e+00 + -1.175956528594255e+00 + -1.187719964000000e+00 + -1.199361901614307e+00 + -1.210883431375302e+00 + -1.222285636497208e+00 + -1.233569592577916e+00 + -1.244736367000000e+00 + -1.255787020138112e+00 + -1.266722603828963e+00 + -1.277544163022812e+00 + -1.288252734706093e+00 + -1.298849349000000e+00 + -1.309335029515858e+00 + -1.319710789766606e+00 + -1.329977637188551e+00 + -1.340136572985217e+00 + -1.350188590000000e+00 + -1.360134674013246e+00 + -1.369975802739515e+00 + -1.379712947700534e+00 + -1.389347073262624e+00 + -1.398879136000000e+00 + -1.408310086019347e+00 + -1.417640866058629e+00 + -1.426872412303706e+00 + -1.436005653062347e+00 + -1.445041510000000e+00 + -1.453980898788701e+00 + -1.462824726507764e+00 + -1.471573894410746e+00 + -1.480229297407323e+00 + -1.488791823000000e+00 + -1.497262352264216e+00 + -1.505641758273629e+00 + -1.513930908641532e+00 + -1.522130664940848e+00 + -1.530241881000000e+00 + -1.538265404312065e+00 + -1.546202075663511e+00 + -1.554052729949587e+00 + -1.561818194964572e+00 + -1.569499292000000e+00 + -1.577096836740392e+00 + -1.584611637631811e+00 + -1.592044497451960e+00 + -1.599396212413425e+00 + -1.606667572000000e+00 + -1.613859360067553e+00 + -1.620972353850301e+00 + -1.628007324820493e+00 + -1.634965037585305e+00 + -1.641846251000000e+00 + -1.648651718746855e+00 + -1.655382187120556e+00 + -1.662038397074428e+00 + -1.668621083574689e+00 + -1.675130975000000e+00 + -1.681568794320005e+00 + -1.687935258528783e+00 + -1.694231079461267e+00 + -1.700456962682860e+00 + -1.706613607000000e+00 + -1.712701705964896e+00 + -1.718721948142722e+00 + -1.724675016499694e+00 + -1.730561586828580e+00 + -1.736382330000000e+00 + -1.742137912457411e+00 + -1.747828994017656e+00 + -1.753456229026646e+00 + -1.759020265526974e+00 + -1.764521747000000e+00 + -1.769961312537754e+00 + -1.775339594027171e+00 + -1.780657218386706e+00 + -1.785914807192779e+00 + -1.791112977000000e+00 + -1.796252339747382e+00 + -1.801333500673453e+00 + -1.806357060360666e+00 + -1.811323614328498e+00 + -1.816233753000000e+00 + -1.821088062150410e+00 + -1.825887120985276e+00 + -1.830631504346673e+00 + -1.835321782397329e+00 + -1.839958520000000e+00 + -1.844542277375300e+00 + -1.849073608812209e+00 + -1.853553064572349e+00 + -1.857981190434394e+00 + -1.862358526000000e+00 + -1.866685606181999e+00 + -1.870962962067843e+00 + -1.875191119963197e+00 + -1.879370599902195e+00 + -1.883501918000000e+00 + -1.887585586736770e+00 + -1.891622112755978e+00 + -1.895611998485404e+00 + -1.899555741651499e+00 + -1.903453835000000e+00 + -1.907306767129976e+00 + -1.911115021853308e+00 + -1.914879078883785e+00 + -1.918599413067872e+00 + -1.922276495000000e+00 + -1.925910791456493e+00 + -1.929502763824010e+00 + -1.933052869645757e+00 + -1.936561562187136e+00 + -1.940029290000000e+00 + -1.943456497684402e+00 + -1.946843625242175e+00 + -1.950191109032846e+00 + -1.953499381145042e+00 + -1.956768869000000e+00 + -1.959999996258605e+00 + -1.963193182622893e+00 + -1.966348843806269e+00 + -1.969467390558660e+00 + -1.972549230000000e+00 + -1.975594766012145e+00 + -1.978604397775412e+00 + -1.981578520793180e+00 + -1.984517526364655e+00 + -1.987421802000000e+00 + -1.990291731861874e+00 + -1.993127695559656e+00 + -1.995930069364364e+00 + -1.998699225767272e+00 + -2.001435533000000e+00 + -2.004139355858378e+00 + -2.006811055538481e+00 + -2.009450989763890e+00 + -2.012059511961222e+00 + -2.014636972000000e+00 + -2.017183716742461e+00 + -2.019700089231340e+00 + -2.022186429001382e+00 + -2.024643071393187e+00 + -2.027070349000000e+00 + -2.029468591629637e+00 + -2.031838124095820e+00 + -2.034179268100215e+00 + -2.036492342201070e+00 + -2.038777662000000e+00 + -2.041035540156417e+00 + -2.043266284490031e+00 + -2.045470200083453e+00 + -2.047647589335177e+00 + -2.049798751000000e+00 + -2.051923980684690e+00 + -2.054023570214561e+00 + -2.056097808968247e+00 + -2.058146983649728e+00 + -2.060171377000000e+00 + -2.062171268551674e+00 + -2.064146934640815e+00 + -2.066098649159696e+00 + -2.068026683134947e+00 + -2.069931304000000e+00 + -2.071812776276246e+00 + -2.073671361471072e+00 + -2.075507318519566e+00 + -2.077320903301032e+00 + -2.079112369000000e+00 + -2.080881966139512e+00 + -2.082629940879466e+00 + -2.084356537486872e+00 + -2.086061998635378e+00 + -2.087746563000000e+00 + -2.089410466229110e+00 + -2.091053941828707e+00 + -2.092677220771137e+00 + -2.094280530723314e+00 + -2.095864097000000e+00 + -2.097428142794787e+00 + -2.098972888139794e+00 + -2.100498550462023e+00 + -2.102005344216306e+00 + -2.103493482000000e+00 + -2.104963174393948e+00 + -2.106414628045970e+00 + -2.107848047779690e+00 + -2.109263636820768e+00 + -2.110661595000000e+00 + -2.112042119471926e+00 + -2.113405405101743e+00 + -2.114751645122076e+00 + -2.116081030815763e+00 + -2.117393750000000e+00 + -2.118689987937912e+00 + -2.119969928191390e+00 + -2.121233752504371e+00 + -2.122481640223532e+00 + -2.123713768000000e+00 + -2.124930310308645e+00 + -2.126131439345980e+00 + -2.127317325650607e+00 + -2.128488137848567e+00 + -2.129644042000000e+00 + -2.130785202020548e+00 + -2.131911779449632e+00 + -2.133023934312029e+00 + -2.134121824962966e+00 + -2.135205607000000e+00 + -2.136275433910872e+00 + -2.137331457489147e+00 + -2.138373827747866e+00 + -2.139402692469194e+00 + -2.140418197999999e+00 + -2.141420489142081e+00 + -2.142409707722756e+00 + -2.143385994030927e+00 + -2.144349486926974e+00 + -2.145300323000000e+00 + -2.146238637076898e+00 + -2.147164562479522e+00 + -2.148078231051947e+00 + -2.148979772720135e+00 + -2.149869315000000e+00 + -2.150746983835218e+00 + -2.151612904662093e+00 + -2.152467200947815e+00 + -2.153309993218032e+00 + -2.154141401000000e+00 + -2.154961542883784e+00 + -2.155770535619624e+00 + -2.156568494253713e+00 + -2.157355531798206e+00 + -2.158131760000000e+00 + -2.158897289576013e+00 + -2.159652229917454e+00 + -2.160396688379106e+00 + -2.161130769707277e+00 + -2.161854579000000e+00 + -2.162568220747540e+00 + -2.163271795520776e+00 + -2.163965403104795e+00 + -2.164649143391419e+00 + -2.165323114000000e+00 + -2.165987410839904e+00 + -2.166642128874188e+00 + -2.167287362019562e+00 + -2.167923202786971e+00 + -2.168549742000000e+00 + -2.169167069289456e+00 + -2.169775273557456e+00 + -2.170374442741976e+00 + -2.170964663390078e+00 + -2.171546020000000e+00 + -2.172118595836383e+00 + -2.172682474202094e+00 + -2.173237737093693e+00 + -2.173784464448991e+00 + -2.174322736000000e+00 + -2.174852630794634e+00 + -2.175374225221640e+00 + -2.175887595177988e+00 + -2.176392816678372e+00 + -2.176889964000000e+00 + -2.177379110090651e+00 + -2.177860327096639e+00 + -2.178333686466943e+00 + -2.178799258747679e+00 + -2.179257113000000e+00 + -2.179707317299531e+00 + -2.180149939409128e+00 + -2.180585046165010e+00 + -2.181012703042297e+00 + -2.181432975000000e+00 + -2.181845926509731e+00 + -2.182251621108717e+00 + -2.182650121032542e+00 + -2.183041486916871e+00 + -2.183425780000000e+00 + -2.183803061402426e+00 + -2.184173389857596e+00 + -2.184536823542827e+00 + -2.184893420600431e+00 + -2.185243238000000e+00 + -2.185586332004080e+00 + -2.185922759056265e+00 + -2.186252574500474e+00 + -2.186575831916728e+00 + -2.186892585000000e+00 + -2.187202887393400e+00 + -2.187506791832633e+00 + -2.187804350256584e+00 + -2.188095613705712e+00 + -2.188380633000000e+00 + -2.188659458643119e+00 + -2.188932140238827e+00 + -2.189198726845117e+00 + -2.189459267002233e+00 + -2.189713809000000e+00 + -2.189962400732430e+00 + -2.190205089044370e+00 + -2.190441920581333e+00 + -2.190672942002169e+00 + -2.190898199000000e+00 + -2.191117736662499e+00 + -2.191331600149922e+00 + -2.191539834109547e+00 + -2.191742482380937e+00 + -2.191939589000000e+00 + -2.192131197889608e+00 + -2.192317351831708e+00 + -2.192498093290244e+00 + -2.192673464653123e+00 + -2.192843508000000e+00 + -2.193008265149897e+00 + -2.193167777657277e+00 + -2.193322086821774e+00 + -2.193471233640668e+00 + -2.193615259000000e+00 + -2.193754203483761e+00 + -2.193888106693908e+00 + -2.194017008313546e+00 + -2.194140948496115e+00 + -2.194259967000000e+00 + -2.194374103230534e+00 + -2.194483396307522e+00 + -2.194587885225862e+00 + -2.194687608881331e+00 + -2.194782606000000e+00 + -2.194872915200472e+00 + -2.194958575082935e+00 + -2.195039623893670e+00 + -2.195116099394696e+00 + -2.195188040000000e+00 + -2.195255484352944e+00 + -2.195318470219860e+00 + -2.195377035223315e+00 + -2.195431217130485e+00 + -2.195481054000000e+00 + -2.195526583926446e+00 + -2.195567844428364e+00 + -2.195604873029958e+00 + -2.195637707516086e+00 + -2.195666386000000e+00 + -2.195690946583878e+00 + -2.195711426545218e+00 + -2.195727863658885e+00 + -2.195740296673043e+00 + -2.195748763000000e+00 + -2.195753299607926e+00 + -2.195753945356280e+00 + -2.195750739132009e+00 + -2.195743719093693e+00 + -2.195732923000000e+00 + -2.195718388771941e+00 + -2.195700155840199e+00 + -2.195678263244660e+00 + -2.195652748934943e+00 + -2.195623652000000e+00 + -2.195591012099144e+00 + -2.195554867993407e+00 + -2.195515258985693e+00 + -2.195472225489285e+00 + -2.195425807000000e+00 + -2.195376042760528e+00 + -2.195322973614639e+00 + -2.195266640267298e+00 + -2.195207082686241e+00 + -2.195144342000000e+00 + -2.195078459809379e+00 + -2.195009476453377e+00 + -2.194937433066797e+00 + -2.194862372466152e+00 + -2.194784337000000e+00 + -2.194703368615792e+00 + -2.194619509257747e+00 + -2.194532801771278e+00 + -2.194443290305841e+00 + -2.194351018000000e+00 + -2.194256027737437e+00 + -2.194158364279782e+00 + -2.194058072250755e+00 + -2.193955195446114e+00 + -2.193849779000000e+00 + -2.193741868885436e+00 + -2.193631510785602e+00 + -2.193518750557477e+00 + -2.193403634464252e+00 + -2.193286209000000e+00 + -2.193166521091908e+00 + -2.193044618884907e+00 + -2.192920550384463e+00 + -2.192794362988794e+00 + -2.192666105000000e+00 + -2.192535825071375e+00 + -2.192403570783236e+00 + -2.192269389306454e+00 + -2.192133327558461e+00 + -2.191995432000000e+00 + -2.191855748732582e+00 + -2.191714323538803e+00 + -2.191571201647935e+00 + -2.191426427571225e+00 + -2.191280046000000e+00 + -2.191132101498374e+00 + -2.190982637495000e+00 + -2.190831697104933e+00 + -2.190679323369808e+00 + -2.190525559000000e+00 + -2.190370446239990e+00 + -2.190214026313205e+00 + -2.190056340263544e+00 + -2.189897429222788e+00 + -2.189737334000000e+00 + -2.189576094894156e+00 + -2.189413751026334e+00 + -2.189250341662785e+00 + -2.189085906605225e+00 + -2.188920484000000e+00 + -2.188754111167896e+00 + -2.188586826517446e+00 + -2.188418667831656e+00 + -2.188249671505532e+00 + -2.188079874000000e+00 + -2.187909311655257e+00 + -2.187738019850499e+00 + -2.187566033827133e+00 + -2.187393388939138e+00 + -2.187220120000000e+00 + -2.187046261198758e+00 + -2.186871845656608e+00 + -2.186696906557450e+00 + -2.186521477499284e+00 + -2.186345591000000e+00 + -2.186169278929348e+00 + -2.185992573469870e+00 + -2.185815507881449e+00 + -2.185638116930091e+00 + -2.185460435000000e+00 + -2.185282496723147e+00 + -2.185104339046128e+00 + -2.184925999471230e+00 + -2.184747515559985e+00 + -2.184568926000000e+00 + -2.184390270419004e+00 + -2.184211589307456e+00 + -2.184032923638580e+00 + -2.183854314918697e+00 + -2.183675806000000e+00 + -2.183497440664645e+00 + -2.183319263062145e+00 + -2.183141318226586e+00 + -2.182963652427783e+00 + -2.182786312000000e+00 + -2.182609343803448e+00 + -2.182432796909005e+00 + -2.182256720850984e+00 + -2.182081165100584e+00 + -2.181906180000000e+00 + -2.181731816772078e+00 + -2.181558127901617e+00 + -2.181385166690271e+00 + -2.181212987245827e+00 + -2.181041644000000e+00 + -2.180871191888493e+00 + -2.180701687214938e+00 + -2.180533187286951e+00 + -2.180365750436408e+00 + -2.180199435000000e+00 + -2.180034299706266e+00 + -2.179870405073628e+00 + -2.179707812476292e+00 + -2.179546583956951e+00 + -2.179386782000000e+00 + -2.179228469569652e+00 + -2.179071710572415e+00 + -2.178916570209873e+00 + -2.178763115274171e+00 + -2.178611412000000e+00 + -2.178461526719817e+00 + -2.178313527960817e+00 + -2.178167485170301e+00 + -2.178023468394170e+00 + -2.177881548000000e+00 + -2.177741794911888e+00 + -2.177604281585152e+00 + -2.177469081134519e+00 + -2.177336267165003e+00 + -2.177205914000000e+00 + -2.177078096781969e+00 + -2.176952892179310e+00 + -2.176830377266735e+00 + -2.176710629289710e+00 + -2.176593727000000e+00 + -2.176479750089725e+00 + -2.176368778170715e+00 + -2.176260892035270e+00 + -2.176156174290115e+00 + -2.176054707000000e+00 + -2.175956572480315e+00 + -2.175861855847520e+00 + -2.175770642395214e+00 + -2.175683016800304e+00 + -2.175599066000000e+00 + -2.175518878125867e+00 + -2.175442540126260e+00 + -2.175370140365612e+00 + -2.175301769822078e+00 + -2.175237519000000e+00 + -2.175177478422601e+00 + -2.175121740498959e+00 + -2.175070398200953e+00 + -2.175023544771637e+00 + -2.174981275000000e+00 + -2.174943684570890e+00 + -2.174910868831887e+00 + -2.174882924320826e+00 + -2.174859949548785e+00 + -2.174842043000000e+00 + -2.174829303331104e+00 + -2.174821830410927e+00 + -2.174819725228089e+00 + -2.174823090027552e+00 + -2.174832027000000e+00 + -2.174846638699704e+00 + -2.174867029559900e+00 + -2.174893304738549e+00 + -2.174925569834485e+00 + -2.174963931000000e+00 + -2.175008495084293e+00 + -2.175059370394289e+00 + -2.175116665887380e+00 + -2.175180491027200e+00 + -2.175250956000000e+00 + -2.175328171805043e+00 + -2.175412250937077e+00 + -2.175503306357684e+00 + -2.175601451271103e+00 + -2.175706800000000e+00 + -2.175819467765500e+00 + -2.175939570549688e+00 + -2.176067225041086e+00 + -2.176202548787411e+00 + -2.176345660000000e+00 + -2.176496677644949e+00 + -2.176655722183164e+00 + -2.176822914601843e+00 + -2.176998376191533e+00 + -2.177182229000000e+00 + -2.177374595921169e+00 + -2.177575601341799e+00 + -2.177785370087074e+00 + -2.178004027196875e+00 + -2.178231699000000e+00 + -2.178468512862275e+00 + -2.178714597011325e+00 + -2.178970080062299e+00 + -2.179235091002944e+00 + -2.179509760000000e+00 + -2.179794218272507e+00 + -2.180088598277346e+00 + -2.180393032665419e+00 + -2.180707654074028e+00 + -2.181032597000000e+00 + -2.181367997183408e+00 + -2.181713990526695e+00 + -2.182070713375897e+00 + -2.182438302803596e+00 + -2.182816897000000e+00 + -2.183206635150288e+00 + -2.183607657686531e+00 + -2.184020105275964e+00 + -2.184444118610974e+00 + -2.184879840000000e+00 + -2.185327412889597e+00 + -2.185786981109685e+00 + -2.186258689137973e+00 + -2.186742682377725e+00 + -2.187239107000000e+00 + -2.187748109862624e+00 + -2.188269838801850e+00 + -2.188804442677439e+00 + -2.189352071523296e+00 + -2.189912875000000e+00 + -2.190487003093955e+00 + -2.191074608322116e+00 + -2.191675843787203e+00 + -2.192290862610885e+00 + -2.192919819000000e+00 + -2.193562867915730e+00 + -2.194220164552147e+00 + -2.194891865404003e+00 + -2.195578128862665e+00 + -2.196279113000000e+00 + -2.196994976008069e+00 + -2.197725877814248e+00 + -2.198471979076637e+00 + -2.199233441010543e+00 + -2.200010426000000e+00 + -2.200803097224683e+00 + -2.201611618154960e+00 + -2.202436152922896e+00 + -2.203276866629103e+00 + -2.204133924999999e+00 + -2.205007494076377e+00 + -2.205897739759803e+00 + -2.206804828380775e+00 + -2.207728927012060e+00 + -2.208670203000000e+00 + -2.209628824019662e+00 + -2.210604958519273e+00 + -2.211598775212799e+00 + -2.212610442959744e+00 + -2.213640131000000e+00 + -2.214688008997179e+00 + -2.215754247358006e+00 + -2.216839016713686e+00 + -2.217942487809252e+00 + -2.219064832000000e+00 + -2.220206221175054e+00 + -2.221366827800435e+00 + -2.222546824259503e+00 + -2.223746382727115e+00 + -2.224965677000000e+00 + -2.226204881636015e+00 + -2.227464170011377e+00 + -2.228743716360184e+00 + -2.230043696636770e+00 + -2.231364286000000e+00 + -2.232705659360148e+00 + -2.234067993079961e+00 + -2.235451464138931e+00 + -2.236856249856844e+00 + -2.238282527000000e+00 + -2.239730472524265e+00 + -2.241200265638654e+00 + -2.242692085287406e+00 + -2.244206109271373e+00 + -2.245742517000000e+00 + -2.247301488882056e+00 + -2.248883204910052e+00 + -2.250487845315945e+00 + -2.252115590911750e+00 + -2.253766623000000e+00 + -2.255441123360801e+00 + -2.257139274542631e+00 + -2.258861259219423e+00 + -2.260607260005651e+00 + -2.262377460000000e+00 + -2.264172042724291e+00 + -2.265991192137124e+00 + -2.267835092970031e+00 + -2.269703930922391e+00 + -2.271597891000000e+00 + -2.273517158096056e+00 + -2.275461918662951e+00 + -2.277432359442962e+00 + -2.279428667085979e+00 + -2.281451029000000e+00 + -2.283499633104788e+00 + -2.285574667389452e+00 + -2.287676320195930e+00 + -2.289804780398682e+00 + -2.291960237000000e+00 + -2.294142879454985e+00 + -2.296352898805551e+00 + -2.298590485768695e+00 + -2.300855830078550e+00 + -2.303149122999999e+00 + -2.305470556795997e+00 + -2.307820323545887e+00 + -2.310198615647936e+00 + -2.312605626084681e+00 + -2.315041548000000e+00 + -2.317506574745174e+00 + -2.320000900224320e+00 + -2.322524719122656e+00 + -2.325078227061991e+00 + -2.327661619000000e+00 + -2.330275089774933e+00 + -2.332918835677479e+00 + -2.335593053485966e+00 + -2.338297940187764e+00 + -2.341033693000000e+00 + -2.343800509406319e+00 + -2.346598587410773e+00 + -2.349428125303093e+00 + -2.352289321686241e+00 + -2.355182376000000e+00 + -2.358107488272472e+00 + -2.361064858786305e+00 + -2.364054687769198e+00 + -2.367077175375671e+00 + -2.370132523000000e+00 + -2.373220932737141e+00 + -2.376342606269360e+00 + -2.379497745810629e+00 + -2.382686554578887e+00 + -2.385909236000000e+00 + -2.389165993580208e+00 + -2.392457030846979e+00 + -2.395782552198058e+00 + -2.399142763298191e+00 + -2.402537869000000e+00 + -2.405968074028050e+00 + -2.409433584975734e+00 + -2.412934608682329e+00 + -2.416471351683178e+00 + -2.420044020999999e+00 + -2.423652824137149e+00 + -2.427297969206864e+00 + -2.430979664728804e+00 + -2.434698119632543e+00 + -2.438453543000000e+00 + -2.442246144168148e+00 + -2.446076133195298e+00 + -2.449943720571852e+00 + -2.453849117184546e+00 + -2.457792534000000e+00 + -2.461774182279110e+00 + -2.465794274341647e+00 + -2.469853022790889e+00 + -2.473950640299041e+00 + -2.478087340000000e+00 + -2.482263335430129e+00 + -2.486478840559030e+00 + -2.490734069661931e+00 + -2.495029237377275e+00 + -2.499364559000000e+00 + -2.503740250274207e+00 + -2.508156527124766e+00 + -2.512613605692700e+00 + -2.517111702474631e+00 + -2.521651035000000e+00 + -2.526231821279088e+00 + -2.530854278665131e+00 + -2.535518625205626e+00 + -2.540225080261870e+00 + -2.544973863000000e+00 + -2.549765192573217e+00 + -2.554599288888442e+00 + -2.559476372494764e+00 + -2.564396664604125e+00 + -2.569360386000000e+00 + -2.574367757513646e+00 + -2.579419001423863e+00 + -2.584514340338642e+00 + -2.589653996851589e+00 + -2.594838193999999e+00 + -2.600067155253186e+00 + -2.605341104662645e+00 + -2.610660266783872e+00 + -2.616024866689215e+00 + -2.621435129000000e+00 + -2.626891278432673e+00 + -2.632393541366548e+00 + -2.637942144571998e+00 + -2.643537314797249e+00 + -2.649179279000000e+00 + -2.654868264471787e+00 + -2.660604499292826e+00 + -2.666388211776204e+00 + -2.672219630341834e+00 + -2.678098984000000e+00 + -2.684026502306392e+00 + -2.690002415494505e+00 + -2.696026954017429e+00 + -2.702100348331385e+00 + -2.708222828000000e+00 + -2.714394620651795e+00 + -2.720615948464916e+00 + -2.726887030965477e+00 + -2.733208085290139e+00 + -2.739579324000000e+00 + -2.746000955816228e+00 + -2.752473185385865e+00 + -2.758996214641690e+00 + -2.765570242469367e+00 + -2.772195463000000e+00 + -2.778872066776275e+00 + -2.785600241574056e+00 + -2.792380171842594e+00 + -2.799212037606767e+00 + -2.806096015000000e+00 + -2.813032277129447e+00 + -2.820020994035244e+00 + -2.827062331911353e+00 + -2.834156451970738e+00 + -2.841303512999999e+00 + -2.848503671252426e+00 + -2.855757077985244e+00 + -2.863063881095023e+00 + -2.870424224890850e+00 + -2.877838249999999e+00 + -2.885306093928394e+00 + -2.892827890388094e+00 + -2.900403769651381e+00 + -2.908033857857018e+00 + -2.915718277999999e+00 + -2.923457150039019e+00 + -2.931250588977881e+00 + -2.939098706889091e+00 + -2.947001612872150e+00 + -2.954959411999998e+00 + -2.962972205810575e+00 + -2.971040091434060e+00 + -2.979163163083816e+00 + -2.987341511831155e+00 + -2.995575224999998e+00 + -3.003864386499012e+00 + -3.012209075546996e+00 + -3.020609368336417e+00 + -3.029065337835770e+00 + -3.037577052999999e+00 + -3.046144579520789e+00 + -3.054767979666142e+00 + -3.063447312238444e+00 + -3.072182631692820e+00 + -3.080973988999999e+00 + -3.089821432099915e+00 + -3.098725004829577e+00 + -3.107684748048404e+00 + -3.116700699185534e+00 + -3.125772890999999e+00 + -3.134901352715559e+00 + -3.144086110699870e+00 + -3.153327188100508e+00 + -3.162624603843408e+00 + -3.171978372999998e+00 + -3.181388507546933e+00 + -3.190855016073102e+00 + -3.200377903469057e+00 + -3.209957169965227e+00 + -3.219592812999998e+00 + -3.229284827363896e+00 + -3.239033203402719e+00 + -3.248837927724427e+00 + -3.258698982652799e+00 + -3.268616347999998e+00 + -3.278590000914418e+00 + -3.288619913401403e+00 + -3.298706054004673e+00 + -3.308848387625167e+00 + -3.319046875999998e+00 + -3.329301477934322e+00 + -3.339612147844228e+00 + -3.349978836592516e+00 + -3.360401490990458e+00 + -3.370880054999998e+00 + -3.381414469596773e+00 + -3.392004670266620e+00 + -3.402650589976440e+00 + -3.413352159446950e+00 + -3.424109303999998e+00 + -3.434921945057392e+00 + -3.445790001867386e+00 + -3.456713390096515e+00 + -3.467692020464062e+00 + -3.478725800999998e+00 + -3.489814636890736e+00 + -3.500958427789171e+00 + -3.512157070764700e+00 + -3.523410460589800e+00 + -3.534718486999998e+00 + -3.546081035880271e+00 + -3.557497990199610e+00 + -3.568969229634993e+00 + -3.580494629586285e+00 + -3.592074061999998e+00 + -3.603707395759104e+00 + -3.615394495446970e+00 + -3.627135222559569e+00 + -3.638929435115233e+00 + -3.650776987000000e+00 + -3.662677728742245e+00 + -3.674631507325805e+00 + -3.686638166579447e+00 + -3.698697546469576e+00 + -3.710809483000000e+00 + -3.722973808828209e+00 + -3.735190352595157e+00 + -3.747458940125227e+00 + -3.759779394010754e+00 + -3.772151532000000e+00 + -3.784575168342916e+00 + -3.797050115062026e+00 + -3.809576180292756e+00 + -3.822153166803174e+00 + -3.834780875000000e+00 + -3.847459102820189e+00 + -3.860187643203489e+00 + -3.872966285667494e+00 + -3.885794816080159e+00 + -3.898673017000000e+00 + -3.911600667911320e+00 + -3.924577543724543e+00 + -3.937603416269590e+00 + -3.950678053996523e+00 + -3.963801221000000e+00 + -3.976972678103186e+00 + -3.990192183512261e+00 + -4.003459491521890e+00 + -4.016774351164608e+00 + -4.030136509000000e+00 + -4.043545709147031e+00 + -4.057001691149858e+00 + -4.070504190840202e+00 + -4.084052939835804e+00 + -4.097647667000000e+00 + -4.111288098419088e+00 + -4.124973955231095e+00 + -4.138704955371648e+00 + -4.152480813397504e+00 + -4.166301240000000e+00 + -4.180165942572286e+00 + -4.194074624537170e+00 + -4.208026986370140e+00 + -4.222022725101974e+00 + -4.236061533000000e+00 + -4.250143098846027e+00 + -4.264267108996074e+00 + -4.278433246261039e+00 + -4.292641188627767e+00 + -4.306890611000000e+00 + -4.321181185386233e+00 + -4.335512579090955e+00 + -4.349884456391911e+00 + -4.364296478350024e+00 + -4.378748302000000e+00 + -4.393239580901771e+00 + -4.407769964394226e+00 + -4.422339098999587e+00 + -4.436946628138934e+00 + -4.451592191000000e+00 + -4.466275423153566e+00 + -4.480995955945297e+00 + -4.495753418053649e+00 + -4.510547435209278e+00 + -4.525377628000000e+00 + -4.540243613376725e+00 + -4.555145006346341e+00 + -4.570081418154680e+00 + -4.585052454714511e+00 + -4.600057719000000e+00 + -4.615096811360685e+00 + -4.630169328028963e+00 + -4.645274861878902e+00 + -4.660413001844071e+00 + -4.675583333000000e+00 + -4.690785437259527e+00 + -4.706018892955582e+00 + -4.721283275134317e+00 + -4.736578154793736e+00 + -4.751903099000000e+00 + -4.767257671719769e+00 + -4.782641433752779e+00 + -4.798053942304498e+00 + -4.813494749953581e+00 + -4.828963406000000e+00 + -4.844459456896403e+00 + -4.859982445123970e+00 + -4.875531909767719e+00 + -4.891107385884028e+00 + -4.906708405000000e+00 + -4.922334495555203e+00 + -4.937985181840751e+00 + -4.953659984814029e+00 + -4.969358421588884e+00 + -4.985080006000000e+00 + -5.000824248840914e+00 + -5.016590656340991e+00 + -5.032378731598668e+00 + -5.048187974277996e+00 + -5.064017880000000e+00 + -5.079867940973151e+00 + -5.095737645459726e+00 + -5.111626480823102e+00 + -5.127533934923204e+00 + -5.143459499999997e+00 + -5.159402672483111e+00 + -5.175362954369220e+00 + -5.191339851303958e+00 + -5.207332872891304e+00 + -5.223341533999998e+00 + -5.239365353688981e+00 + -5.255403855076862e+00 + -5.271456565155025e+00 + -5.287523015792912e+00 + -5.303602743999997e+00 + -5.319695290809813e+00 + -5.335800201177900e+00 + -5.351917023850607e+00 + -5.368045312413205e+00 + -5.384184625999997e+00 + -5.400334527792659e+00 + -5.416494583936292e+00 + -5.432664365069262e+00 + -5.448843447960337e+00 + -5.465031412999998e+00 + -5.481227843963791e+00 + -5.497432330198948e+00 + -5.513644465003656e+00 + -5.529863846001168e+00 + -5.546090074999998e+00 + -5.562322757794160e+00 + -5.578561506278419e+00 + -5.594805935740466e+00 + -5.611055664858841e+00 + -5.627310317999997e+00 + -5.643569523867812e+00 + -5.659832914603254e+00 + -5.676100126691991e+00 + -5.692370802379017e+00 + -5.708644587999997e+00 + -5.724921133351554e+00 + -5.741200092816319e+00 + -5.757481124931912e+00 + -5.773763893288205e+00 + -5.790048065999996e+00 + -5.806333315035348e+00 + -5.822619317227983e+00 + -5.838905753168464e+00 + -5.855192307810226e+00 + -5.871478670999998e+00 + -5.887764536642293e+00 + -5.904049603224381e+00 + -5.920333573291820e+00 + -5.936616154283948e+00 + -5.952897057999997e+00 + -5.969175999997380e+00 + -5.985452700712029e+00 + -6.001726884408804e+00 + -6.017998279848357e+00 + -6.034266620999998e+00 + -6.050531645917840e+00 + -6.066793096565948e+00 + -6.083050719154321e+00 + -6.099304265265417e+00 + -6.115553489999997e+00 + -6.131798152082848e+00 + -6.148038016850530e+00 + -6.164272852974367e+00 + -6.180502432221914e+00 + -6.196726531999996e+00 + -6.212944934149927e+00 + -6.229157424398349e+00 + -6.245363792906498e+00 + -6.261563835474968e+00 + -6.277757350999996e+00 + -6.293944141712862e+00 + -6.310124016459128e+00 + -6.326296787703612e+00 + -6.342462271423720e+00 + -6.358620288999997e+00 + -6.374770666045148e+00 + -6.390913231950829e+00 + -6.407047820441565e+00 + -6.423174270811367e+00 + -6.439292425999997e+00 + -6.455402132459349e+00 + -6.471503242529590e+00 + -6.487595612302881e+00 + -6.503679101824962e+00 + -6.519753575999998e+00 + -6.535818903908587e+00 + -6.551874959678351e+00 + -6.567921621040533e+00 + -6.583958769810407e+00 + -6.599986292999997e+00 + -6.616004081814032e+00 + -6.632012031732002e+00 + -6.648010042213294e+00 + -6.663998017633650e+00 + -6.679975866999997e+00 + -6.695943503153600e+00 + -6.711900843425355e+00 + -6.727847809100131e+00 + -6.743784326279565e+00 + -6.759710325999998e+00 + -6.775625743222601e+00 + -6.791530516957345e+00 + -6.807424590371784e+00 + -6.823307911868753e+00 + -6.839180434000000e+00 + -6.855042113095231e+00 + -6.870892911056689e+00 + -6.886732793343159e+00 + -6.902561729209182e+00 + -6.918379693000000e+00 + -6.934186663377454e+00 + -6.949982623974822e+00 + -6.965767562024019e+00 + -6.981541468861679e+00 + -6.997304341000000e+00 + -7.013056179149240e+00 + -7.028796988372566e+00 + -7.044526777567921e+00 + -7.060245560311449e+00 + -7.075953355000000e+00 + -7.091650184045992e+00 + -7.107336074531737e+00 + -7.123011057484204e+00 + -7.138675168614964e+00 + -7.154328448000000e+00 + -7.169970939523060e+00 + -7.185602692089082e+00 + -7.201223758621499e+00 + -7.216834196750369e+00 + -7.232434069000000e+00 + -7.248023441650831e+00 + -7.263602384451917e+00 + -7.279170972021925e+00 + -7.294729285389790e+00 + -7.310277408000000e+00 + -7.325815426345584e+00 + -7.341343434315181e+00 + -7.356861529294433e+00 + -7.372369811668458e+00 + -7.387868387000000e+00 + -7.403357365322682e+00 + -7.418836861854650e+00 + -7.434306995371617e+00 + -7.449767888578398e+00 + -7.465219669000000e+00 + -7.480662468362842e+00 + -7.496096423635060e+00 + -7.511521675509539e+00 + -7.526938368822738e+00 + -7.542346653000000e+00 + -7.557746681513402e+00 + -7.573138613106206e+00 + -7.588522610348394e+00 + -7.603898840111169e+00 + -7.619267474000000e+00 + -7.634628687556104e+00 + -7.649982660818226e+00 + -7.665329578157865e+00 + -7.680669629212444e+00 + -7.696003007000000e+00 + -7.711329908041110e+00 + -7.726650535322031e+00 + -7.741965095546557e+00 + -7.757273799081691e+00 + -7.772576861000000e+00 + -7.787874500482072e+00 + -7.803166941820082e+00 + -7.818454413357075e+00 + -7.833737148083599e+00 + -7.849015383000000e+00 + -7.864289358883056e+00 + -7.879559322305754e+00 + -7.894825523493435e+00 + -7.910088216491122e+00 + -7.925347660000000e+00 + -7.940604117030949e+00 + -7.955857856656654e+00 + -7.971109151088187e+00 + -7.986358275553978e+00 + -8.001605510999999e+00 + -8.016851143003514e+00 + -8.032095461580266e+00 + -8.047338760488806e+00 + -8.062581337981985e+00 + -8.077823497000001e+00 + -8.093065544394179e+00 + -8.108307791577566e+00 + -8.123550554205780e+00 + -8.138794153086785e+00 + -8.154038912999997e+00 + -8.169285162395173e+00 + -8.184533235306290e+00 + -8.199783469410583e+00 + -8.215036206329255e+00 + -8.230291792999997e+00 + -8.245550580528674e+00 + -8.260812923863309e+00 + -8.276079182235739e+00 + -8.291349720375887e+00 + -8.306624906999998e+00 + -8.321905114401627e+00 + -8.337190720129563e+00 + -8.352482105562956e+00 + -8.367779656388556e+00 + -8.383083762999997e+00 + -8.398394819954635e+00 + -8.413713227271362e+00 + -8.429039388338369e+00 + -8.444373710123081e+00 + -8.459716604999997e+00 + -8.475068489773328e+00 + -8.490429785807564e+00 + -8.505800918193746e+00 + -8.521182316459129e+00 + -8.536574414999997e+00 + -8.551977652264261e+00 + -8.567392471335332e+00 + -8.582819319266729e+00 + -8.598258647834163e+00 + -8.613710912999998e+00 + -8.629176574613023e+00 + -8.644656098315155e+00 + -8.660149953296175e+00 + -8.675658612460374e+00 + -8.691182553999997e+00 + -8.706722260346437e+00 + -8.722278218076596e+00 + -8.737850917931995e+00 + -8.753440855866357e+00 + -8.769048531999996e+00 + -8.784674450012169e+00 + -8.800319118245726e+00 + -8.815983049456063e+00 + -8.831666761752983e+00 + -8.847370776999997e+00 + -8.863095620572640e+00 + -8.878841823409489e+00 + -8.894609920288579e+00 + -8.910400450202564e+00 + -8.926213956999996e+00 + -8.942050988642462e+00 + -8.957912097948702e+00 + -8.973797841249739e+00 + -8.989708778929735e+00 + -9.005645476999996e+00 + -9.021608505891232e+00 + -9.037598440089864e+00 + -9.053615857732957e+00 + -9.069661341483132e+00 + -9.085735478999997e+00 + -9.101838862125330e+00 + -9.117972087538210e+00 + -9.134135755396374e+00 + -9.150330469883286e+00 + -9.166556840999997e+00 + -9.182815483079517e+00 + -9.199107013675079e+00 + -9.215432054583626e+00 + -9.231791233265957e+00 + -9.248185180999997e+00 + -9.264614532683632e+00 + -9.281079929109534e+00 + -9.297582014702897e+00 + -9.314121437654787e+00 + -9.330698850999996e+00 + -9.347314912003528e+00 + -9.363970283188971e+00 + -9.380665630745685e+00 + -9.397401624940878e+00 + -9.414178940999998e+00 + -9.430998258244742e+00 + -9.447860260503859e+00 + -9.464765635761319e+00 + -9.481715077019253e+00 + -9.498709280999996e+00 + -9.515748948037771e+00 + -9.532834784442173e+00 + -9.549967500513571e+00 + -9.567147810785697e+00 + -9.584376433999996e+00 + -9.601654092679016e+00 + -9.618981514566920e+00 + -9.636359431588877e+00 + -9.653788580476306e+00 + -9.671269701999998e+00 + -9.688803540653204e+00 + -9.706390846469759e+00 + -9.724032373152683e+00 + -9.741728878390186e+00 + -9.759481124999997e+00 + -9.777289880050315e+00 + -9.795155915261526e+00 + -9.813080006095444e+00 + -9.831062932433847e+00 + -9.849105478999997e+00 + -9.867208434586416e+00 + -9.885372592754736e+00 + -9.903598750780114e+00 + -9.921887710289466e+00 + -9.940240277999996e+00 + -9.958657264807959e+00 + -9.977139486156000e+00 + -9.995687760994512e+00 + -1.001430291248964e+01 + -1.003298577000000e+01 + -1.005173716741557e+01 + -1.007055794170109e+01 + -1.008944893313717e+01 + -1.010841098632109e+01 + -1.012744495000000e+01 + -1.014655167843811e+01 + -1.016573203809090e+01 + -1.018498689280856e+01 + -1.020431709913334e+01 + -1.022372353000000e+01 + -1.024320706932260e+01 + -1.026276860085199e+01 + -1.028240900753776e+01 + -1.030212917226508e+01 + -1.032192999000000e+01 + -1.034181236270672e+01 + -1.036177718920571e+01 + -1.038182537682789e+01 + -1.040195784652257e+01 + -1.042217551000000e+01 + -1.044247927800797e+01 + -1.046287008429113e+01 + -1.048334886197221e+01 + -1.050391653528098e+01 + -1.052457404000000e+01 + -1.054532232135862e+01 + -1.056616233045517e+01 + -1.058709501821704e+01 + -1.060812133414642e+01 + -1.062924224000000e+01 + -1.065045870478386e+01 + -1.067177169452023e+01 + -1.069318217826154e+01 + -1.071469113174534e+01 + -1.073629954000000e+01 + -1.075800839260036e+01 + -1.077981867489276e+01 + -1.080173138018267e+01 + -1.082374751517595e+01 + -1.084586808000000e+01 + -1.086809407471729e+01 + -1.089042651905600e+01 + -1.091286643223940e+01 + -1.093541482646511e+01 + -1.095807273000000e+01 + -1.098084118002027e+01 + -1.100372120627546e+01 + -1.102671384288809e+01 + -1.104982013415814e+01 + -1.107304113000000e+01 + -1.109637788366843e+01 + -1.111983144958023e+01 + -1.114340288661973e+01 + -1.116709326043168e+01 + -1.119090364000000e+01 + -1.121483509746141e+01 + -1.123888871051257e+01 + -1.126306556352589e+01 + -1.128736674843474e+01 + -1.131179335000000e+01 + -1.133634645465832e+01 + -1.136102717560643e+01 + -1.138583662099362e+01 + -1.141077588304154e+01 + -1.143584608000000e+01 + -1.146104834278586e+01 + -1.148638378227322e+01 + -1.151185351768562e+01 + -1.153745868860526e+01 + -1.156320043000000e+01 + -1.158907987623694e+01 + -1.161509817570602e+01 + -1.164125647703402e+01 + -1.166755592508495e+01 + -1.169399768000000e+01 + -1.172058291041322e+01 + -1.174731277861368e+01 + -1.177418844974534e+01 + -1.180121109657743e+01 + -1.182838189999999e+01 + -1.185570204579555e+01 + -1.188317272006268e+01 + -1.191079511332836e+01 + -1.193857042312863e+01 + -1.196649984999999e+01 + -1.199458459773775e+01 + -1.202282587700695e+01 + -1.205122490269986e+01 + -1.207978289369385e+01 + -1.210850107000000e+01 + -1.213738065440104e+01 + -1.216642287877588e+01 + -1.219562898067898e+01 + -1.222500020303083e+01 + -1.225453779000000e+01 + -1.228424298812801e+01 + -1.231411705144751e+01 + -1.234416123799292e+01 + -1.237437680934280e+01 + -1.240476502999999e+01 + -1.243532716862228e+01 + -1.246606450371358e+01 + -1.249697831672881e+01 + -1.252806989018327e+01 + -1.255934050999999e+01 + -1.259079146663310e+01 + -1.262242406008652e+01 + -1.265423959346900e+01 + -1.268623937120698e+01 + -1.271842469999999e+01 + -1.275079689157634e+01 + -1.278335727210730e+01 + -1.281610716656313e+01 + -1.284904789393346e+01 + -1.288218078999999e+01 + -1.291550719883721e+01 + -1.294902845305343e+01 + -1.298274589252790e+01 + -1.301666087289315e+01 + -1.305077474999999e+01 + -1.308508888000826e+01 + -1.311960462335401e+01 + -1.315432334734565e+01 + -1.318924642810263e+01 + -1.322437523999999e+01 + -1.325971115825294e+01 + -1.329525556797775e+01 + -1.333100986045822e+01 + -1.336697543256989e+01 + -1.340315367999999e+01 + -1.343954599979837e+01 + -1.347615379884262e+01 + -1.351297848978655e+01 + -1.355002149050285e+01 + -1.358728421999999e+01 + -1.362476809975143e+01 + -1.366247455932581e+01 + -1.370040503112116e+01 + -1.373856094968169e+01 + -1.377694375999999e+01 + -1.381555491216690e+01 + -1.385439585021465e+01 + -1.389346802391279e+01 + -1.393277289476898e+01 + -1.397231192999999e+01 + -1.401208659972945e+01 + -1.405209837359219e+01 + -1.409234872413346e+01 + -1.413283912961124e+01 + -1.417357107999999e+01 + -1.421454607102127e+01 + -1.425576559243092e+01 + -1.429723113890712e+01 + -1.433894421536596e+01 + -1.438090632999999e+01 + -1.442311899332400e+01 + -1.446558371903957e+01 + -1.450830202765583e+01 + -1.455127544879836e+01 + -1.459450550999999e+01 + -1.463799373973040e+01 + -1.468174167767078e+01 + -1.472575086722947e+01 + -1.477002285383887e+01 + -1.481455918999999e+01 + -1.485936143285468e+01 + -1.490443114031971e+01 + -1.494976987417127e+01 + -1.499537920245961e+01 + -1.504126069999999e+01 + -1.508741594534576e+01 + -1.513384651602358e+01 + -1.518055399754435e+01 + -1.522753998778076e+01 + -1.527480607999999e+01 + -1.532235386668537e+01 + -1.537018495189360e+01 + -1.541830094448208e+01 + -1.546670345694024e+01 + -1.551539410999999e+01 + -1.556437452805486e+01 + -1.561364632974880e+01 + -1.566321114034704e+01 + -1.571307059764629e+01 + -1.576322633999999e+01 + -1.581368000886133e+01 + -1.586443325932826e+01 + -1.591548774371123e+01 + -1.596684510633198e+01 + -1.601850700999999e+01 + -1.607047512763592e+01 + -1.612275112310131e+01 + -1.617533666634843e+01 + -1.622823344044972e+01 + -1.628144312999999e+01 + -1.633496741991010e+01 + -1.638880799548294e+01 + -1.644296655089035e+01 + -1.649744479372194e+01 + -1.655224442999999e+01 + -1.660736716535794e+01 + -1.666281471098534e+01 + -1.671858878335293e+01 + -1.677469110504862e+01 + -1.683112339999999e+01 + -1.688788739643611e+01 + -1.694498483759857e+01 + -1.700241746523662e+01 + -1.706018701398879e+01 + -1.711829522999999e+01 + -1.717674386844329e+01 + -1.723553468924849e+01 + -1.729466945458232e+01 + -1.735414992865347e+01 + -1.741397788000000e+01 + -1.747415508099178e+01 + -1.753468330867943e+01 + -1.759556434768274e+01 + -1.765679999203036e+01 + -1.771839203000000e+01 + -1.778034225038387e+01 + -1.784265246094084e+01 + -1.790532446959567e+01 + -1.796836007816754e+01 + -1.803176110000000e+01 + -1.809552935720682e+01 + -1.815966667575656e+01 + -1.822417488078262e+01 + -1.828905579597735e+01 + -1.835431126000000e+01 + -1.841994311958100e+01 + -1.848595321475806e+01 + -1.855234338946741e+01 + -1.861911549709020e+01 + -1.868627140000000e+01 + -1.875381296458075e+01 + -1.882174205218422e+01 + -1.889006052915166e+01 + -1.895877027181235e+01 + -1.902787316000000e+01 + -1.909737107797965e+01 + -1.916726592078253e+01 + -1.923755957952593e+01 + -1.930825393714747e+01 + -1.937935090000000e+01 + -1.945085238647674e+01 + -1.952276030080435e+01 + -1.959507655061933e+01 + -1.966780305499929e+01 + -1.974094174000000e+01 + -1.981449453618971e+01 + -1.988846337667131e+01 + -1.996285019538828e+01 + -2.003765692760933e+01 + -2.011288552000000e+01 + -2.018853792621815e+01 + -2.026461609876727e+01 + -2.034112199510926e+01 + -2.041805758075517e+01 + -2.049542482000000e+01 + -2.057322568006621e+01 + -2.065146214471047e+01 + -2.073013619484967e+01 + -2.080924980240138e+01 + -2.088880495999999e+01 + -2.096880367169853e+01 + -2.104924793164823e+01 + -2.113013973534757e+01 + -2.121148108557274e+01 + -2.129327399999999e+01 + -2.137552050224973e+01 + -2.145822260294134e+01 + -2.154138232011573e+01 + -2.162500168821614e+01 + -2.170908273999999e+01 + -2.179362750931509e+01 + -2.187863804236205e+01 + -2.196411638687395e+01 + -2.205006458870765e+01 + -2.213648469999998e+01 + -2.222337877888525e+01 + -2.231074889087196e+01 + -2.239859710699802e+01 + -2.248692550346576e+01 + -2.257573614999999e+01 + -2.266503111825121e+01 + -2.275481250533621e+01 + -2.284508240538831e+01 + -2.293584289979252e+01 + -2.302709608999998e+01 + -2.311884408797131e+01 + -2.321108899308483e+01 + -2.330383291176923e+01 + -2.339707796636268e+01 + -2.349082627999998e+01 + -2.358507997580010e+01 + -2.367984117841246e+01 + -2.377511201965127e+01 + -2.387089464198777e+01 + -2.396719118999998e+01 + -2.406400380890314e+01 + -2.416133464354705e+01 + -2.425918584868402e+01 + -2.435755959379285e+01 + -2.445645803999998e+01 + -2.455588334675619e+01 + -2.465583769169985e+01 + -2.475632325621871e+01 + -2.485734222078360e+01 + -2.495889676999998e+01 + -2.506098909313465e+01 + -2.516362138717989e+01 + -2.526679585042222e+01 + -2.537051468121954e+01 + -2.547478008999998e+01 + -2.557959429416147e+01 + -2.568495950782481e+01 + -2.579087794894506e+01 + -2.589735184307986e+01 + -2.600438342000000e+01 + -2.611197491414786e+01 + -2.622012856937007e+01 + -2.632884662779481e+01 + -2.643813132670684e+01 + -2.654798492000000e+01 + -2.665840967182968e+01 + -2.676940784333127e+01 + -2.688098169572292e+01 + -2.699313349264397e+01 + -2.710586551000000e+01 + -2.721918002998715e+01 + -2.733307932892397e+01 + -2.744756569229334e+01 + -2.756264142140602e+01 + -2.767830881000000e+01 + -2.779457014947893e+01 + -2.791142774510221e+01 + -2.802888390934165e+01 + -2.814694096000261e+01 + -2.826560121000000e+01 + -2.838486697334714e+01 + -2.850474058224100e+01 + -2.862522436986088e+01 + -2.874632066501767e+01 + -2.886803181000000e+01 + -2.899036015446983e+01 + -2.911330804180568e+01 + -2.923687781925054e+01 + -2.936107184331833e+01 + -2.948589248000000e+01 + -2.961134209910011e+01 + -2.973742306338333e+01 + -2.986413774439124e+01 + -2.999148852946843e+01 + -3.011947780000000e+01 + -3.024810793640462e+01 + -3.037738133418102e+01 + -3.050730039248281e+01 + -3.063786751048302e+01 + -3.076908508999995e+01 + -3.090095553801825e+01 + -3.103348127528379e+01 + -3.116666472242296e+01 + -3.130050829541340e+01 + -3.143501442000000e+01 + -3.157018552904646e+01 + -3.170602405762165e+01 + -3.184253244456109e+01 + -3.197971313442012e+01 + -3.211756857999995e+01 + -3.225610123724459e+01 + -3.239531355479052e+01 + -3.253520799120049e+01 + -3.267578702273956e+01 + -3.281705312000000e+01 + -3.295900875199519e+01 + -3.310165640007095e+01 + -3.324499854981137e+01 + -3.338903768896250e+01 + -3.353377630999994e+01 + -3.367921690900354e+01 + -3.382536198480008e+01 + -3.397221404252836e+01 + -3.411977559608952e+01 + -3.426804916000000e+01 + -3.441703725062087e+01 + -3.456674239208418e+01 + -3.471716711255165e+01 + -3.486831394366235e+01 + -3.502018541999994e+01 + -3.517278408030963e+01 + -3.532611247314257e+01 + -3.548017315006599e+01 + -3.563496866378715e+01 + -3.579050157000000e+01 + -3.594677442911821e+01 + -3.610378981290209e+01 + -3.626155029281482e+01 + -3.642005843692588e+01 + -3.657931682999995e+01 + -3.673932806450152e+01 + -3.690009471994529e+01 + -3.706161938636306e+01 + -3.722390467398576e+01 + -3.738695318000000e+01 + -3.755076749872251e+01 + -3.771535025114206e+01 + -3.788070405790332e+01 + -3.804683152981476e+01 + -3.821373528999994e+01 + -3.838141797088579e+01 + -3.854988220802454e+01 + -3.871913063690455e+01 + -3.888916589260551e+01 + -3.905999062000000e+01 + -3.923160747217621e+01 + -3.940401910958111e+01 + -3.957722819105772e+01 + -3.975123737141870e+01 + -3.992604931999993e+01 + -4.010166671506960e+01 + -4.027809223215413e+01 + -4.045532854769397e+01 + -4.063337834194039e+01 + -4.081224431000000e+01 + -4.099192915380458e+01 + -4.117243556512365e+01 + -4.135376624213356e+01 + -4.153592389680070e+01 + -4.171891123999993e+01 + -4.190273098404973e+01 + -4.208738585293229e+01 + -4.227287857304115e+01 + -4.245921187066173e+01 + -4.264638848000000e+01 + -4.283441114000928e+01 + -4.302328258823673e+01 + -4.321300556988346e+01 + -4.340358284230594e+01 + -4.359501715999993e+01 + -4.378731127802528e+01 + -4.398046796379431e+01 + -4.417448998677844e+01 + -4.436938011574802e+01 + -4.456514113000000e+01 + -4.476177581459606e+01 + -4.495928695037842e+01 + -4.515767732491430e+01 + -4.535694973791337e+01 + -4.555710698999992e+01 + -4.575815188265746e+01 + -4.596008722124420e+01 + -4.616291581815827e+01 + -4.636664049491562e+01 + -4.657126407000000e+01 + -4.677678936329625e+01 + -4.698321921012249e+01 + -4.719055644645270e+01 + -4.739880390447049e+01 + -4.760796442999992e+01 + -4.781804087601476e+01 + -4.802903608806237e+01 + -4.824095291782189e+01 + -4.845379422957746e+01 + -4.866756289000000e+01 + -4.888226176793614e+01 + -4.909789373759357e+01 + -4.931446167624577e+01 + -4.953196846445736e+01 + -4.975041698999991e+01 + -4.996981014490353e+01 + -5.019015082042799e+01 + -5.041144191483806e+01 + -5.063368633767777e+01 + -5.085688700000000e+01 + -5.108104681363623e+01 + -5.130616869222998e+01 + -5.153225555529192e+01 + -5.175931033093434e+01 + -5.198733594999992e+01 + -5.221633534632925e+01 + -5.244631146064819e+01 + -5.267726723814258e+01 + -5.290920562821528e+01 + -5.314212958000000e+01 + -5.337604204603855e+01 + -5.361094599406351e+01 + -5.384684439089011e+01 + -5.408374019738633e+01 + -5.432163638999991e+01 + -5.456053595406303e+01 + -5.480044186892415e+01 + -5.504135712069162e+01 + -5.528328470761400e+01 + -5.552622762000000e+01 + -5.577018884907901e+01 + -5.601517141284037e+01 + -5.626117832498171e+01 + -5.650821258404100e+01 + -5.675627720999991e+01 + -5.700537523542541e+01 + -5.725550968441112e+01 + -5.750668358361924e+01 + -5.775889996777224e+01 + -5.801216188000000e+01 + -5.826647236807500e+01 + -5.852183447842080e+01 + -5.877825126017432e+01 + -5.903572576827786e+01 + -5.929426106999990e+01 + -5.955386023803142e+01 + -5.981452633626228e+01 + -6.007626243395679e+01 + -6.033907161265666e+01 + -6.060295696000000e+01 + -6.086792156620618e+01 + -6.113396851889991e+01 + -6.140110091270849e+01 + -6.166932185413050e+01 + -6.193863444999990e+01 + -6.220904180836374e+01 + -6.248054704389698e+01 + -6.275315327303672e+01 + -6.302686361375484e+01 + -6.330168120000000e+01 + -6.357760917255472e+01 + -6.385465065869710e+01 + -6.413280879313753e+01 + -6.441208672717147e+01 + -6.469248761000000e+01 + -6.497401459207148e+01 + -6.525667083785252e+01 + -6.554045951020694e+01 + -6.582538376572784e+01 + -6.611144677999999e+01 + -6.639865173804421e+01 + -6.668700181225154e+01 + -6.697650018322166e+01 + -6.726715004865791e+01 + -6.755895460000001e+01 + -6.785191702754956e+01 + -6.814604053706709e+01 + -6.844132833746227e+01 + -6.873778363702108e+01 + -6.903540965000001e+01 + -6.933420959649558e+01 + -6.963418670433917e+01 + -6.993534420146086e+01 + -7.023768531400407e+01 + -7.054121327999999e+01 + -7.084593134500037e+01 + -7.115184275353549e+01 + -7.145895075432828e+01 + -7.176725860332733e+01 + -7.207676955999989e+01 + -7.238748688676215e+01 + -7.269941385037480e+01 + -7.301255372170849e+01 + -7.332690977697268e+01 + -7.364248530000000e+01 + -7.395928357816831e+01 + -7.427730789465394e+01 + -7.459656154154558e+01 + -7.491704782593285e+01 + -7.523877004999987e+01 + -7.556173151464178e+01 + -7.588593553169946e+01 + -7.621138541944168e+01 + -7.653808450163682e+01 + -7.686603610000000e+01 + -7.719524353805981e+01 + -7.752571015313293e+01 + -7.785743928572181e+01 + -7.819043427631229e+01 + -7.852469846999988e+01 + -7.886023521608357e+01 + -7.919704786882774e+01 + -7.953513978918475e+01 + -7.987451434652237e+01 + -8.021517491000000e+01 + -8.055712484895521e+01 + -8.090036753598372e+01 + -8.124490635250547e+01 + -8.159074469172391e+01 + -8.193788593999987e+01 + -8.228633348416081e+01 + -8.263609073313938e+01 + -8.298716109541151e+01 + -8.333954797103145e+01 + -8.369325477000000e+01 + -8.404828491028280e+01 + -8.440464181412719e+01 + -8.476232890793086e+01 + -8.512134962287996e+01 + -8.548170738999985e+01 + -8.584340564352934e+01 + -8.620644783199920e+01 + -8.657083740482393e+01 + -8.693657780824788e+01 + -8.730367250000000e+01 + -8.767212494371194e+01 + -8.804193859639375e+01 + -8.841311692593891e+01 + -8.878566341870939e+01 + -8.915958154999986e+01 + -8.953487479168578e+01 + -8.991154663471849e+01 + -9.028960057395298e+01 + -9.066904010318096e+01 + -9.104986872000001e+01 + -9.143208992686800e+01 + -9.181570723560402e+01 + -9.220072416114067e+01 + -9.258714421980083e+01 + -9.297497092999986e+01 + -9.336420781525349e+01 + -9.375485841426487e+01 + -9.414692626313253e+01 + -9.454041488965315e+01 + -9.493532784000000e+01 + -9.533166867059026e+01 + -9.572944092950100e+01 + -9.612864816948191e+01 + -9.652929395407540e+01 + -9.693138184999984e+01 + -9.733491542620831e+01 + -9.773989825487583e+01 + -9.814633391500892e+01 + -9.855422599474231e+01 + -9.896357808000000e+01 + -9.937439375553778e+01 + -9.978667660951464e+01 + -1.002004302498873e+02 + -1.006156583113964e+02 + -1.010323643999998e+02 + -1.014505521033422e+02 + -1.018702250167552e+02 + -1.022913867648504e+02 + -1.027140410123223e+02 + -1.031381914000000e+02 + -1.035638415489902e+02 + -1.039909950728156e+02 + -1.044196556182856e+02 + -1.048498268814360e+02 + -1.052815124999998e+02 + -1.057147160851634e+02 + -1.061494413040264e+02 + -1.065856918490972e+02 + -1.070234714266025e+02 + -1.074627837000000e+02 + -1.079036323073108e+02 + -1.083460208994117e+02 + -1.087899531790557e+02 + -1.092354329168266e+02 + -1.096824637999998e+02 + -1.101310494754678e+02 + -1.105811936593542e+02 + -1.110329000832090e+02 + -1.114861724712110e+02 + -1.119410145000000e+02 + -1.123974298499644e+02 + -1.128554223365672e+02 + -1.133149957276709e+02 + -1.137761536721920e+02 + -1.142388998999998e+02 + -1.147032381951767e+02 + -1.151691723223398e+02 + -1.156367060267053e+02 + -1.161058430340008e+02 + -1.165765871000000e+02 + -1.170489420136372e+02 + -1.175229116129598e+02 + -1.179984996644491e+02 + -1.184757098197581e+02 + -1.189545458999998e+02 + -1.194350118004852e+02 + -1.199171112501401e+02 + -1.204008480035419e+02 + -1.208862259161727e+02 + -1.213732488000000e+02 + -1.218619204419225e+02 + -1.223522446637547e+02 + -1.228442252993658e+02 + -1.233378661832514e+02 + -1.238331710999998e+02 + -1.243301438371392e+02 + -1.248287883218059e+02 + -1.253291084262349e+02 + -1.258311078947497e+02 + -1.263347906000000e+02 + -1.268401604692280e+02 + -1.273472212887372e+02 + -1.278559768940106e+02 + -1.283664312433523e+02 + -1.288785881999998e+02 + -1.293924515790011e+02 + -1.299080252758494e+02 + -1.304253131892150e+02 + -1.309443191941241e+02 + -1.314650472000000e+02 + -1.319875011267033e+02 + -1.325116848422910e+02 + -1.330376022476014e+02 + -1.335652573069860e+02 + -1.340946538999998e+02 + -1.346257958892211e+02 + -1.351586873010075e+02 + -1.356933320835031e+02 + -1.362297340185993e+02 + -1.367678971000000e+02 + -1.373078254089903e+02 + -1.378495227933144e+02 + -1.383929931342302e+02 + -1.389382404506380e+02 + -1.394852686999998e+02 + -1.400340818144768e+02 + -1.405846838138316e+02 + -1.411370786853171e+02 + -1.416912703410207e+02 + -1.422472628000000e+02 + -1.428050601118040e+02 + -1.433646661562381e+02 + -1.439260848884736e+02 + -1.444893204354731e+02 + -1.450543767999998e+02 + -1.456212579173106e+02 + -1.461899678121767e+02 + -1.467605105156440e+02 + -1.473328900351724e+02 + -1.479071104000000e+02 + -1.484831756560318e+02 + -1.490610898537698e+02 + -1.496408570188282e+02 + -1.502224811435522e+02 + -1.508059663000000e+02 + -1.513913165938295e+02 + -1.519785360517816e+02 + -1.525676287092765e+02 + -1.531585986433134e+02 + -1.537514498999998e+02 + -1.543461865304275e+02 + -1.549428126935839e+02 + -1.555413324987964e+02 + -1.561417499433451e+02 + -1.567440691000000e+02 + -1.573482940967384e+02 + -1.579544290618548e+02 + -1.585624781173004e+02 + -1.591724453723425e+02 + -1.597843348999998e+02 + -1.603981507727703e+02 + -1.610138971515657e+02 + -1.616315781879854e+02 + -1.622511979884414e+02 + -1.628727607000000e+02 + -1.634962704824068e+02 + -1.641217314319286e+02 + -1.647491476962519e+02 + -1.653785235159056e+02 + -1.660098629999998e+02 + -1.666431702132707e+02 + -1.672784494025457e+02 + -1.679157047805161e+02 + -1.685549404452619e+02 + -1.691961606000000e+02 + -1.698393694891200e+02 + -1.704845712252935e+02 + -1.711317699673001e+02 + -1.717809699904770e+02 + -1.724321754999998e+02 + -1.730853906636835e+02 + -1.737406197069920e+02 + -1.743978668390772e+02 + -1.750571362267288e+02 + -1.757184321000000e+02 + -1.763817587329360e+02 + -1.770471204015179e+02 + -1.777145213342737e+02 + -1.783839656916257e+02 + -1.790554576999997e+02 + -1.797290016348361e+02 + -1.804046017812607e+02 + -1.810822624074922e+02 + -1.817619877551305e+02 + -1.824437821000000e+02 + -1.831276497291853e+02 + -1.838135948809234e+02 + -1.845016218201234e+02 + -1.851917348679873e+02 + -1.858839382999998e+02 + -1.865782363787204e+02 + -1.872746334451072e+02 + -1.879731338116912e+02 + -1.886737417231232e+02 + -1.893764615000000e+02 + -1.900812975069574e+02 + -1.907882540726982e+02 + -1.914973354967214e+02 + -1.922085460544087e+02 + -1.929218900999998e+02 + -1.936373720194894e+02 + -1.943549961169706e+02 + -1.950747667123967e+02 + -1.957966881820183e+02 + -1.965207649000000e+02 + -1.972470012275908e+02 + -1.979754014928308e+02 + -1.987059700466389e+02 + -1.994387112870401e+02 + -2.001736295999997e+02 + -2.009107293657797e+02 + -2.016500149830146e+02 + -2.023914908231823e+02 + -2.031351612174878e+02 + -2.038810306000000e+02 + -2.046291034502658e+02 + -2.053793841544099e+02 + -2.061318770718700e+02 + -2.068865865659776e+02 + -2.076435170999997e+02 + -2.084026731731890e+02 + -2.091640591716557e+02 + -2.099276794995817e+02 + -2.106935386300690e+02 + -2.114616410000000e+02 + -2.122319910255027e+02 + -2.130045931490896e+02 + -2.137794518362718e+02 + -2.145565715736208e+02 + -2.153359567999997e+02 + -2.161176119510612e+02 + -2.169015415764713e+02 + -2.176877501734888e+02 + -2.184762421224458e+02 + -2.192670219000000e+02 + -2.200600940505141e+02 + -2.208554631128035e+02 + -2.216531335679009e+02 + -2.224531098207831e+02 + -2.232553963999997e+02 + -2.240599978997985e+02 + -2.248669188401317e+02 + -2.256761637092654e+02 + -2.264877369816284e+02 + -2.273016432000000e+02 + -2.281178869415660e+02 + -2.289364727411434e+02 + -2.297574051254879e+02 + -2.305806886273285e+02 + -2.314063277999997e+02 + -2.322343272067604e+02 + -2.330646913987978e+02 + -2.338974249389632e+02 + -2.347325324124834e+02 + -2.355700184000000e+02 + -2.364098874788788e+02 + -2.372521442301691e+02 + -2.380967932316097e+02 + -2.389438390563173e+02 + -2.397932862999997e+02 + -2.406451395699314e+02 + -2.414994034603918e+02 + -2.423560825773335e+02 + -2.432151815494090e+02 + -2.440767050000000e+02 + -2.449406575494712e+02 + -2.458070438246558e+02 + -2.466758684438366e+02 + -2.475471360143982e+02 + -2.484208511999996e+02 + -2.492970186823550e+02 + -2.501756430678903e+02 + -2.510567289807632e+02 + -2.519402811010791e+02 + -2.528263041000000e+02 + -2.537148026392507e+02 + -2.546057813786489e+02 + -2.554992449908618e+02 + -2.563951981686694e+02 + -2.572936455999996e+02 + -2.581945919701986e+02 + -2.590980419711499e+02 + -2.600040002856254e+02 + -2.609124715849065e+02 + -2.618234606000000e+02 + -2.627369720784054e+02 + -2.636530106775519e+02 + -2.645715811006028e+02 + -2.654926881465747e+02 + -2.664163364999997e+02 + -2.673425308135112e+02 + -2.682712759281322e+02 + -2.692025766270102e+02 + -2.701364375396030e+02 + -2.710728634000000e+02 + -2.720118590149941e+02 + -2.729534291763713e+02 + -2.738975786300543e+02 + -2.748443120676544e+02 + -2.757936342999997e+02 + -2.767455501787848e+02 + -2.777000643997414e+02 + -2.786571816989448e+02 + -2.796169069311608e+02 + -2.805792449000000e+02 + -2.815442003833780e+02 + -2.825117782146733e+02 + -2.834819832031981e+02 + -2.844548201023671e+02 + -2.854302936999997e+02 + -2.864084088239409e+02 + -2.873891703596309e+02 + -2.883725831483384e+02 + -2.893586519481404e+02 + -2.903473816000000e+02 + -2.913387769892773e+02 + -2.923328429461174e+02 + -2.933295843055886e+02 + -2.943290059327143e+02 + -2.953311126999996e+02 + -2.963359094688269e+02 + -2.973434010446480e+02 + -2.983535922946355e+02 + -2.993664881931061e+02 + -3.003820936000000e+02 + -3.014004133189056e+02 + -3.024214522481863e+02 + -3.034452153213495e+02 + -3.044717074825456e+02 + -3.055009335999997e+02 + -3.065328985120693e+02 + -3.075676071368449e+02 + -3.086050644279766e+02 + -3.096452753553112e+02 + -3.106882447999996e+02 + -3.117339776182681e+02 + -3.127824787990806e+02 + -3.138337533166528e+02 + -3.148878060732492e+02 + -3.159446419999996e+02 + -3.170042660531784e+02 + -3.180666831989414e+02 + -3.191318984100575e+02 + -3.201999166629245e+02 + -3.212707428999996e+02 + -3.223443820580361e+02 + -3.234208391391681e+02 + -3.245001191331626e+02 + -3.255822269890704e+02 + -3.266671677000000e+02 + -3.277549462905472e+02 + -3.288455677875165e+02 + -3.299390371822755e+02 + -3.310353594166140e+02 + -3.321345394999996e+02 + -3.332365824854958e+02 + -3.343414934139734e+02 + -3.354492773045120e+02 + -3.365599391517927e+02 + -3.376734840000000e+02 + -3.387899169180687e+02 + -3.399092429402671e+02 + -3.410314670886827e+02 + -3.421565943845668e+02 + -3.432846298999996e+02 + -3.444155787253185e+02 + -3.455494458929433e+02 + -3.466862364711880e+02 + -3.478259555971263e+02 + -3.489686083000000e+02 + -3.501141995805723e+02 + -3.512627346186799e+02 + -3.524142185495274e+02 + -3.535686563776662e+02 + -3.547260531999996e+02 + -3.558864141649047e+02 + -3.570497443588797e+02 + -3.582160488781844e+02 + -3.593853328562213e+02 + -3.605576014000000e+02 + -3.617328596203877e+02 + -3.629111127187760e+02 + -3.640923658296630e+02 + -3.652766239636252e+02 + -3.664638922999996e+02 + -3.676541760949210e+02 + -3.688474804479435e+02 + -3.700438104641370e+02 + -3.712431713192604e+02 + -3.724455682000000e+02 + -3.736510062917217e+02 + -3.748594907617601e+02 + -3.760710267615897e+02 + -3.772856194324323e+02 + -3.785032739999995e+02 + -3.797239957155368e+02 + -3.809477897125092e+02 + -3.821746611611994e+02 + -3.834046153278488e+02 + -3.846376574000000e+02 + -3.858737925371631e+02 + -3.871130260109560e+02 + -3.883553630688227e+02 + -3.896008088834330e+02 + -3.908493686999996e+02 + -3.921010477895665e+02 + -3.933558513233559e+02 + -3.946137845264227e+02 + -3.958748527376295e+02 + -3.971390612000000e+02 + -3.984064151109821e+02 + -3.996769197585321e+02 + -4.009505804131749e+02 + -4.022274022910074e+02 + -4.035073906999995e+02 + -4.047905509809469e+02 + -4.060768883569506e+02 + -4.073664080804837e+02 + -4.086591154923789e+02 + -4.099550159000000e+02 + -4.112541145837781e+02 + -4.125564168232350e+02 + -4.138619279264533e+02 + -4.151706532436620e+02 + -4.164825980999995e+02 + -4.177977678009782e+02 + -4.191161676484082e+02 + -4.204378029703449e+02 + -4.217626791355866e+02 + -4.230908015000000e+02 + -4.244221753941545e+02 + -4.257568060915354e+02 + -4.270946989435893e+02 + -4.284358594322071e+02 + -4.297802928999995e+02 + -4.311280046181774e+02 + -4.324789999621266e+02 + -4.338332843406102e+02 + -4.351908631680312e+02 + -4.365517418000000e+02 + -4.379159255766193e+02 + -4.392834199290754e+02 + -4.406542302894571e+02 + -4.420283620541414e+02 + -4.434058205999995e+02 + -4.447866113008026e+02 + -4.461707395611302e+02 + -4.475582108324218e+02 + -4.489490306166049e+02 + -4.503432043000000e+02 + -4.517407372272212e+02 + -4.531416348857516e+02 + -4.545459027469901e+02 + -4.559535462055746e+02 + -4.573645706999994e+02 + -4.587789817092769e+02 + -4.601967847429616e+02 + -4.616179852648949e+02 + -4.630425886631084e+02 + -4.644706004000000e+02 + -4.659020259847504e+02 + -4.673368709042424e+02 + -4.687751406384571e+02 + -4.702168406679589e+02 + -4.716619764999995e+02 + -4.731105536500806e+02 + -4.745625775980045e+02 + -4.760180538297375e+02 + -4.774769878558138e+02 + -4.789393852000000e+02 + -4.804052513950263e+02 + -4.818745919803221e+02 + -4.833474124743740e+02 + -4.848237183656923e+02 + -4.863035151999994e+02 + -4.877868085453354e+02 + -4.892736039052198e+02 + -4.907639068268139e+02 + -4.922577229425711e+02 + -4.937550578000000e+02 + -4.952559169027076e+02 + -4.967603058178435e+02 + -4.982682301416788e+02 + -4.997796954853789e+02 + -5.012947073999994e+02 + -5.028132714194502e+02 + -5.043353931676896e+02 + -5.058610782730644e+02 + -5.073903323321115e+02 + -5.089231609000000e+02 + -5.104595695386481e+02 + -5.119995639358939e+02 + -5.135431497453484e+02 + -5.150903325247945e+02 + -5.166411178999995e+02 + -5.181955115300345e+02 + -5.197535190105982e+02 + -5.213151459679927e+02 + -5.228803980937674e+02 + -5.244492810000000e+02 + -5.260218002787250e+02 + -5.275979616609960e+02 + -5.291777708470047e+02 + -5.307612334400695e+02 + -5.323483550999995e+02 + -5.339391415258532e+02 + -5.355335984055753e+02 + -5.371317314077364e+02 + -5.387335461786030e+02 + -5.403390484000000e+02 + -5.419482437848817e+02 + -5.435611380730882e+02 + -5.451777369746786e+02 + -5.467980461477442e+02 + -5.484220712999994e+02 + -5.500498181706811e+02 + -5.516812924850077e+02 + -5.533164999617454e+02 + -5.549554463178829e+02 + -5.565981372999994e+02 + -5.582445786620732e+02 + -5.598947761113077e+02 + -5.615487353817318e+02 + -5.632064622631164e+02 + -5.648679624999994e+02 + -5.665332418238921e+02 + -5.682023060429622e+02 + -5.698751609401672e+02 + -5.715518122344557e+02 + -5.732322656999993e+02 + -5.749165271427032e+02 + -5.766046023386582e+02 + -5.782964970855437e+02 + -5.799922172195480e+02 + -5.816917684999994e+02 + -5.833951566715182e+02 + -5.851023876259811e+02 + -5.868134672043320e+02 + -5.885284011210334e+02 + -5.902471951999993e+02 + -5.919698553287222e+02 + -5.936963873375214e+02 + -5.954267970459595e+02 + -5.971610902818754e+02 + -5.988992728999993e+02 + -6.006413507613062e+02 + -6.023873296844711e+02 + -6.041372155081436e+02 + -6.058910141182311e+02 + -6.076487313999993e+02 + -6.094103732273231e+02 + -6.111759454411804e+02 + -6.129454539033432e+02 + -6.147189045204622e+02 + -6.164963032000001e+02 + -6.182776558378988e+02 + -6.200629682925465e+02 + -6.218522464515877e+02 + -6.236454962577363e+02 + -6.254427235999993e+02 + -6.272439343587159e+02 + -6.290491345295636e+02 + -6.308583300588987e+02 + -6.326715267788347e+02 + -6.344887306000001e+02 + -6.363099474872770e+02 + -6.381351833941451e+02 + -6.399644442896572e+02 + -6.417977361628546e+02 + -6.436350648999993e+02 + -6.454764363684590e+02 + -6.473218566292518e+02 + -6.491713317125350e+02 + -6.510248675271151e+02 + -6.528824700000000e+02 + -6.547441451008573e+02 + -6.566098988943567e+02 + -6.584797373808955e+02 + -6.603536664383599e+02 + -6.622316920999993e+02 + -6.641138204605409e+02 + -6.660000574329696e+02 + -6.678904089961051e+02 + -6.697848812885560e+02 + -6.716834803000000e+02 + -6.735862119462199e+02 + -6.754930822688247e+02 + -6.774040973560088e+02 + -6.793192633079277e+02 + -6.812385860999992e+02 + -6.831620716800010e+02 + -6.850897262089175e+02 + -6.870215557906422e+02 + -6.889575663679858e+02 + -6.908977640000001e+02 + -6.928421548145254e+02 + -6.947907448778678e+02 + -6.967435402481692e+02 + -6.987005469959722e+02 + -7.006617711999993e+02 + -7.026272189534540e+02 + -7.045968963883171e+02 + -7.065708096021641e+02 + -7.085489646318440e+02 + -7.105313676000000e+02 + -7.125180246666655e+02 + -7.145089419064375e+02 + -7.165041254104643e+02 + -7.185035813280965e+02 + -7.205073157999993e+02 + -7.225153349567163e+02 + -7.245276449240899e+02 + -7.265442518464891e+02 + -7.285651618951177e+02 + -7.305903812000000e+02 + -7.326199158832851e+02 + -7.346537721472801e+02 + -7.366919561576724e+02 + -7.387344740033407e+02 + -7.407813318999993e+02 + -7.428325361060332e+02 + -7.448880927069067e+02 + -7.469480078569493e+02 + -7.490122878703525e+02 + -7.510809389000000e+02 + -7.531539670309830e+02 + -7.552313785295614e+02 + -7.573131796529314e+02 + -7.593993765758833e+02 + -7.614899754999992e+02 + -7.635849826556215e+02 + -7.656844042998581e+02 + -7.677882466626152e+02 + -7.698965159244972e+02 + -7.720092183000000e+02 + -7.741263600422619e+02 + -7.762479474594095e+02 + -7.783739868214470e+02 + -7.805044843222228e+02 + -7.826394461999992e+02 + -7.847788787331309e+02 + -7.869227882276110e+02 + -7.890711809493379e+02 + -7.912240631015031e+02 + -7.933814410000000e+02 + -7.955433210068886e+02 + -7.977097093640062e+02 + -7.998806123337483e+02 + -8.020560362523712e+02 + -8.042359873999992e+02 + -8.064204720446323e+02 + -8.086094965664213e+02 + -8.108030673053524e+02 + -8.130011905039750e+02 + -8.152038725000000e+02 + -8.174111196729889e+02 + -8.196229383028856e+02 + -8.218393347070877e+02 + -8.240603152936806e+02 + -8.262858863999992e+02 + -8.285160543327190e+02 + -8.307508254786889e+02 + -8.329902062108882e+02 + -8.352342028550189e+02 + -8.374828218000000e+02 + -8.397360694583805e+02 + -8.419939521636865e+02 + -8.442564762682848e+02 + -8.465236481828920e+02 + -8.487954742999991e+02 + -8.510719610066617e+02 + -8.533531147279726e+02 + -8.556389418758610e+02 + -8.579294488297842e+02 + -8.602246420000000e+02 + -8.625245278111681e+02 + -8.648291126593311e+02 + -8.671384029773127e+02 + -8.694524052578884e+02 + -8.717711258999991e+02 + -8.740945712693658e+02 + -8.764227478547659e+02 + -8.787556621558366e+02 + -8.810933206360036e+02 + -8.834357297000000e+02 + -8.857828957473700e+02 + -8.881348253000285e+02 + -8.904915248639498e+02 + -8.928530008754832e+02 + -8.952192597999992e+02 + -8.975903081273602e+02 + -8.999661523549036e+02 + -9.023467989915350e+02 + -9.047322545556517e+02 + -9.071225255000001e+02 + -9.095176182642568e+02 + -9.119175394073789e+02 + -9.143222954857910e+02 + -9.167318930073794e+02 + -9.191463384999990e+02 + -9.215656384936522e+02 + -9.239897994653498e+02 + -9.264188279352956e+02 + -9.288527305046009e+02 + -9.312915136999991e+02 + -9.337351840217814e+02 + -9.361837480744202e+02 + -9.386372124507991e+02 + -9.410955836845210e+02 + -9.435588682999991e+02 + -9.460270728408829e+02 + -9.485002039375468e+02 + -9.509782681943145e+02 + -9.534612721474879e+02 + -9.559492223999991e+02 + -9.584421255818636e+02 + -9.609399882443452e+02 + -9.634428169536077e+02 + -9.659506183300578e+02 + -9.684633989999990e+02 + -9.709811655949806e+02 + -9.735039247636544e+02 + -9.760316831188370e+02 + -9.785644472182881e+02 + -9.811022236999991e+02 + -9.836450192407020e+02 + -9.861928404528198e+02 + -9.887456939756197e+02 + -9.913035865101050e+02 + -9.938665246999991e+02 + -9.964345151588749e+02 + -9.990075645416573e+02 + -1.001585679508894e+03 + -1.004168866725833e+03 + -1.006757132999999e+03 + -1.009350485088950e+03 + -1.011948929193798e+03 + -1.014552472017065e+03 + -1.017161121149714e+03 + -1.019774882999999e+03 + -1.022393763477281e+03 + -1.025017769745607e+03 + -1.027646908706880e+03 + -1.030281186449000e+03 + -1.032920609999999e+03 + -1.035565186772393e+03 + -1.038214923106640e+03 + -1.040869825393126e+03 + -1.043529900494233e+03 + -1.046195154999999e+03 + -1.048865595518282e+03 + -1.051541229525064e+03 + -1.054222063914517e+03 + -1.056908104442841e+03 + -1.059599357999999e+03 + -1.062295832060889e+03 + -1.064997533253958e+03 + -1.067704468147946e+03 + -1.070416643533965e+03 + -1.073134066000000e+03 + -1.075856742179453e+03 + -1.078584679453085e+03 + -1.081317884808236e+03 + -1.084056364390976e+03 + -1.086800124999999e+03 + -1.089549173751501e+03 + -1.092303517161580e+03 + -1.095063162007773e+03 + -1.097828115663242e+03 + -1.100598385000000e+03 + -1.103373976558467e+03 + -1.106154896984741e+03 + -1.108941153067517e+03 + -1.111732751768433e+03 + -1.114529699999999e+03 + -1.117332004656191e+03 + -1.120139672719820e+03 + -1.122952711033347e+03 + -1.125771126202256e+03 + -1.128594925000000e+03 + -1.131424114391635e+03 + -1.134258701616623e+03 + -1.137098693726371e+03 + -1.139944097362279e+03 + -1.142794918999999e+03 + -1.145651165298781e+03 + -1.148512843975744e+03 + -1.151379962155369e+03 + -1.154252525751212e+03 + -1.157130542000000e+03 + -1.160014018647746e+03 + -1.162902961796701e+03 + -1.165797378002409e+03 + -1.168697275086654e+03 + -1.171602659999999e+03 + -1.174513539162380e+03 + -1.177429919411751e+03 + -1.180351807863387e+03 + -1.183279211838743e+03 + -1.186212138000000e+03 + -1.189150592852472e+03 + -1.192094584016226e+03 + -1.195044118757429e+03 + -1.197999203418656e+03 + -1.200959844999999e+03 + -1.203926050837031e+03 + -1.206897827704460e+03 + -1.209875182550288e+03 + -1.212858122751144e+03 + -1.215846655000000e+03 + -1.218840785789916e+03 + -1.221840522671824e+03 + -1.224845872987532e+03 + -1.227856843385819e+03 + -1.230873440999999e+03 + -1.233895673111333e+03 + -1.236923546192640e+03 + -1.239957067026041e+03 + -1.242996243170224e+03 + -1.246041082000000e+03 + -1.249091590572761e+03 + -1.252147775334870e+03 + -1.255209643154299e+03 + -1.258277201722480e+03 + -1.261350457999999e+03 + -1.264429418673061e+03 + -1.267514091394071e+03 + -1.270604483350415e+03 + -1.273700600740823e+03 + -1.276802451000000e+03 + -1.279910042060014e+03 + -1.283023380429426e+03 + -1.286142472688095e+03 + -1.289267326090673e+03 + -1.292397947999999e+03 + -1.295534345774196e+03 + -1.298676526619691e+03 + -1.301824497524618e+03 + -1.304978265240164e+03 + -1.308137837000000e+03 + -1.311303220289759e+03 + -1.314474422298625e+03 + -1.317651449999705e+03 + -1.320834310180745e+03 + -1.324023009999999e+03 + -1.327217556910311e+03 + -1.330417958531748e+03 + -1.333624221890711e+03 + -1.336836353161044e+03 + -1.340054360000000e+03 + -1.343278250622039e+03 + -1.346508031450468e+03 + -1.349743709083262e+03 + -1.352985291084887e+03 + -1.356232784999999e+03 + -1.359486198143600e+03 + -1.362745537187869e+03 + -1.366010809195683e+03 + -1.369282022032803e+03 + -1.372559183000000e+03 + -1.375842299046690e+03 + -1.379131377356324e+03 + -1.382426425045151e+03 + -1.385727449081319e+03 + -1.389034456999999e+03 + -1.392347456524483e+03 + -1.395666454635196e+03 + -1.398991458464414e+03 + -1.402322475610563e+03 + -1.405659513000000e+03 + -1.409002577344138e+03 + -1.412351676333446e+03 + -1.415706817605260e+03 + -1.419068008353351e+03 + -1.422435255999999e+03 + -1.425808567964313e+03 + -1.429187950969556e+03 + -1.432573411998037e+03 + -1.435964958719179e+03 + -1.439362599000000e+03 + -1.442766340439504e+03 + -1.446176189225631e+03 + -1.449592152360744e+03 + -1.453014238529275e+03 + -1.456442454999999e+03 + -1.459876808245435e+03 + -1.463317305593544e+03 + -1.466763954627232e+03 + -1.470216762966463e+03 + -1.473675738000000e+03 + -1.477140887005819e+03 + -1.480612217422037e+03 + -1.484089736511155e+03 + -1.487573451238554e+03 + -1.491063368999999e+03 + -1.494559497483912e+03 + -1.498061844364458e+03 + -1.501570417048641e+03 + -1.505085222548716e+03 + -1.508606267999999e+03 + -1.512133560790714e+03 + -1.515667108906144e+03 + -1.519206919999463e+03 + -1.522753000996140e+03 + -1.526305358999999e+03 + -1.529864001422905e+03 + -1.533428936276007e+03 + -1.537000171179230e+03 + -1.540577712963951e+03 + -1.544161568999999e+03 + -1.547751747018138e+03 + -1.551348254621497e+03 + -1.554951099280540e+03 + -1.558560288310355e+03 + -1.562175828999999e+03 + -1.565797728689906e+03 + -1.569425994958102e+03 + -1.573060635400619e+03 + -1.576701657553543e+03 + -1.580349068999999e+03 + -1.584002877314684e+03 + -1.587663089901193e+03 + -1.591329713983250e+03 + -1.595002756657312e+03 + -1.598682225999999e+03 + -1.602368130282464e+03 + -1.606060476003810e+03 + -1.609759270166310e+03 + -1.613464521194204e+03 + -1.617176236999999e+03 + -1.620894424965112e+03 + -1.624619091996676e+03 + -1.628350245392540e+03 + -1.632087893195996e+03 + -1.635832042999999e+03 + -1.639582702219653e+03 + -1.643339878880967e+03 + -1.647103580589531e+03 + -1.650873814125992e+03 + -1.654650586999999e+03 + -1.658433907212865e+03 + -1.662223782656940e+03 + -1.666020220927786e+03 + -1.669823229228366e+03 + -1.673632814999999e+03 + -1.677448985879331e+03 + -1.681271749582079e+03 + -1.685101113981527e+03 + -1.688937087086922e+03 + -1.692779675999999e+03 + -1.696628887589857e+03 + -1.700484730162859e+03 + -1.704347211664743e+03 + -1.708216339003609e+03 + -1.712092119999999e+03 + -1.715974562928047e+03 + -1.719863675285195e+03 + -1.723759464515762e+03 + -1.727661938269359e+03 + -1.731571103999999e+03 + -1.735486969207614e+03 + -1.739409542118944e+03 + -1.743338830603560e+03 + -1.747274841733617e+03 + -1.751217582999999e+03 + -1.755167062315964e+03 + -1.759123288009624e+03 + -1.763086267739076e+03 + -1.767056008086542e+03 + -1.771032516999999e+03 + -1.775015803098183e+03 + -1.779005873908525e+03 + -1.783002736756524e+03 + -1.787006399126180e+03 + -1.791016869000000e+03 + -1.795034154506543e+03 + -1.799058263107071e+03 + -1.803089202321619e+03 + -1.807126980015989e+03 + -1.811171603999998e+03 + -1.815223082025504e+03 + -1.819281421835726e+03 + -1.823346631247284e+03 + -1.827418718154363e+03 + -1.831497690000000e+03 + -1.835583554215247e+03 + -1.839676319355180e+03 + -1.843775993459518e+03 + -1.847882583413063e+03 + -1.851996096999998e+03 + -1.856116542493618e+03 + -1.860243927542824e+03 + -1.864378259929794e+03 + -1.868519547846306e+03 + -1.872667799000000e+03 + -1.876823020804590e+03 + -1.880985220849939e+03 + -1.885154407199114e+03 + -1.889330588466962e+03 + -1.893513771999998e+03 + -1.897703964771626e+03 + -1.901901175652285e+03 + -1.906105412964281e+03 + -1.910316683519642e+03 + -1.914534995000000e+03 + -1.918760355678869e+03 + -1.922992773607248e+03 + -1.927232256642219e+03 + -1.931478812455566e+03 + -1.935732448999998e+03 + -1.939993174351127e+03 + -1.944260996306766e+03 + -1.948535922544343e+03 + -1.952817960710870e+03 + -1.957107119000000e+03 + -1.961403405774432e+03 + -1.965706828656891e+03 + -1.970017395435863e+03 + -1.974335114379128e+03 + -1.978659992999998e+03 + -1.982992038634364e+03 + -1.987331259958815e+03 + -1.991677665406797e+03 + -1.996031262526477e+03 + -2.000392059000000e+03 + -2.004760062662544e+03 + -2.009135281420714e+03 + -2.013517723394651e+03 + -2.017907396963966e+03 + -2.022304309999998e+03 + -2.026708470129302e+03 + -2.031119885371980e+03 + -2.035538563783372e+03 + -2.039964513323241e+03 + -2.044397742000000e+03 + -2.048838257821666e+03 + -2.053286068652342e+03 + -2.057741182613919e+03 + -2.062203608203423e+03 + -2.066673352999998e+03 + -2.071150424296029e+03 + -2.075634830690161e+03 + -2.080126580708826e+03 + -2.084625682242698e+03 + -2.089132143000000e+03 + -2.093645970720743e+03 + -2.098167173603271e+03 + -2.102695759983049e+03 + -2.107231738184183e+03 + -2.111775115999998e+03 + -2.116325901057278e+03 + -2.120884101683680e+03 + -2.125449726108530e+03 + -2.130022782163747e+03 + -2.134603278000000e+03 + -2.139191221924691e+03 + -2.143786621957349e+03 + -2.148389485981318e+03 + -2.152999821836896e+03 + -2.157617637999998e+03 + -2.162242943074696e+03 + -2.166875744534069e+03 + -2.171516050262848e+03 + -2.176163869133622e+03 + -2.180819209000000e+03 + -2.185482077327040e+03 + -2.190152482871076e+03 + -2.194830433950524e+03 + -2.199515937814626e+03 + -2.204209002999998e+03 + -2.208909638535144e+03 + -2.213617851857877e+03 + -2.218333650781778e+03 + -2.223057044225780e+03 + -2.227788040000000e+03 + -2.232526645541351e+03 + -2.237272869895315e+03 + -2.242026721580949e+03 + -2.246788207788726e+03 + -2.251557336999999e+03 + -2.256334118250544e+03 + -2.261118559161676e+03 + -2.265910667352767e+03 + -2.270710451040453e+03 + -2.275517918999998e+03 + -2.280333080014267e+03 + -2.285155941593499e+03 + -2.289986511539142e+03 + -2.294824798563590e+03 + -2.299670810999998e+03 + -2.304524556927484e+03 + -2.309386044592275e+03 + -2.314255282195762e+03 + -2.319132277836808e+03 + -2.324017039999998e+03 + -2.328909577220459e+03 + -2.333809897224200e+03 + -2.338718008145149e+03 + -2.343633918994289e+03 + -2.348557637999998e+03 + -2.353489172912608e+03 + -2.358428531797252e+03 + -2.363375723128402e+03 + -2.368330755820120e+03 + -2.373293637999998e+03 + -2.378264377427878e+03 + -2.383242982526706e+03 + -2.388229461889857e+03 + -2.393223824063392e+03 + -2.398226076999998e+03 + -2.403236228525699e+03 + -2.408254287477974e+03 + -2.413280262494937e+03 + -2.418314161547432e+03 + -2.423355992999998e+03 + -2.428405765403690e+03 + -2.433463486890615e+03 + -2.438529165661329e+03 + -2.443602810190047e+03 + -2.448684428999998e+03 + -2.453774030522361e+03 + -2.458871622761939e+03 + -2.463977214083831e+03 + -2.469090813490053e+03 + -2.474212428999998e+03 + -2.479342068313477e+03 + -2.484479740528551e+03 + -2.489625454539379e+03 + -2.494779218375099e+03 + -2.499941039999998e+03 + -2.505110927624773e+03 + -2.510288890409285e+03 + -2.515474937052364e+03 + -2.520669075212765e+03 + -2.525871312999998e+03 + -2.531081658990217e+03 + -2.536300122220549e+03 + -2.541526711425658e+03 + -2.546761434689014e+03 + -2.552004299999998e+03 + -2.557255315555890e+03 + -2.562514490474060e+03 + -2.567781833501355e+03 + -2.573057352510476e+03 + -2.578341055999998e+03 + -2.583632952807378e+03 + -2.588933051312229e+03 + -2.594241359879810e+03 + -2.599557887014108e+03 + -2.604882640999998e+03 + -2.610215630196411e+03 + -2.615556863841929e+03 + -2.620906350516468e+03 + -2.626264097579904e+03 + -2.631630113999998e+03 + -2.637004409331177e+03 + -2.642386991059378e+03 + -2.647777867311749e+03 + -2.653177047878767e+03 + -2.658584540999998e+03 + -2.664000354190328e+03 + -2.669424496446296e+03 + -2.674856976816473e+03 + -2.680297803844182e+03 + -2.685746985999998e+03 + -2.691204531699670e+03 + -2.696670449221115e+03 + -2.702144747143579e+03 + -2.707627434527642e+03 + -2.713118519999998e+03 + -2.718618011850782e+03 + -2.724125918263655e+03 + -2.729642247949266e+03 + -2.735167010382318e+03 + -2.740700213999998e+03 + -2.746241866699575e+03 + -2.751791977076697e+03 + -2.757350554006517e+03 + -2.762917606479738e+03 + -2.768493143000000e+03 + -2.774077171851313e+03 + -2.779669701724049e+03 + -2.785270741295763e+03 + -2.790880299095185e+03 + -2.796498383999998e+03 + -2.802125004968091e+03 + -2.807760170344632e+03 + -2.813403888501788e+03 + -2.819056168106567e+03 + -2.824717018000000e+03 + -2.830386447079949e+03 + -2.836064464083062e+03 + -2.841751077483182e+03 + -2.847446295475182e+03 + -2.853150126999998e+03 + -2.858862581346520e+03 + -2.864583667200011e+03 + -2.870313392988722e+03 + -2.876051767015894e+03 + -2.881798798000000e+03 + -2.887554494917011e+03 + -2.893318866668325e+03 + -2.899091922031221e+03 + -2.904873669618204e+03 + -2.910664117999998e+03 + -2.916463275808826e+03 + -2.922271151990944e+03 + -2.928087755363771e+03 + -2.933913094445438e+03 + -2.939747178000000e+03 + -2.945590014959967e+03 + -2.951441614226611e+03 + -2.957301984588625e+03 + -2.963171134666888e+03 + -2.969049072999998e+03 + -2.974935808208847e+03 + -2.980831349417174e+03 + -2.986735705755709e+03 + -2.992648886132359e+03 + -2.998570899000000e+03 + -3.004501752659864e+03 + -3.010441455936329e+03 + -3.016390017858416e+03 + -3.022347447554316e+03 + -3.028313753999998e+03 + -3.034288946006182e+03 + -3.040273032141582e+03 + -3.046266021078663e+03 + -3.052267921739149e+03 + -3.058278743000000e+03 + -3.064298493698561e+03 + -3.070327182685398e+03 + -3.076364818827055e+03 + -3.082411411001809e+03 + -3.088466967999998e+03 + -3.094531498652569e+03 + -3.100605012177602e+03 + -3.106687517554494e+03 + -3.112779023262762e+03 + -3.118879538000000e+03 + -3.124989070695736e+03 + -3.131107630527278e+03 + -3.137235226491168e+03 + -3.143371867254447e+03 + -3.149517561999998e+03 + -3.155672319953097e+03 + -3.161836149115856e+03 + -3.168009058152651e+03 + -3.174191057129923e+03 + -3.180382155000000e+03 + -3.186582360055111e+03 + -3.192791681144167e+03 + -3.199010127274887e+03 + -3.205237707493432e+03 + -3.211474430999998e+03 + -3.217720306909038e+03 + -3.223975343611381e+03 + -3.230239549889151e+03 + -3.236512935355519e+03 + -3.242795509000000e+03 + -3.249087279482800e+03 + -3.255388255944051e+03 + -3.261698447489179e+03 + -3.268017862963467e+03 + -3.274346510999998e+03 + -3.280684400401313e+03 + -3.287031541121871e+03 + -3.293387942369432e+03 + -3.299753611888159e+03 + -3.306128559000000e+03 + -3.312512793688797e+03 + -3.318906324190636e+03 + -3.325309159228279e+03 + -3.331721308860119e+03 + -3.338142781999998e+03 + -3.344573586933190e+03 + -3.351013732692941e+03 + -3.357463228591956e+03 + -3.363922084041577e+03 + -3.370390307999997e+03 + -3.376867909193196e+03 + -3.383354896616435e+03 + -3.389851279509897e+03 + -3.396357067337695e+03 + -3.402872268999997e+03 + -3.409396893188545e+03 + -3.415930949268335e+03 + -3.422474446597307e+03 + -3.429027394238212e+03 + -3.435589800999997e+03 + -3.442161675674332e+03 + -3.448743027574159e+03 + -3.455333866086035e+03 + -3.461934200499129e+03 + -3.468544039999997e+03 + -3.475163393634110e+03 + -3.481792270131310e+03 + -3.488430678432703e+03 + -3.495078627927044e+03 + -3.501736127999997e+03 + -3.508403187927853e+03 + -3.515079816661691e+03 + -3.521766023349286e+03 + -3.528461817510494e+03 + -3.535167207999998e+03 + -3.541882203477436e+03 + -3.548606813620608e+03 + -3.555341047964333e+03 + -3.562084915431213e+03 + -3.568838424999997e+03 + -3.575601585773337e+03 + -3.582374407066678e+03 + -3.589156898185297e+03 + -3.595949068354316e+03 + -3.602750926999998e+03 + -3.609562483468940e+03 + -3.616383746264014e+03 + -3.623214724488114e+03 + -3.630055428392298e+03 + -3.636905866999997e+03 + -3.643766048793625e+03 + -3.650635983516456e+03 + -3.657515680680700e+03 + -3.664405149014039e+03 + -3.671304397999997e+03 + -3.678213437374473e+03 + -3.685132275760156e+03 + -3.692060922035369e+03 + -3.698999385901594e+03 + -3.705947676999997e+03 + -3.712905804797643e+03 + -3.719873778403308e+03 + -3.726851606752037e+03 + -3.733839298718909e+03 + -3.740836863999997e+03 + -3.747844312547997e+03 + -3.754861653211674e+03 + -3.761888895074585e+03 + -3.768926047955085e+03 + -3.775973120999997e+03 + -3.783030123074376e+03 + -3.790097063813473e+03 + -3.797173952745452e+03 + -3.804260798968512e+03 + -3.811357611999997e+03 + -3.818464401472386e+03 + -3.825581176303607e+03 + -3.832707945585785e+03 + -3.839844718954238e+03 + -3.846991505999998e+03 + -3.854148316201428e+03 + -3.861315158822798e+03 + -3.868492043129635e+03 + -3.875678978458987e+03 + -3.882875973999997e+03 + -3.890083038997269e+03 + -3.897300183310783e+03 + -3.904527416633095e+03 + -3.911764748154242e+03 + -3.919012186999997e+03 + -3.926269742452437e+03 + -3.933537424464352e+03 + -3.940815242621297e+03 + -3.948103205720294e+03 + -3.955401322999997e+03 + -3.962709604107895e+03 + -3.970028058998795e+03 + -3.977356697163350e+03 + -3.984695527326294e+03 + -3.992044558999997e+03 + -3.999403802144854e+03 + -4.006773266302151e+03 + -4.014152960820400e+03 + -4.021542894928240e+03 + -4.028943077999997e+03 + -4.036353519583599e+03 + -4.043774229513202e+03 + -4.051205217431756e+03 + -4.058646492564995e+03 + -4.066098063999997e+03 + -4.073559940995820e+03 + -4.081032133755226e+03 + -4.088514652104142e+03 + -4.096007504979870e+03 + -4.103510702000000e+03 + -4.111024253097677e+03 + -4.118548167510886e+03 + -4.126082454416708e+03 + -4.133627123225803e+03 + -4.141182184000000e+03 + -4.148747646853033e+03 + -4.156323520486682e+03 + -4.163909814098826e+03 + -4.171506538118098e+03 + -4.179113701999993e+03 + -4.186731314728609e+03 + -4.194359386189681e+03 + -4.201997926116393e+03 + -4.209646943720986e+03 + -4.217306449000000e+03 + -4.224976452122954e+03 + -4.232656961817923e+03 + -4.240347987240753e+03 + -4.248049538718602e+03 + -4.255761626000000e+03 + -4.263484258397591e+03 + -4.271217445246016e+03 + -4.278961196276451e+03 + -4.286715521743729e+03 + -4.294480431000000e+03 + -4.302255933066315e+03 + -4.310042038085233e+03 + -4.317838756007569e+03 + -4.325646096087124e+03 + -4.333464067999995e+03 + -4.341292681602012e+03 + -4.349131946225953e+03 + -4.356981871466000e+03 + -4.364842467485710e+03 + -4.372713744000000e+03 + -4.380595710372718e+03 + -4.388488375861229e+03 + -4.396391750264422e+03 + -4.404305844160006e+03 + -4.412230667000000e+03 + -4.420166227736474e+03 + -4.428112536405837e+03 + -4.436069603088062e+03 + -4.444037437481978e+03 + -4.452016049000000e+03 + -4.460005447074678e+03 + -4.468005641867199e+03 + -4.476016643204103e+03 + -4.484038460200805e+03 + -4.492071102999995e+03 + -4.500114582060231e+03 + -4.508168906299815e+03 + -4.516234085098625e+03 + -4.524310129075975e+03 + -4.532397048000000e+03 + -4.540494851104932e+03 + -4.548603547974418e+03 + -4.556723148595408e+03 + -4.564853663376500e+03 + -4.572995102000000e+03 + -4.581147473755137e+03 + -4.589310788334117e+03 + -4.597485055595826e+03 + -4.605670285497996e+03 + -4.613866488000000e+03 + -4.622073673035890e+03 + -4.630291850451470e+03 + -4.638521029781397e+03 + -4.646761220207367e+03 + -4.655012431999994e+03 + -4.663274675842531e+03 + -4.671547961173437e+03 + -4.679832297267551e+03 + -4.688127693710799e+03 + -4.696434161000000e+03 + -4.704751709752170e+03 + -4.713080348793824e+03 + -4.721420087561993e+03 + -4.729770937031085e+03 + -4.738132907000000e+03 + -4.746506006604411e+03 + -4.754890245715343e+03 + -4.763285634540516e+03 + -4.771692183448750e+03 + -4.780109901999994e+03 + -4.788538799466528e+03 + -4.796978886096308e+03 + -4.805430172162917e+03 + -4.813892667577567e+03 + -4.822366381999994e+03 + -4.830851325088147e+03 + -4.839347507060687e+03 + -4.847854938123849e+03 + -4.856373628221604e+03 + -4.864903587000000e+03 + -4.873444824101478e+03 + -4.881997349859994e+03 + -4.890561174364984e+03 + -4.899136307099045e+03 + -4.907722758000000e+03 + -4.916320537362784e+03 + -4.924929655587317e+03 + -4.933550122528648e+03 + -4.942181947260091e+03 + -4.950825139999994e+03 + -4.959479711457802e+03 + -4.968145671175524e+03 + -4.976823028809056e+03 + -4.985511794647833e+03 + -4.994211978999993e+03 + -5.002923592007461e+03 + -5.011646643245516e+03 + -5.020381142480443e+03 + -5.029127099988444e+03 + -5.037884526000000e+03 + -5.046653430641881e+03 + -5.055433823857634e+03 + -5.064225715527905e+03 + -5.073029115529123e+03 + -5.081844034000000e+03 + -5.090670481166524e+03 + -5.099508466925161e+03 + -5.108358001180135e+03 + -5.117219094017154e+03 + -5.126091755999993e+03 + -5.134975997626908e+03 + -5.143871827953957e+03 + -5.152779256704796e+03 + -5.161698295082459e+03 + -5.170628952999993e+03 + -5.179571239778229e+03 + -5.188525166029071e+03 + -5.197490742130520e+03 + -5.206467977662855e+03 + -5.215456883000000e+03 + -5.224457468752808e+03 + -5.233469744262066e+03 + -5.242493719420175e+03 + -5.251529405361101e+03 + -5.260576812000000e+03 + -5.269635948683524e+03 + -5.278706825911151e+03 + -5.287789454172008e+03 + -5.296883843495917e+03 + -5.305990003999993e+03 + -5.315107945871962e+03 + -5.324237679253967e+03 + -5.333379214299059e+03 + -5.342532561186867e+03 + -5.351697729999993e+03 + -5.360874730862462e+03 + -5.370063574314554e+03 + -5.379264270604972e+03 + -5.388476829418795e+03 + -5.397701261000000e+03 + -5.406937575887649e+03 + -5.416185784228891e+03 + -5.425445896082409e+03 + -5.434717921547790e+03 + -5.444001871000000e+03 + -5.453297754883592e+03 + -5.462605583222671e+03 + -5.471925366229407e+03 + -5.481257114511772e+03 + -5.490600837999993e+03 + -5.499956546429602e+03 + -5.509324250592133e+03 + -5.518703961068116e+03 + -5.528095687743725e+03 + -5.537499440999993e+03 + -5.546915231361128e+03 + -5.556343068498348e+03 + -5.565782962476614e+03 + -5.575234924238452e+03 + -5.584698964000000e+03 + -5.594175091687396e+03 + -5.603663318133420e+03 + -5.613163653827974e+03 + -5.622676108416792e+03 + -5.632200692000000e+03 + -5.641737415094019e+03 + -5.651286288507457e+03 + -5.660847322604374e+03 + -5.670420527008017e+03 + -5.680005911999993e+03 + -5.689603488367688e+03 + -5.699213267038800e+03 + -5.708835258184300e+03 + -5.718469470901945e+03 + -5.728115915999992e+03 + -5.737774605002216e+03 + -5.747445547634946e+03 + -5.757128753652110e+03 + -5.766824233562706e+03 + -5.776531998000000e+03 + -5.786252057564687e+03 + -5.795984422575620e+03 + -5.805729103393980e+03 + -5.815486110517342e+03 + -5.825255454000000e+03 + -5.835037143865786e+03 + -5.844831191184440e+03 + -5.854637606668529e+03 + -5.864456400127173e+03 + -5.874287581999993e+03 + -5.884131163070459e+03 + -5.893987153665426e+03 + -5.903855564090225e+03 + -5.913736404781277e+03 + -5.923629686000000e+03 + -5.933535418061928e+03 + -5.943453611973397e+03 + -5.953384278439786e+03 + -5.963327427438455e+03 + -5.973283069000000e+03 + -5.983251213525049e+03 + -5.993231872573540e+03 + -6.003225056744913e+03 + -6.013230774891098e+03 + -6.023249038000000e+03 + -6.033279857947543e+03 + -6.043323244258373e+03 + -6.053379206808452e+03 + -6.063447756872029e+03 + -6.073528904999992e+03 + -6.083622661322729e+03 + -6.093729036471069e+03 + -6.103848041095609e+03 + -6.113979685699232e+03 + -6.124123981000000e+03 + -6.134280937646101e+03 + -6.144450565430006e+03 + -6.154632874755873e+03 + -6.164827877191976e+03 + -6.175035583000000e+03 + -6.185256001935748e+03 + -6.195489145360168e+03 + -6.205735024246584e+03 + -6.215993648382001e+03 + -6.226265028000000e+03 + -6.236549173839082e+03 + -6.246846097225249e+03 + -6.257155808917487e+03 + -6.267478318647532e+03 + -6.277813636999993e+03 + -6.288161775039973e+03 + -6.298522743318929e+03 + -6.308896552332133e+03 + -6.319283212690151e+03 + -6.329682735000000e+03 + -6.340095129841887e+03 + -6.350520407726451e+03 + -6.360958579315570e+03 + -6.371409655512612e+03 + -6.381873647000000e+03 + -6.392350564293673e+03 + -6.402840417873253e+03 + -6.413343218449801e+03 + -6.423858977046060e+03 + -6.434387704000000e+03 + -6.444929409479182e+03 + -6.455484104800415e+03 + -6.466051800878468e+03 + -6.476632507643834e+03 + -6.487226235999992e+03 + -6.497832997260840e+03 + -6.508452801649206e+03 + -6.519085659445890e+03 + -6.529731581482401e+03 + -6.540390579000000e+03 + -6.551062663116751e+03 + -6.561747843536938e+03 + -6.572446130639311e+03 + -6.583157536279329e+03 + -6.593882071000000e+03 + -6.604619744763902e+03 + -6.615370568925604e+03 + -6.626134554413510e+03 + -6.636911711064461e+03 + -6.647702049999992e+03 + -6.658505582840396e+03 + -6.669322319643610e+03 + -6.680152270785791e+03 + -6.690995447680221e+03 + -6.701851860999993e+03 + -6.712721521095499e+03 + -6.723604439166686e+03 + -6.734500626205399e+03 + -6.745410092579864e+03 + -6.756332849000000e+03 + -6.767268906377159e+03 + -6.778218275408767e+03 + -6.789180966973428e+03 + -6.800156992270870e+03 + -6.811146362000000e+03 + -6.822149086596000e+03 + -6.833165176809022e+03 + -6.844194643583943e+03 + -6.855237498022965e+03 + -6.866293750999992e+03 + -6.877363413246921e+03 + -6.888446495554016e+03 + -6.899543008749155e+03 + -6.910652963678518e+03 + -6.921776370999992e+03 + -6.932913241460104e+03 + -6.944063586638155e+03 + -6.955227417524035e+03 + -6.966404743973952e+03 + -6.977595577000000e+03 + -6.988799928187199e+03 + -7.000017808173458e+03 + -7.011249227729020e+03 + -7.022494198140058e+03 + -7.033752730000000e+03 + -7.045024833685246e+03 + -7.056310520605544e+03 + -7.067609802022496e+03 + -7.078922688575014e+03 + -7.090249190999992e+03 + -7.101589320140114e+03 + -7.112943086874141e+03 + -7.124310502404821e+03 + -7.135691578318693e+03 + -7.147086324999992e+03 + -7.158494752468790e+03 + -7.169916872475182e+03 + -7.181352696510882e+03 + -7.192802235036809e+03 + -7.204265499000000e+03 + -7.215742499578333e+03 + -7.227233247391960e+03 + -7.238737753333357e+03 + -7.250256028887258e+03 + -7.261788085000000e+03 + -7.273333932305611e+03 + -7.284893581718426e+03 + -7.296467044458082e+03 + -7.308054332025791e+03 + -7.319655454999992e+03 + -7.331270423697963e+03 + -7.342899249833889e+03 + -7.354541944862027e+03 + -7.366198519315069e+03 + -7.377868983999992e+03 + -7.389553350032372e+03 + -7.401251628839381e+03 + -7.412963831476617e+03 + -7.424689968390403e+03 + -7.436430051000000e+03 + -7.448184091042194e+03 + -7.459952098877861e+03 + -7.471734085033887e+03 + -7.483530060856024e+03 + -7.495340038000000e+03 + -7.507164027981260e+03 + -7.519002041143363e+03 + -7.530854088088799e+03 + -7.542720180263871e+03 + -7.554600328999991e+03 + -7.566494545476364e+03 + -7.578402840747203e+03 + -7.590325225748156e+03 + -7.602261711334166e+03 + -7.614212308999990e+03 + -7.626177030375796e+03 + -7.638155885970103e+03 + -7.650148886679111e+03 + -7.662156044359298e+03 + -7.674177370000000e+03 + -7.686212874133715e+03 + -7.698262567927072e+03 + -7.710326462858042e+03 + -7.722404570574414e+03 + -7.734496902000000e+03 + -7.746603467726753e+03 + -7.758724278933637e+03 + -7.770859347070250e+03 + -7.783008683730720e+03 + -7.795172299999991e+03 + -7.807350206747154e+03 + -7.819542415323729e+03 + -7.831748936928582e+03 + -7.843969782379609e+03 + -7.856204962999991e+03 + -7.868454490389281e+03 + -7.880718375856472e+03 + -7.892996630459527e+03 + -7.905289265033489e+03 + -7.917596291000000e+03 + -7.929917719977670e+03 + -7.942253562811284e+03 + -7.954603830544003e+03 + -7.966968534806996e+03 + -7.979347687000000e+03 + -7.991741298275563e+03 + -8.004149379532312e+03 + -8.016571941875890e+03 + -8.029008996834256e+03 + -8.041460555999991e+03 + -8.053926630829063e+03 + -8.066407232164392e+03 + -8.078902371093302e+03 + -8.091412059283156e+03 + -8.103936307999991e+03 + -8.116475128348727e+03 + -8.129028531951320e+03 + -8.141596530160446e+03 + -8.154179133753471e+03 + -8.166776354000000e+03 + -8.179388202500601e+03 + -8.192014690774517e+03 + -8.204655830155478e+03 + -8.217311631752342e+03 + -8.229982107000000e+03 + -8.242667267443709e+03 + -8.255367124181290e+03 + -8.268081688506129e+03 + -8.280810972124258e+03 + -8.293554985999990e+03 + -8.306313740930424e+03 + -8.319087249046417e+03 + -8.331875522237864e+03 + -8.344678571514016e+03 + -8.357496408000001e+03 + -8.370329042981793e+03 + -8.383176487893126e+03 + -8.396038754236946e+03 + -8.408915853538949e+03 + -8.421807796999999e+03 + -8.434714595760075e+03 + -8.447636261558551e+03 + -8.460572806112319e+03 + -8.473524240852536e+03 + -8.486490577000000e+03 + -8.499471825734363e+03 + -8.512467998550057e+03 + -8.525479106914012e+03 + -8.538505162160651e+03 + -8.551546175999989e+03 + -8.564602160205039e+03 + -8.577673125806519e+03 + -8.590759084111069e+03 + -8.603860047081474e+03 + -8.616976026000000e+03 + -8.630107031891077e+03 + -8.643253076647045e+03 + -8.656414171854482e+03 + -8.669590328370747e+03 + -8.682781558000001e+03 + -8.695987872906602e+03 + -8.709209284090668e+03 + -8.722445802791402e+03 + -8.735697441007769e+03 + -8.748964210000000e+03 + -8.762246120773598e+03 + -8.775543185390141e+03 + -8.788855415760128e+03 + -8.802182823160678e+03 + -8.815525418999991e+03 + -8.828883214778334e+03 + -8.842256221886048e+03 + -8.855644451817980e+03 + -8.869047916276779e+03 + -8.882466627000000e+03 + -8.895900595656447e+03 + -8.909349833599377e+03 + -8.922814352320849e+03 + -8.936294163622882e+03 + -8.949789279000001e+03 + -8.963299709806102e+03 + -8.976825467697834e+03 + -8.990366564425331e+03 + -9.003923011733321e+03 + -9.017494820999989e+03 + -9.031082003570244e+03 + -9.044684571593196e+03 + -9.058302536801553e+03 + -9.071935910043938e+03 + -9.085584702999991e+03 + -9.099248927805194e+03 + -9.112928596071970e+03 + -9.126623719398038e+03 + -9.140334309541779e+03 + -9.154060378000000e+03 + -9.167801936143134e+03 + -9.181558995552323e+03 + -9.195331568052554e+03 + -9.209119665711642e+03 + -9.222923300000000e+03 + -9.236742482191501e+03 + -9.250577224366738e+03 + -9.264427538348678e+03 + -9.278293435300837e+03 + -9.292174926999989e+03 + -9.306072025514850e+03 + -9.319984742353185e+03 + -9.333913089239737e+03 + -9.347857078372264e+03 + -9.361816720999988e+03 + -9.375792028076301e+03 + -9.389783011923670e+03 + -9.403789684689073e+03 + -9.417812057757259e+03 + -9.431850143000000e+03 + -9.445903952452922e+03 + -9.459973497383473e+03 + -9.474058789280491e+03 + -9.488159840242311e+03 + -9.502276662000000e+03 + -9.516409266099370e+03 + -9.530557664431180e+03 + -9.544721868948835e+03 + -9.558901891554557e+03 + -9.573097743999990e+03 + -9.587309437907974e+03 + -9.601536984771086e+03 + -9.615780396443668e+03 + -9.630039685320193e+03 + -9.644314862999989e+03 + -9.658605940694017e+03 + -9.672912930240365e+03 + -9.687235843732506e+03 + -9.701574693379191e+03 + -9.715929491000001e+03 + -9.730300248146514e+03 + -9.744686976337332e+03 + -9.759089687418043e+03 + -9.773508393705906e+03 + -9.787943106999999e+03 + -9.802393838806964e+03 + -9.816860600913731e+03 + -9.831343405392694e+03 + -9.845842264577648e+03 + -9.860357189999990e+03 + -9.874888192875391e+03 + -9.889435285292540e+03 + -9.903998479577405e+03 + -9.918577788007758e+03 + -9.933173221999989e+03 + -9.947784792760667e+03 + -9.962412512866247e+03 + -9.977056394630243e+03 + -9.991716449514770e+03 + -1.000639269000000e+04 + -1.002108512827329e+04 + -1.003579377260607e+04 + -1.005051863510592e+04 + -1.006525973450424e+04 + -1.008001708000000e+04 + -1.009479067738785e+04 + -1.010958054488143e+04 + -1.012438669671707e+04 + -1.013920913701616e+04 + -1.015404787999999e+04 + -1.016890294387830e+04 + -1.018377433451942e+04 + -1.019866206104612e+04 + -1.021356614193473e+04 + -1.022848658999999e+04 + -1.024342341422066e+04 + -1.025837662498870e+04 + -1.027334623400460e+04 + -1.028833225445739e+04 + -1.030333470000000e+04 + -1.031835358368700e+04 + -1.033338891534757e+04 + -1.034844070623467e+04 + -1.036350897076085e+04 + -1.037859372000000e+04 + -1.039369496373519e+04 + -1.040881271589677e+04 + -1.042394698923176e+04 + -1.043909779325773e+04 + -1.045426513999999e+04 + -1.046944904320307e+04 + -1.048464951626011e+04 + -1.049986657147346e+04 + -1.051510021951631e+04 + -1.053035046999999e+04 + -1.054561733346490e+04 + -1.056090082649770e+04 + -1.057620096382862e+04 + -1.059151775501402e+04 + -1.060685121000000e+04 + -1.062220133982168e+04 + -1.063756815769902e+04 + -1.065295167679853e+04 + -1.066835190951487e+04 + -1.068376887000000e+04 + -1.069920257172756e+04 + -1.071465302083269e+04 + -1.073012022736731e+04 + -1.074560420973240e+04 + -1.076110497999999e+04 + -1.077662254698522e+04 + -1.079215692468130e+04 + -1.080770812602064e+04 + -1.082327616047952e+04 + -1.083886103999999e+04 + -1.085446277832030e+04 + -1.087008138917964e+04 + -1.088571688451933e+04 + -1.090136927381244e+04 + -1.091703857000000e+04 + -1.093272478707855e+04 + -1.094842793381667e+04 + -1.096414802233435e+04 + -1.097988507119364e+04 + -1.099563909000000e+04 + -1.101141008471468e+04 + -1.102719807131307e+04 + -1.104300306633410e+04 + -1.105882508291791e+04 + -1.107466412999999e+04 + -1.109052021511841e+04 + -1.110639335038349e+04 + -1.112228355101184e+04 + -1.113819083451417e+04 + -1.115411520999999e+04 + -1.117005668368749e+04 + -1.118601527242516e+04 + -1.120199099177015e+04 + -1.121798385114509e+04 + -1.123399386000000e+04 + -1.125002102929127e+04 + -1.126606537454743e+04 + -1.128212691149971e+04 + -1.129820565394142e+04 + -1.131430161000000e+04 + -1.133041478676655e+04 + -1.134654520134832e+04 + -1.136269286880464e+04 + -1.137885779750048e+04 + -1.139503999999999e+04 + -1.141123949060806e+04 + -1.142745627830015e+04 + -1.144369037489935e+04 + -1.145994179819328e+04 + -1.147621056000000e+04 + -1.149249666929363e+04 + -1.150880014040390e+04 + -1.152512098586926e+04 + -1.154145921396878e+04 + -1.155781484000000e+04 + -1.157418788116290e+04 + -1.159057834325663e+04 + -1.160698623573862e+04 + -1.162341157750990e+04 + -1.163985437999999e+04 + -1.165631465167572e+04 + -1.167279241049137e+04 + -1.168928767028212e+04 + -1.170580043559586e+04 + -1.172233071999999e+04 + -1.173887854137991e+04 + -1.175544390951204e+04 + -1.177202683447122e+04 + -1.178862732998528e+04 + -1.180524541000000e+04 + -1.182188108744826e+04 + -1.183853437152247e+04 + -1.185520527417456e+04 + -1.187189381268968e+04 + -1.188860000000000e+04 + -1.190532384555031e+04 + -1.192206535739879e+04 + -1.193882454957090e+04 + -1.195560144455150e+04 + -1.197239604999999e+04 + -1.198920836801576e+04 + -1.200603841862060e+04 + -1.202288622005816e+04 + -1.203975178087160e+04 + -1.205663510999999e+04 + -1.207353621858988e+04 + -1.209045512337030e+04 + -1.210739183876488e+04 + -1.212434637372514e+04 + -1.214131874000000e+04 + -1.215830895146994e+04 + -1.217531702204386e+04 + -1.219234296344827e+04 + -1.220938678454126e+04 + -1.222644850000000e+04 + -1.224352812665298e+04 + -1.226062567426862e+04 + -1.227774115320455e+04 + -1.229487457740941e+04 + -1.231202595999999e+04 + -1.232919531361497e+04 + -1.234638265190763e+04 + -1.236358798719477e+04 + -1.238081132944854e+04 + -1.239805268999999e+04 + -1.241531208224867e+04 + -1.243258952373459e+04 + -1.244988502746729e+04 + -1.246719859854622e+04 + -1.248453025000000e+04 + -1.250187999937352e+04 + -1.251924785997314e+04 + -1.253663384322633e+04 + -1.255403795942784e+04 + -1.257146022000000e+04 + -1.258890063824787e+04 + -1.260635923174130e+04 + -1.262383601348307e+04 + -1.264133098848631e+04 + -1.265884416999999e+04 + -1.267637557565735e+04 + -1.269392521755929e+04 + -1.271149310837373e+04 + -1.272907926320349e+04 + -1.274668368999999e+04 + -1.276430639533002e+04 + -1.278194739917555e+04 + -1.279960671868295e+04 + -1.281728436173200e+04 + -1.283498034000000e+04 + -1.285269466774105e+04 + -1.287042735763161e+04 + -1.288817842259290e+04 + -1.290594787633268e+04 + -1.292373573000000e+04 + -1.294154199365508e+04 + -1.295936667981993e+04 + -1.297720980264566e+04 + -1.299507137776778e+04 + -1.301295141999999e+04 + -1.303084994203483e+04 + -1.304876695061188e+04 + -1.306670245535235e+04 + -1.308465647270176e+04 + -1.310262901999999e+04 + -1.312062011245361e+04 + -1.313862975604251e+04 + -1.315665796055223e+04 + -1.317470474462057e+04 + -1.319277012000000e+04 + -1.321085409533674e+04 + -1.322895668661698e+04 + -1.324707790969205e+04 + -1.326521777718879e+04 + -1.328337630000000e+04 + -1.330155348851537e+04 + -1.331974935491385e+04 + -1.333796391362056e+04 + -1.335619718140170e+04 + -1.337444916999999e+04 + -1.339271988874384e+04 + -1.341100935082290e+04 + -1.342931756987852e+04 + -1.344764455872234e+04 + -1.346599032999999e+04 + -1.348435489664480e+04 + -1.350273827293541e+04 + -1.352114047045672e+04 + -1.353956149695641e+04 + -1.355800137000000e+04 + -1.357646010972569e+04 + -1.359493772045508e+04 + -1.361343421245217e+04 + -1.363194960986463e+04 + -1.365048392000000e+04 + -1.366903714425879e+04 + -1.368760930691950e+04 + -1.370620042690972e+04 + -1.372481050688227e+04 + -1.374343955999999e+04 + -1.376208760473736e+04 + -1.378075465005087e+04 + -1.379944070672178e+04 + -1.381814579185646e+04 + -1.383686991999999e+04 + -1.385561310323773e+04 + -1.387437535187516e+04 + -1.389315667691767e+04 + -1.391195709130158e+04 + -1.393077661000000e+04 + -1.394961524865979e+04 + -1.396847302075987e+04 + -1.398734993803961e+04 + -1.400624601054151e+04 + -1.402516125000000e+04 + -1.404409566989268e+04 + -1.406304928589068e+04 + -1.408202211294199e+04 + -1.410101416389908e+04 + -1.412002544999999e+04 + -1.413905598164077e+04 + -1.415810576964489e+04 + -1.417717482985645e+04 + -1.419626318438561e+04 + -1.421537083999999e+04 + -1.423449779870650e+04 + -1.425364408441530e+04 + -1.427280971628262e+04 + -1.429199469812906e+04 + -1.431119904000000e+04 + -1.433042275698022e+04 + -1.434966586475864e+04 + -1.436892837784737e+04 + -1.438821030874347e+04 + -1.440751167000000e+04 + -1.442683247464153e+04 + -1.444617273693482e+04 + -1.446553246845352e+04 + -1.448491167696990e+04 + -1.450431037999999e+04 + -1.452372859766022e+04 + -1.454316633455064e+04 + -1.456262360064903e+04 + -1.458210041914065e+04 + -1.460159679999998e+04 + -1.462111274782618e+04 + -1.464064828275210e+04 + -1.466020342227678e+04 + -1.467977817411577e+04 + -1.469937255000000e+04 + -1.471898656425745e+04 + -1.473862022905967e+04 + -1.475827355783788e+04 + -1.477794656667906e+04 + -1.479763927000000e+04 + -1.481735168022591e+04 + -1.483708380678435e+04 + -1.485683566151257e+04 + -1.487660726097236e+04 + -1.489639861999999e+04 + -1.491620975095499e+04 + -1.493604066191429e+04 + -1.495589136614776e+04 + -1.497576188555746e+04 + -1.499565223000000e+04 + -1.501556240451360e+04 + -1.503549242791256e+04 + -1.505544231896391e+04 + -1.507541209051985e+04 + -1.509540175000000e+04 + -1.511541130465759e+04 + -1.513544077375174e+04 + -1.515549017500820e+04 + -1.517555951933053e+04 + -1.519564881999999e+04 + -1.521575809081157e+04 + -1.523588734003694e+04 + -1.525603658036919e+04 + -1.527620583277191e+04 + -1.529639510999998e+04 + -1.531660442034102e+04 + -1.533683377728779e+04 + -1.535708319443415e+04 + -1.537735268372974e+04 + -1.539764226000000e+04 + -1.541795193921473e+04 + -1.543828173411788e+04 + -1.545863165713528e+04 + -1.547900172144165e+04 + -1.549939194000000e+04 + -1.551980232569260e+04 + -1.554023289178161e+04 + -1.556068365093051e+04 + -1.558115461506463e+04 + -1.560164579999999e+04 + -1.562215722197032e+04 + -1.564268888875843e+04 + -1.566324081295743e+04 + -1.568381301679686e+04 + -1.570440550999999e+04 + -1.572501829745089e+04 + -1.574565139909988e+04 + -1.576630483256334e+04 + -1.578697860613842e+04 + -1.580767273000000e+04 + -1.582838721786272e+04 + -1.584912209052793e+04 + -1.586987736246908e+04 + -1.589065303680525e+04 + -1.591144913000000e+04 + -1.593226566336701e+04 + -1.595310264072689e+04 + -1.597396007256353e+04 + -1.599483798518879e+04 + -1.601573638999999e+04 + -1.603665529105288e+04 + -1.605759470511345e+04 + -1.607855464838671e+04 + -1.609953513161670e+04 + -1.612053616999998e+04 + -1.614155778004294e+04 + -1.616259997081388e+04 + -1.618366275348782e+04 + -1.620474614514243e+04 + -1.622585016000000e+04 + -1.624697481007329e+04 + -1.626812010728422e+04 + -1.628928606592670e+04 + -1.631047270337715e+04 + -1.633168003000000e+04 + -1.635290805454565e+04 + -1.637415679800609e+04 + -1.639542627592296e+04 + -1.641671649164588e+04 + -1.643802745999998e+04 + -1.645935920179283e+04 + -1.648071172935930e+04 + -1.650208505431875e+04 + -1.652347919039644e+04 + -1.654489414999998e+04 + -1.656632994566724e+04 + -1.658778659427021e+04 + -1.660926410974226e+04 + -1.663076250029903e+04 + -1.665228178000000e+04 + -1.667382196574960e+04 + -1.669538306946903e+04 + -1.671696510420377e+04 + -1.673856808645374e+04 + -1.676019203000000e+04 + -1.678183694695306e+04 + -1.680350285053965e+04 + -1.682518975366077e+04 + -1.684689766850574e+04 + -1.686862660999998e+04 + -1.689037659405139e+04 + -1.691214763326874e+04 + -1.693393974023558e+04 + -1.695575292860696e+04 + -1.697758720999999e+04 + -1.699944259677470e+04 + -1.702131910964586e+04 + -1.704321676352935e+04 + -1.706513556209216e+04 + -1.708707552000000e+04 + -1.710903665790321e+04 + -1.713101898938551e+04 + -1.715302252479822e+04 + -1.717504727292061e+04 + -1.719709325000000e+04 + -1.721916047545785e+04 + -1.724124896174670e+04 + -1.726335872072246e+04 + -1.728548976595536e+04 + -1.730764210999998e+04 + -1.732981576548871e+04 + -1.735201074834581e+04 + -1.737422707347338e+04 + -1.739646475299950e+04 + -1.741872379999999e+04 + -1.744100422776899e+04 + -1.746330604747148e+04 + -1.748562927386205e+04 + -1.750797392735042e+04 + -1.753034002000000e+04 + -1.755272756012299e+04 + -1.757513656385630e+04 + -1.759756704698003e+04 + -1.762001902165879e+04 + -1.764249250000000e+04 + -1.766498749503509e+04 + -1.768750402279969e+04 + -1.771004209797560e+04 + -1.773260173223158e+04 + -1.775518293999998e+04 + -1.777778573725219e+04 + -1.780041013823985e+04 + -1.782305615380817e+04 + -1.784572379125812e+04 + -1.786841306999998e+04 + -1.789112401297027e+04 + -1.791385662520795e+04 + -1.793661091551944e+04 + -1.795938690506571e+04 + -1.798218461000000e+04 + -1.800500404157250e+04 + -1.802784520726593e+04 + -1.805070812126097e+04 + -1.807359280836476e+04 + -1.809649928000000e+04 + -1.811942754077938e+04 + -1.814237760512274e+04 + -1.816534949154946e+04 + -1.818834322012563e+04 + -1.821135879999998e+04 + -1.823439623670011e+04 + -1.825745554979138e+04 + -1.828053675712656e+04 + -1.830363986886144e+04 + -1.832676489999998e+04 + -1.834991186721417e+04 + -1.837308077964562e+04 + -1.839627164834952e+04 + -1.841948449011883e+04 + -1.844271932000000e+04 + -1.846597615248214e+04 + -1.848925500551537e+04 + -1.851255589141731e+04 + -1.853587881367952e+04 + -1.855922379000000e+04 + -1.858259084403384e+04 + -1.860597998459446e+04 + -1.862939122130964e+04 + -1.865282457086503e+04 + -1.867628004999998e+04 + -1.869975767367864e+04 + -1.872325745125265e+04 + -1.874677939400045e+04 + -1.877032351829797e+04 + -1.879388983999998e+04 + -1.881747837395618e+04 + -1.884108913346766e+04 + -1.886472213066709e+04 + -1.888837737699991e+04 + -1.891205489000000e+04 + -1.893575468867888e+04 + -1.895947678210836e+04 + -1.898322118083019e+04 + -1.900698790146692e+04 + -1.903077696000000e+04 + -1.905458837122149e+04 + -1.907842214817702e+04 + -1.910227830328199e+04 + -1.912615684887913e+04 + -1.915005779999998e+04 + -1.917398117252360e+04 + -1.919792697876641e+04 + -1.922189523162585e+04 + -1.924588594641727e+04 + -1.926989913999998e+04 + -1.929393482787026e+04 + -1.931799301676958e+04 + -1.934207372057390e+04 + -1.936617696590223e+04 + -1.939030276000000e+04 + -1.941445110358184e+04 + -1.943862202401437e+04 + -1.946281554213054e+04 + -1.948703165938676e+04 + -1.951127038999998e+04 + -1.953553175531375e+04 + -1.955981576784185e+04 + -1.958412243887696e+04 + -1.960845178142548e+04 + -1.963280380999998e+04 + -1.965717854010317e+04 + -1.968157598764983e+04 + -1.970599616682758e+04 + -1.973043908928781e+04 + -1.975490477000000e+04 + -1.977939322508017e+04 + -1.980390446612710e+04 + -1.982843850672890e+04 + -1.985299536464853e+04 + -1.987757505000000e+04 + -1.990217757132781e+04 + -1.992680295149982e+04 + -1.995145120910892e+04 + -1.997612235106852e+04 + -2.000081638999998e+04 + -2.002553334291399e+04 + -2.005027322692212e+04 + -2.007503605456619e+04 + -2.009982183242756e+04 + -2.012463057999998e+04 + -2.014946232050470e+04 + -2.017431705750974e+04 + -2.019919480128925e+04 + -2.022409557869796e+04 + -2.024901940000000e+04 + -2.027396626876084e+04 + -2.029893620825063e+04 + -2.032392923791251e+04 + -2.034894536415857e+04 + -2.037398460000000e+04 + -2.039904696273123e+04 + -2.042413246681671e+04 + -2.044924112749899e+04 + -2.047437296177629e+04 + -2.049952797999998e+04 + -2.052470619105544e+04 + -2.054990761544010e+04 + -2.057513226917888e+04 + -2.060038015795331e+04 + -2.062565129999998e+04 + -2.065094571854568e+04 + -2.067626342205840e+04 + -2.070160442063917e+04 + -2.072696873225083e+04 + -2.075235637000000e+04 + -2.077776734528838e+04 + -2.080320167696219e+04 + -2.082865938214449e+04 + -2.085414047244347e+04 + -2.087964496000000e+04 + -2.090517285805777e+04 + -2.093072418166662e+04 + -2.095629894648643e+04 + -2.098189716820891e+04 + -2.100751885999998e+04 + -2.103316403410750e+04 + -2.105883270565323e+04 + -2.108452489048020e+04 + -2.111024060421226e+04 + -2.113597985999998e+04 + -2.116174267011207e+04 + -2.118752904964539e+04 + -2.121333901446366e+04 + -2.123917258027344e+04 + -2.126502976000000e+04 + -2.129091056582378e+04 + -2.131681501406569e+04 + -2.134274311932949e+04 + -2.136869489244911e+04 + -2.139467035000000e+04 + -2.142066951097057e+04 + -2.144669238829770e+04 + -2.147273899344209e+04 + -2.149880933837191e+04 + -2.152490343999998e+04 + -2.155102131698669e+04 + -2.157716298229367e+04 + -2.160332844744082e+04 + -2.162951772437019e+04 + -2.165573082999998e+04 + -2.168196778300156e+04 + -2.170822859627157e+04 + -2.173451328139519e+04 + -2.176082185053812e+04 + -2.178715432000000e+04 + -2.181351070821111e+04 + -2.183989103150391e+04 + -2.186629530288904e+04 + -2.189272353140778e+04 + -2.191917573000000e+04 + -2.194565191497987e+04 + -2.197215210512397e+04 + -2.199867631637375e+04 + -2.202522455969623e+04 + -2.205179684999998e+04 + -2.207839320363563e+04 + -2.210501363148952e+04 + -2.213165814783550e+04 + -2.215832677350680e+04 + -2.218501951999998e+04 + -2.221173639532642e+04 + -2.223847741910094e+04 + -2.226524260850044e+04 + -2.229203197306979e+04 + -2.231884553000000e+04 + -2.234568329890952e+04 + -2.237254528774074e+04 + -2.239943150806158e+04 + -2.242634198094407e+04 + -2.245327672000000e+04 + -2.248023573585785e+04 + -2.250721904853940e+04 + -2.253422667413636e+04 + -2.256125861967753e+04 + -2.258831489999998e+04 + -2.261539553480252e+04 + -2.264250054093014e+04 + -2.266962993052693e+04 + -2.269678371062339e+04 + -2.272396189999998e+04 + -2.275116452153000e+04 + -2.277839158301363e+04 + -2.280564309534358e+04 + -2.283291907950545e+04 + -2.286021955000000e+04 + -2.288754451771620e+04 + -2.291489399812011e+04 + -2.294226800704672e+04 + -2.296966655917324e+04 + -2.299708967000000e+04 + -2.302453735490683e+04 + -2.305200962641978e+04 + -2.307950649734620e+04 + -2.310702798236498e+04 + -2.313457409999998e+04 + -2.316214486856879e+04 + -2.318974029613145e+04 + -2.321736039447026e+04 + -2.324500518448656e+04 + -2.327267467999998e+04 + -2.330036889175501e+04 + -2.332808783850466e+04 + -2.335583153764027e+04 + -2.338360000121357e+04 + -2.341139324000000e+04 + -2.343921126672951e+04 + -2.346705410394532e+04 + -2.349492176951476e+04 + -2.352281427072812e+04 + -2.355073162000000e+04 + -2.357867383414989e+04 + -2.360664093193761e+04 + -2.363463292919098e+04 + -2.366264983684289e+04 + -2.369069166999998e+04 + -2.371875844507638e+04 + -2.374685017193912e+04 + -2.377496686591387e+04 + -2.380310855200050e+04 + -2.383127523999998e+04 + -2.385946693403709e+04 + -2.388768365690810e+04 + -2.391592542840760e+04 + -2.394419225713039e+04 + -2.397248416000000e+04 + -2.400080115691363e+04 + -2.402914325571854e+04 + -2.405751046800998e+04 + -2.408590281513924e+04 + -2.411432031000000e+04 + -2.414276296293371e+04 + -2.417123079796694e+04 + -2.419972383203242e+04 + -2.422824206706797e+04 + -2.425678551999998e+04 + -2.428535421485334e+04 + -2.431394816200073e+04 + -2.434256737430352e+04 + -2.437121187300517e+04 + -2.439988166999998e+04 + -2.442857677325164e+04 + -2.445729720103540e+04 + -2.448604297194451e+04 + -2.451481410082247e+04 + -2.454361060000000e+04 + -2.457243248182022e+04 + -2.460127976431730e+04 + -2.463015246522100e+04 + -2.465905059942621e+04 + -2.468797417999998e+04 + -2.471692321945616e+04 + -2.474589773222362e+04 + -2.477489773480251e+04 + -2.480392324581322e+04 + -2.483297427999998e+04 + -2.486205084917968e+04 + -2.489115296405446e+04 + -2.492028064024748e+04 + -2.494943390068509e+04 + -2.497861275999998e+04 + -2.500781722863925e+04 + -2.503704732334474e+04 + -2.506630305872901e+04 + -2.509558444448151e+04 + -2.512489150000000e+04 + -2.515422424757818e+04 + -2.518358269497819e+04 + -2.521296685328307e+04 + -2.524237674371633e+04 + -2.527181237999998e+04 + -2.530127377273551e+04 + -2.533076094162915e+04 + -2.536027390311031e+04 + -2.538981266594794e+04 + -2.541937724999998e+04 + -2.544896767801531e+04 + -2.547858395406606e+04 + -2.550822608802068e+04 + -2.553789410507042e+04 + -2.556758802000000e+04 + -2.559730784246711e+04 + -2.562705359189478e+04 + -2.565682528525429e+04 + -2.568662293247029e+04 + -2.571644655000000e+04 + -2.574629615713407e+04 + -2.577617176604459e+04 + -2.580607338891535e+04 + -2.583600104095400e+04 + -2.586595473999998e+04 + -2.589593450400127e+04 + -2.592594034522632e+04 + -2.595597227610649e+04 + -2.598603031176017e+04 + -2.601611446999998e+04 + -2.604622476879967e+04 + -2.607636122042706e+04 + -2.610652383730750e+04 + -2.613671263455693e+04 + -2.616692763000000e+04 + -2.619716884161440e+04 + -2.622743628160562e+04 + -2.625772996246335e+04 + -2.628804989951923e+04 + -2.631839611000000e+04 + -2.634876861165059e+04 + -2.637916741999917e+04 + -2.640959254907723e+04 + -2.644004401148305e+04 + -2.647052181999998e+04 + -2.650102598983489e+04 + -2.653155654491748e+04 + -2.656211350103963e+04 + -2.659269685960091e+04 + -2.662330663999998e+04 + -2.665394286925502e+04 + -2.668460555517195e+04 + -2.671529470758218e+04 + -2.674601034670874e+04 + -2.677675249000000e+04 + -2.680752115154265e+04 + -2.683831634157976e+04 + -2.686913807564717e+04 + -2.689998637786448e+04 + -2.693086126000000e+04 + -2.696176272916689e+04 + -2.699269080763921e+04 + -2.702364551500381e+04 + -2.705462686113689e+04 + -2.708563485999998e+04 + -2.711666952809900e+04 + -2.714773087942122e+04 + -2.717881892991400e+04 + -2.720993369901594e+04 + -2.724107519999998e+04 + -2.727224344400496e+04 + -2.730343845029767e+04 + -2.733466023562860e+04 + -2.736590881019403e+04 + -2.739718419000000e+04 + -2.742848639412166e+04 + -2.745981543741902e+04 + -2.749117133427855e+04 + -2.752255410001475e+04 + -2.755396375000000e+04 + -2.758540029934807e+04 + -2.761686376223195e+04 + -2.764835415476579e+04 + -2.767987149599797e+04 + -2.771141579999997e+04 + -2.774298707825893e+04 + -2.777458534547961e+04 + -2.780621061813955e+04 + -2.783786291408934e+04 + -2.786954224999998e+04 + -2.790124864062599e+04 + -2.793298209650868e+04 + -2.796474263269793e+04 + -2.799653027215609e+04 + -2.802834503000000e+04 + -2.806018691678819e+04 + -2.809205594694776e+04 + -2.812395213748840e+04 + -2.815587550739401e+04 + -2.818782607000000e+04 + -2.821980383662756e+04 + -2.825180882556652e+04 + -2.828384105454396e+04 + -2.831590053775406e+04 + -2.834798728999998e+04 + -2.838010132665882e+04 + -2.841224266303304e+04 + -2.844441131398464e+04 + -2.847660729395342e+04 + -2.850883061999998e+04 + -2.854108131015271e+04 + -2.857335937943549e+04 + -2.860566484233479e+04 + -2.863799771378746e+04 + -2.867035801000000e+04 + -2.870274574643904e+04 + -2.873516093256026e+04 + -2.876760358436649e+04 + -2.880007372909855e+04 + -2.883257138000000e+04 + -2.886509654365616e+04 + -2.889764923870952e+04 + -2.893022948372628e+04 + -2.896283729252707e+04 + -2.899547267999998e+04 + -2.902813566168059e+04 + -2.906082625190441e+04 + -2.909354446650575e+04 + -2.912629032389195e+04 + -2.915906383999998e+04 + -2.919186502919167e+04 + -2.922469390650106e+04 + -2.925755048733507e+04 + -2.929043478725063e+04 + -2.932334682000000e+04 + -2.935628660019129e+04 + -2.938925415043921e+04 + -2.942224948806936e+04 + -2.945527261979538e+04 + -2.948832356000000e+04 + -2.952140232838030e+04 + -2.955450894361826e+04 + -2.958764342089699e+04 + -2.962080577090842e+04 + -2.965399600999997e+04 + -2.968721415715336e+04 + -2.972046022629084e+04 + -2.975373423280592e+04 + -2.978703619592538e+04 + -2.982036612999997e+04 + -2.985372404652352e+04 + -2.988710995909819e+04 + -2.992052388535540e+04 + -2.995396584761164e+04 + -2.998743586000000e+04 + -3.002093393306212e+04 + -3.005446008516229e+04 + -3.008801433415196e+04 + -3.012159669410924e+04 + -3.015520717999998e+04 + -3.018884580728562e+04 + -3.022251259030155e+04 + -3.025620754490144e+04 + -3.028993068950620e+04 + -3.032368203999997e+04 + -3.035746161072607e+04 + -3.039126941699663e+04 + -3.042510547393189e+04 + -3.045896979612689e+04 + -3.049286239999998e+04 + -3.052678330335746e+04 + -3.056073252467363e+04 + -3.059471007984014e+04 + -3.062871598061521e+04 + -3.066275024000000e+04 + -3.069681287356475e+04 + -3.073090390293319e+04 + -3.076502334621974e+04 + -3.079917121404217e+04 + -3.083334751999997e+04 + -3.086755228023463e+04 + -3.090178551167921e+04 + -3.093604723225209e+04 + -3.097033746063074e+04 + -3.100465620999998e+04 + -3.103900349175437e+04 + -3.107337932446759e+04 + -3.110778372555016e+04 + -3.114221670819769e+04 + -3.117667828999997e+04 + -3.121116848957679e+04 + -3.124568731744623e+04 + -3.128023478769976e+04 + -3.131481092268889e+04 + -3.134941574000000e+04 + -3.138404925321087e+04 + -3.141871147408727e+04 + -3.145340241670125e+04 + -3.148812209933996e+04 + -3.152287053999997e+04 + -3.155764775577736e+04 + -3.159245376185456e+04 + -3.162728857279384e+04 + -3.166215220315186e+04 + -3.169704466999997e+04 + -3.173196599132658e+04 + -3.176691618227545e+04 + -3.180189525721778e+04 + -3.183690323068162e+04 + -3.187194012000000e+04 + -3.190700594323912e+04 + -3.194210071422084e+04 + -3.197722444868163e+04 + -3.201237716636405e+04 + -3.204755888000000e+04 + -3.208276960047787e+04 + -3.211800935023939e+04 + -3.215327814773632e+04 + -3.218857600158763e+04 + -3.222390292999997e+04 + -3.225925895535199e+04 + -3.229464408973093e+04 + -3.233005834462857e+04 + -3.236550173499735e+04 + -3.240097427999997e+04 + -3.243647600017841e+04 + -3.247200691143220e+04 + -3.250756702669824e+04 + -3.254315635678932e+04 + -3.257877492000000e+04 + -3.261442273792899e+04 + -3.265009982555360e+04 + -3.268580619650297e+04 + -3.272154186523808e+04 + -3.275730685000000e+04 + -3.279310116911321e+04 + -3.282892483177912e+04 + -3.286477785354587e+04 + -3.290066026202923e+04 + -3.293657206999998e+04 + -3.297251328378459e+04 + -3.300848392497173e+04 + -3.304448401407080e+04 + -3.308051356412061e+04 + -3.311657258999998e+04 + -3.315266110752813e+04 + -3.318877912993135e+04 + -3.322492667413289e+04 + -3.326110376297835e+04 + -3.329731041000000e+04 + -3.333354662539115e+04 + -3.336981243131684e+04 + -3.340610784669931e+04 + -3.344243288124504e+04 + -3.347878755000000e+04 + -3.351517187197135e+04 + -3.355158586607149e+04 + -3.358802954742114e+04 + -3.362450292603845e+04 + -3.366100601999997e+04 + -3.369753885113907e+04 + -3.373410143432544e+04 + -3.377069378324004e+04 + -3.380731591267414e+04 + -3.384396783999997e+04 + -3.388064958319757e+04 + -3.391736115628571e+04 + -3.395410257481684e+04 + -3.399087385786547e+04 + -3.402767502000000e+04 + -3.406450607498593e+04 + -3.410136704569621e+04 + -3.413825794940452e+04 + -3.417517879208190e+04 + -3.421212959000000e+04 + -3.424911036577375e+04 + -3.428612113814977e+04 + -3.432316192150666e+04 + -3.436023272579379e+04 + -3.439733356999998e+04 + -3.443446447621852e+04 + -3.447162545467578e+04 + -3.450881651888796e+04 + -3.454603769159935e+04 + -3.458328898999997e+04 + -3.462057042742187e+04 + -3.465788201822977e+04 + -3.469522377879945e+04 + -3.473259572783916e+04 + -3.476999788000000e+04 + -3.480743024920274e+04 + -3.484489285737491e+04 + -3.488238572274076e+04 + -3.491990885516477e+04 + -3.495746227000000e+04 + -3.499504598627476e+04 + -3.503266002163332e+04 + -3.507030439252002e+04 + -3.510797911399110e+04 + -3.514568419999997e+04 + -3.518341966538968e+04 + -3.522118553105182e+04 + -3.525898181605689e+04 + -3.529680853433678e+04 + -3.533466569999997e+04 + -3.537255332834070e+04 + -3.541047143775194e+04 + -3.544842004484196e+04 + -3.548639916273562e+04 + -3.552440881000000e+04 + -3.556244900769237e+04 + -3.560051977192599e+04 + -3.563862111528000e+04 + -3.567675304773126e+04 + -3.571491559000000e+04 + -3.575310876677514e+04 + -3.579133259025522e+04 + -3.582958707123636e+04 + -3.586787222387752e+04 + -3.590618806999997e+04 + -3.594453463328037e+04 + -3.598291192576281e+04 + -3.602131995866808e+04 + -3.605975874687618e+04 + -3.609822830999997e+04 + -3.613672866872384e+04 + -3.617525983663845e+04 + -3.621382182906358e+04 + -3.625241466608051e+04 + -3.629103836000000e+04 + -3.632969292147818e+04 + -3.636837837559645e+04 + -3.640709474306420e+04 + -3.644584203277781e+04 + -3.648462026000000e+04 + -3.652342944409004e+04 + -3.656226960170399e+04 + -3.660114075023598e+04 + -3.664004290875195e+04 + -3.667897608999997e+04 + -3.671794030512006e+04 + -3.675693557547587e+04 + -3.679596192058374e+04 + -3.683501935339895e+04 + -3.687410788999997e+04 + -3.691322754870546e+04 + -3.695237834723054e+04 + -3.699156030212950e+04 + -3.703077342834224e+04 + -3.707001774000000e+04 + -3.710929325203417e+04 + -3.714859998429457e+04 + -3.718793795692067e+04 + -3.722730718808447e+04 + -3.726670768999997e+04 + -3.730613947399677e+04 + -3.734560256273038e+04 + -3.738509697431393e+04 + -3.742462271654350e+04 + -3.746417980999997e+04 + -3.750376828013417e+04 + -3.754338813660564e+04 + -3.758303939267375e+04 + -3.762272207245150e+04 + -3.766243618999997e+04 + -3.770218175518502e+04 + -3.774195878945678e+04 + -3.778176731257176e+04 + -3.782160733741773e+04 + -3.786147888000000e+04 + -3.790138195864309e+04 + -3.794131659132003e+04 + -3.798128279431332e+04 + -3.802128058166662e+04 + -3.806130996999997e+04 + -3.810137097737330e+04 + -3.814146362046005e+04 + -3.818158791697343e+04 + -3.822174388615621e+04 + -3.826193153999997e+04 + -3.830215088938532e+04 + -3.834240195971708e+04 + -3.838268477130731e+04 + -3.842299933188884e+04 + -3.846334565999997e+04 + -3.850372377842891e+04 + -3.854413369633216e+04 + -3.858457542816681e+04 + -3.862504900090371e+04 + -3.866555443000000e+04 + -3.870609172486966e+04 + -3.874666090332934e+04 + -3.878726198409722e+04 + -3.882789498399841e+04 + -3.886855991999997e+04 + -3.890925680946781e+04 + -3.894998567054054e+04 + -3.899074651957592e+04 + -3.903153937021765e+04 + -3.907236423999997e+04 + -3.911322114736439e+04 + -3.915411010384250e+04 + -3.919503112747372e+04 + -3.923598424716172e+04 + -3.927696946999997e+04 + -3.931798680038228e+04 + -3.935903628964794e+04 + -3.940011793259103e+04 + -3.944123164809230e+04 + -3.948237770000000e+04 + -3.952355620675491e+04 + -3.956476584662273e+04 + -3.960600801403134e+04 + -3.964728735507886e+04 + -3.968859104999997e+04 + 1.819886919039960e+01 + 1.816405130921966e+01 + 1.812918280091599e+01 + 1.809426424653407e+01 + 1.805929622711938e+01 + 1.802427932371738e+01 + 1.798921411737356e+01 + 1.795410118913339e+01 + 1.791894112004234e+01 + 1.788373449114589e+01 + 1.784848188348950e+01 + 1.781318387811866e+01 + 1.777784105607885e+01 + 1.774245399841551e+01 + 1.770702328617416e+01 + 1.767154950040025e+01 + 1.763603322213927e+01 + 1.760047503243666e+01 + 1.756487551233793e+01 + 1.752923524288855e+01 + 1.749355480513398e+01 + 1.745783478011969e+01 + 1.742207574889119e+01 + 1.738627829249391e+01 + 1.735044299197335e+01 + 1.731457042837498e+01 + 1.727866118274428e+01 + 1.724271583612671e+01 + 1.720673496956776e+01 + 1.717071916411290e+01 + 1.713466900080759e+01 + 1.709858506069732e+01 + 1.706246792482757e+01 + 1.702631817424381e+01 + 1.699013638999150e+01 + 1.695392315311613e+01 + 1.691767904466317e+01 + 1.688140464567809e+01 + 1.684510053720637e+01 + 1.680876730029348e+01 + 1.677240551598491e+01 + 1.673601576532611e+01 + 1.669959862936258e+01 + 1.666315468913977e+01 + 1.662668452570318e+01 + 1.659018872009826e+01 + 1.655366785337049e+01 + 1.651712250656536e+01 + 1.648055326072833e+01 + 1.644396069690488e+01 + 1.640734539614049e+01 + 1.637070793948063e+01 + 1.633404890797076e+01 + 1.629736888265638e+01 + 1.626066844458294e+01 + 1.622394817479593e+01 + 1.618720865434082e+01 + 1.615045046426309e+01 + 1.611367418560821e+01 + 1.607688039942165e+01 + 1.604006968674890e+01 + 1.600324262863542e+01 + 1.596639980612668e+01 + 1.592954180026817e+01 + 1.589266919210536e+01 + 1.585578256268371e+01 + 1.581888249304872e+01 + 1.578196956424585e+01 + 1.574504435732057e+01 + 1.570810745331836e+01 + 1.567115943328470e+01 + 1.563420087826506e+01 + 1.559723236930490e+01 + 1.556025448744972e+01 + 1.552326781374499e+01 + 1.548627292923617e+01 + 1.544927041496875e+01 + 1.541226085198818e+01 + 1.537524482133997e+01 + 1.533822290406956e+01 + 1.530119568122245e+01 + 1.526416373384410e+01 + 1.522712764298000e+01 + 1.519008798967561e+01 + 1.515304535497640e+01 + 1.511600031992786e+01 + 1.507895346557546e+01 + 1.504190537296467e+01 + 1.500485662314096e+01 + 1.496780779714983e+01 + 1.493075947603672e+01 + 1.489371224084713e+01 + 1.485666667262652e+01 + 1.481962335242037e+01 + 1.478258286127416e+01 + 1.474554578023335e+01 + 1.470851269034343e+01 + 1.467148417264987e+01 + 1.463446080819813e+01 + 1.459744317803371e+01 + 1.456043186320207e+01 + 1.452342744474869e+01 + 1.448643050371903e+01 + 1.444944162115858e+01 + 1.441246137811282e+01 + 1.437549035562720e+01 + 1.433852913474722e+01 + 1.430157829651834e+01 + 1.426463842198604e+01 + 1.422771009219579e+01 + 1.419079388819307e+01 + 1.415389039102334e+01 + 1.411700018173210e+01 + 1.408012384136481e+01 + 1.404326195096694e+01 + 1.400641509158398e+01 + 1.396958384426139e+01 + 1.393276879004464e+01 + 1.389597050997923e+01 + 1.385918958511061e+01 + 1.382242659648426e+01 + 1.378568212514566e+01 + 1.374895675214029e+01 + 1.371225105851361e+01 + 1.367556562531110e+01 + 1.363890103357824e+01 + 1.360225786436050e+01 + 1.356563669870335e+01 + 1.352903811765228e+01 + 1.349246270225274e+01 + 1.345591103355023e+01 + 1.341938369259021e+01 + 1.338288126041816e+01 + 1.334640431807955e+01 + 1.330995344661985e+01 + 1.327352922708455e+01 + 1.323713224051912e+01 + 1.320076306796902e+01 + 1.316442229047974e+01 + 1.312811048909675e+01 + 1.309182824486553e+01 + 1.305557613883154e+01 + 1.301935475204027e+01 + 1.298316466553718e+01 + 1.294700646036776e+01 + 1.291088071757748e+01 + 1.287478801821181e+01 + 1.283872894331622e+01 + 1.280270407393619e+01 + 1.276671399111720e+01 + 1.273075927590472e+01 + 1.269484050934423e+01 + 1.265895827248119e+01 + 1.262311314636109e+01 + 1.258730571202939e+01 + 1.255153655053158e+01 + 1.251580624291313e+01 + 1.248011537021950e+01 + 1.244446451349619e+01 + 1.240885425378865e+01 + 1.237328517214237e+01 + 1.233775784960283e+01 + 1.230227286721548e+01 + 1.226683080602582e+01 + 1.223143224707931e+01 + 1.219607777142142e+01 + 1.216076796009764e+01 + 1.212550339415344e+01 + 1.209028465463429e+01 + 1.205511232258567e+01 + 1.201998697905305e+01 + 1.198490920508190e+01 + 1.194987958171770e+01 + 1.191489869000593e+01 + 1.187996711099206e+01 + 1.184508542572157e+01 + 1.181025421523992e+01 + 1.177547406059259e+01 + 1.174074554282507e+01 + 1.170606924298281e+01 + 1.167144574211130e+01 + 1.163687562125602e+01 + 1.160235946146243e+01 + 1.156789784377601e+01 + 1.153349134924223e+01 + 1.149914055890657e+01 + 1.146484605381452e+01 + 1.143060841501152e+01 + 1.139642822354308e+01 + 1.136230606045464e+01 + 1.132824250679170e+01 + 1.129423814359974e+01 + 1.126029355192421e+01 + 1.122640931281059e+01 + 1.119258600730437e+01 + 1.115882421645102e+01 + 1.112512452129600e+01 + 1.109148750288479e+01 + 1.105791374226288e+01 + 1.102440382047573e+01 + 1.099095831856882e+01 + 1.095757781758762e+01 + 1.092426289857761e+01 + 1.089101414258426e+01 + 1.085783213065305e+01 + 1.082471744382945e+01 + 1.079167066315892e+01 + 1.075869236968697e+01 + 1.072578314445905e+01 + 1.069294356852064e+01 + 1.066017422291721e+01 + 1.062747568869424e+01 + 1.059484854689720e+01 + 1.056229337857157e+01 + 1.052981076476282e+01 + 1.049740128651643e+01 + 1.046506552487787e+01 + 1.043280406089261e+01 + 1.040061747560614e+01 + 1.036850635006392e+01 + 1.033647126531143e+01 + 1.030451280239414e+01 + 1.027263154235754e+01 + 1.024082806624708e+01 + 1.020910295510825e+01 + 1.017745678998653e+01 + 1.014589015192738e+01 + 1.011440362197629e+01 + 1.008299778117872e+01 + 1.005167321058014e+01 + 1.002043049122605e+01 + 9.989270204161905e+00 + 9.958192930433185e+00 + 9.927199251085362e+00 + 9.896289747163912e+00 + 9.865464999714312e+00 + 9.834725589782032e+00 + 9.804072098412551e+00 + 9.773505106651340e+00 + 9.743025195543877e+00 + 9.712632946135633e+00 + 9.682328939472086e+00 + 9.652113756598707e+00 + 9.621987978560973e+00 + 9.591952186404358e+00 + 9.562006961174337e+00 + 9.532152883916382e+00 + 9.502390535675973e+00 + 9.472720497498580e+00 + 9.443143350429677e+00 + 9.413659675514742e+00 + 9.384270053799249e+00 + 9.354975066328668e+00 + 9.325775294148480e+00 + 9.296671318304156e+00 + 9.267663719841170e+00 + 9.238753079804999e+00 + 9.209939979241115e+00 + 9.181224999194994e+00 + 9.152608720712111e+00 + 9.124091724837941e+00 + 9.095674592617954e+00 + 9.067357905097630e+00 + 9.039142243322441e+00 + 9.011028188337864e+00 + 8.983016292445305e+00 + 8.955106689123129e+00 + 8.927299195109086e+00 + 8.899593619314148e+00 + 8.871989770649284e+00 + 8.844487458025464e+00 + 8.817086490353660e+00 + 8.789786676544839e+00 + 8.762587825509975e+00 + 8.735489746160036e+00 + 8.708492247405990e+00 + 8.681595138158814e+00 + 8.654798227329470e+00 + 8.628101323828931e+00 + 8.601504236568170e+00 + 8.575006774458155e+00 + 8.548608746409855e+00 + 8.522309961334242e+00 + 8.496110228142285e+00 + 8.470009355744955e+00 + 8.444007153053219e+00 + 8.418103428978053e+00 + 8.392297992430423e+00 + 8.366590652321301e+00 + 8.340981217561657e+00 + 8.315469497062459e+00 + 8.290055299734679e+00 + 8.264738434489287e+00 + 8.239518710237252e+00 + 8.214395935889545e+00 + 8.189369920357137e+00 + 8.164440472550996e+00 + 8.139607401382095e+00 + 8.114870515761401e+00 + 8.090229624599887e+00 + 8.065684536808520e+00 + 8.041235061298275e+00 + 8.016881006980118e+00 + 7.992622182765018e+00 + 7.968458397563950e+00 + 7.944389460287880e+00 + 7.920415179847782e+00 + 7.896535365154623e+00 + 7.872749825119373e+00 + 7.849058368653004e+00 + 7.825460804666484e+00 + 7.801956942070785e+00 + 7.778546589776878e+00 + 7.755229556695729e+00 + 7.732005651738313e+00 + 7.708874683815599e+00 + 7.685836461838553e+00 + 7.662890794718152e+00 + 7.640037491365358e+00 + 7.617276360691148e+00 + 7.594607211606489e+00 + 7.572029853022352e+00 + 7.549544093849708e+00 + 7.527149742999527e+00 + 7.504846609382777e+00 + 7.482634501910429e+00 + 7.460513229493455e+00 + 7.438482601042820e+00 + 7.416542425469502e+00 + 7.394692511684465e+00 + 7.372932668598682e+00 + 7.351262705123121e+00 + 7.329682430168755e+00 + 7.308191652646553e+00 + 7.286790181467484e+00 + 7.265477825542517e+00 + 7.244254393782626e+00 + 7.223119695098779e+00 + 7.202073538401945e+00 + 7.181115732603097e+00 + 7.160246086613204e+00 + 7.139464409343233e+00 + 7.118770509704159e+00 + 7.098164196606950e+00 + 7.077645278962576e+00 + 7.057213565682006e+00 + 7.036868865676214e+00 + 7.016610987856166e+00 + 6.996439741132834e+00 + 6.976354934417186e+00 + 6.956356376620197e+00 + 6.936443876652832e+00 + 6.916617243426063e+00 + 6.896876285850862e+00 + 6.877220812838198e+00 + 6.857650633299039e+00 + 6.838165556144358e+00 + 6.818765390285123e+00 + 6.799449944632306e+00 + 6.780219028096876e+00 + 6.761072449589803e+00 + 6.742010018022058e+00 + 6.723031542304612e+00 + 6.704136831348434e+00 + 6.685325694064492e+00 + 6.666597939363759e+00 + 6.647953376157203e+00 + 6.629391813355797e+00 + 6.610913059870509e+00 + 6.592516924612310e+00 + 6.574203216492170e+00 + 6.555971744421058e+00 + 6.537822317309946e+00 + 6.519754744069803e+00 + 6.501768833611600e+00 + 6.483864394846306e+00 + 6.466041236684891e+00 + 6.448299168038327e+00 + 6.430637997817582e+00 + 6.413057534933630e+00 + 6.395557588297437e+00 + 6.378137966819973e+00 + 6.360798479412210e+00 + 6.343538934985117e+00 + 6.326359142449666e+00 + 6.309258910716826e+00 + 6.292238048697567e+00 + 6.275296365302859e+00 + 6.258433669443673e+00 + 6.241649770030978e+00 + 6.224944475975745e+00 + 6.208317596188945e+00 + 6.191768939581546e+00 + 6.175298315064517e+00 + 6.158905531548833e+00 + 6.142590397945462e+00 + 6.126352723165372e+00 + 6.110192316119535e+00 + 6.094108985718921e+00 + 6.078102540874499e+00 + 6.062172790497243e+00 + 6.046319543498117e+00 + 6.030542608788095e+00 + 6.014841795278149e+00 + 5.999216911879244e+00 + 5.983667767502356e+00 + 5.968194171058448e+00 + 5.952795931458497e+00 + 5.937472857613470e+00 + 5.922224758434336e+00 + 5.907051442832068e+00 + 5.891952719717633e+00 + 5.876928398002004e+00 + 5.861978286596150e+00 + 5.847102194411042e+00 + 5.832299930357648e+00 + 5.817571303346941e+00 + 5.802916122289888e+00 + 5.788334196097460e+00 + 5.773825333680629e+00 + 5.759389343950365e+00 + 5.745026035817635e+00 + 5.730735218193414e+00 + 5.716516699988668e+00 + 5.702370290114368e+00 + 5.688295797481486e+00 + 5.674293031000990e+00 + 5.660361799583851e+00 + 5.646501912141039e+00 + 5.632713177583526e+00 + 5.618995404822279e+00 + 5.605348402768269e+00 + 5.591771980332467e+00 + 5.578265946425844e+00 + 5.564830109959368e+00 + 5.551464279844009e+00 + 5.538168264990741e+00 + 5.524941874310529e+00 + 5.511784916714348e+00 + 5.498697201113163e+00 + 5.485678536417949e+00 + 5.472728731539672e+00 + 5.459847595389306e+00 + 5.447031950668924e+00 + 5.434258417587328e+00 + 5.421509378931341e+00 + 5.408790202589383e+00 + 5.396101725502927e+00 + 5.383442552149385e+00 + 5.370812984345576e+00 + 5.358213038980171e+00 + 5.345642546622406e+00 + 5.333101463094469e+00 + 5.320589727671893e+00 + 5.308107264570896e+00 + 5.295654006874094e+00 + 5.283229887334054e+00 + 5.270834837715376e+00 + 5.258468789944041e+00 + 5.246131676000384e+00 + 5.233823428164784e+00 + 5.221543979540164e+00 + 5.209293263422158e+00 + 5.197071212820756e+00 + 5.184877759775030e+00 + 5.172712837552656e+00 + 5.160576380473933e+00 + 5.148468322098453e+00 + 5.136388595958436e+00 + 5.124337135726472e+00 + 5.112313875048248e+00 + 5.100318748362747e+00 + 5.088351690727316e+00 + 5.076412636057685e+00 + 5.064501518839616e+00 + 5.052618274474592e+00 + 5.040762837425790e+00 + 5.028935142799786e+00 + 5.017135126690616e+00 + 5.005362723409764e+00 + 4.993617868159293e+00 + 4.981900498164465e+00 + 4.970210548621045e+00 + 4.958547954855285e+00 + 4.946912653797631e+00 + 4.935304581602165e+00 + 4.923723674430540e+00 + 4.912169869219803e+00 + 4.900643102722201e+00 + 4.889143311682699e+00 + 4.877670433140317e+00 + 4.866224404407401e+00 + 4.854805162838305e+00 + 4.843412645620055e+00 + 4.832046790470948e+00 + 4.820707535433044e+00 + 4.809394818449872e+00 + 4.798108577205163e+00 + 4.786848749644187e+00 + 4.775615274885523e+00 + 4.764408091170769e+00 + 4.753227136475494e+00 + 4.742072350593207e+00 + 4.730943672320340e+00 + 4.719841039761676e+00 + 4.708764392793848e+00 + 4.697713670947889e+00 + 4.686688813308958e+00 + 4.675689760077749e+00 + 4.664716450776882e+00 + 4.653768824355144e+00 + 4.642846821849820e+00 + 4.631950383864691e+00 + 4.621079449911801e+00 + 4.610233960271208e+00 + 4.599413855814052e+00 + 4.588619077726644e+00 + 4.577849566594633e+00 + 4.567105263324451e+00 + 4.556386109499332e+00 + 4.545692045920192e+00 + 4.535023013830533e+00 + 4.524378955558838e+00 + 4.513759812087895e+00 + 4.503165524982657e+00 + 4.492596037765430e+00 + 4.482051291649467e+00 + 4.471531227881555e+00 + 4.461035790191556e+00 + 4.450564920902143e+00 + 4.440118561957067e+00 + 4.429696656605228e+00 + 4.419299148008472e+00 + 4.408925979093191e+00 + 4.398577092681390e+00 + 4.388252432408600e+00 + 4.377951941955149e+00 + 4.367675563647617e+00 + 4.357423241911437e+00 + 4.347194921954801e+00 + 4.336990545878941e+00 + 4.326810057907454e+00 + 4.316653403787664e+00 + 4.306520526395858e+00 + 4.296411370095723e+00 + 4.286325880845187e+00 + 4.276264002847886e+00 + 4.266225680614548e+00 + 4.256210859410026e+00 + 4.246219484255977e+00 + 4.236251500111860e+00 + 4.226306852214246e+00 + 4.216385487075638e+00 + 4.206487350454588e+00 + 4.196612386873075e+00 + 4.186760542346591e+00 + 4.176931763412669e+00 + 4.167125996517556e+00 + 4.157343187931778e+00 + 4.147583283659849e+00 + 4.137846229598449e+00 + 4.128131973158434e+00 + 4.118440461620333e+00 + 4.108771640965568e+00 + 4.099125458381889e+00 + 4.089501861348100e+00 + 4.079900796648449e+00 + 4.070322211621146e+00 + 4.060766054068578e+00 + 4.051232271987449e+00 + 4.041720812903752e+00 + 4.032231624389123e+00 + 4.022764654839604e+00 + 4.013319852102482e+00 + 4.003897163974346e+00 + 3.994496539375360e+00 + 3.985117927005318e+00 + 3.975761275162846e+00 + 3.966426531955261e+00 + 3.957113646635731e+00 + 3.947822569087183e+00 + 3.938553247905866e+00 + 3.929305631685672e+00 + 3.920079669524097e+00 + 3.910875311427285e+00 + 3.901692507178291e+00 + 3.892531206204140e+00 + 3.883391358396833e+00 + 3.874272913657344e+00 + 3.865175821793835e+00 + 3.856100032879644e+00 + 3.847045497225832e+00 + 3.838012165320722e+00 + 3.828999987558789e+00 + 3.820008914413926e+00 + 3.811038896595444e+00 + 3.802089885254187e+00 + 3.793161831237620e+00 + 3.784254684890362e+00 + 3.775368398252875e+00 + 3.766502923100440e+00 + 3.757658209579520e+00 + 3.748834209295840e+00 + 3.740030874433959e+00 + 3.731248156753105e+00 + 3.722486008117639e+00 + 3.713744380387517e+00 + 3.705023225280349e+00 + 3.696322495070079e+00 + 3.687642142318910e+00 + 3.678982119432947e+00 + 3.670342378801804e+00 + 3.661722872867589e+00 + 3.653123554235752e+00 + 3.644544376023881e+00 + 3.635985291634798e+00 + 3.627446254192749e+00 + 3.618927216218316e+00 + 3.610428130335504e+00 + 3.601948951051534e+00 + 3.593489632231203e+00 + 3.585050126838500e+00 + 3.576630388153148e+00 + 3.568230370457688e+00 + 3.559850028500160e+00 + 3.551489315016799e+00 + 3.543148183918080e+00 + 3.534826590807976e+00 + 3.526524489935861e+00 + 3.518241835011243e+00 + 3.509978579991082e+00 + 3.501734680809670e+00 + 3.493510092697708e+00 + 3.485304769188242e+00 + 3.477118665289576e+00 + 3.468951736765370e+00 + 3.460803939385451e+00 + 3.452675227557384e+00 + 3.444565556219660e+00 + 3.436474882078371e+00 + 3.428403160505571e+00 + 3.420350346590145e+00 + 3.412316396304707e+00 + 3.404301265792477e+00 + 3.396304911176187e+00 + 3.388327288491305e+00 + 3.380368354160161e+00 + 3.372428064623584e+00 + 3.364506375833607e+00 + 3.356603244211646e+00 + 3.348718626640740e+00 + 3.340852480236101e+00 + 3.333004761693424e+00 + 3.325175427644580e+00 + 3.317364435527293e+00 + 3.309571742493616e+00 + 3.301797305408617e+00 + 3.294041081414062e+00 + 3.286303028194412e+00 + 3.278583103726798e+00 + 3.270881265443555e+00 + 3.263197471011956e+00 + 3.255531678390248e+00 + 3.247883845209004e+00 + 3.240253929642225e+00 + 3.232641890347850e+00 + 3.225047684908113e+00 + 3.217471271425511e+00 + 3.209912608923088e+00 + 3.202371655667555e+00 + 3.194848370181972e+00 + 3.187342711658790e+00 + 3.179854638881827e+00 + 3.172384110380890e+00 + 3.164931084707051e+00 + 3.157495521384956e+00 + 3.150077380058776e+00 + 3.142676619877814e+00 + 3.135293200073600e+00 + 3.127927079946843e+00 + 3.120578218905895e+00 + 3.113246577218404e+00 + 3.105932114937815e+00 + 3.098634790846833e+00 + 3.091354565058342e+00 + 3.084091398271724e+00 + 3.076845250485478e+00 + 3.069616081755534e+00 + 3.062403852244847e+00 + 3.055208522175611e+00 + 3.048030052549899e+00 + 3.040868404439169e+00 + 3.033723537524932e+00 + 3.026595412527113e+00 + 3.019483991059011e+00 + 3.012389234177664e+00 + 3.005311102428361e+00 + 2.998249556405217e+00 + 2.991204558170310e+00 + 2.984176069441633e+00 + 2.977164051262350e+00 + 2.970168464749357e+00 + 2.963189271462946e+00 + 2.956226433370632e+00 + 2.949279912203107e+00 + 2.942349669912605e+00 + 2.935435668735732e+00 + 2.928537870478263e+00 + 2.921656236941307e+00 + 2.914790730227295e+00 + 2.907941313056430e+00 + 2.901107948114753e+00 + 2.894290597705669e+00 + 2.887489224186201e+00 + 2.880703790004707e+00 + 2.873934257778331e+00 + 2.867180590721052e+00 + 2.860442752157053e+00 + 2.853720705060128e+00 + 2.847014412004529e+00 + 2.840323835735203e+00 + 2.833648939802342e+00 + 2.826989688360529e+00 + 2.820346045051714e+00 + 2.813717971838249e+00 + 2.807105432809830e+00 + 2.800508392739872e+00 + 2.793926814285955e+00 + 2.787360661385112e+00 + 2.780809898806732e+00 + 2.774274490318202e+00 + 2.767754399418414e+00 + 2.761249589980850e+00 + 2.754760027276096e+00 + 2.748285675987487e+00 + 2.741826499952791e+00 + 2.735382462817888e+00 + 2.728953529693038e+00 + 2.722539666591989e+00 + 2.716140837199994e+00 + 2.709757005928732e+00 + 2.703388138477217e+00 + 2.697034199719776e+00 + 2.690695154612969e+00 + 2.684370968516638e+00 + 2.678061606722120e+00 + 2.671767034388748e+00 + 2.665487216663756e+00 + 2.659222119420964e+00 + 2.652971708466898e+00 + 2.646735949170681e+00 + 2.640514807328682e+00 + 2.634308248813400e+00 + 2.628116239351709e+00 + 2.621938745026414e+00 + 2.615775731978081e+00 + 2.609627166192403e+00 + 2.603493014125890e+00 + 2.597373242133807e+00 + 2.591267815972944e+00 + 2.585176702619338e+00 + 2.579099869106871e+00 + 2.573037281022092e+00 + 2.566988905295047e+00 + 2.560954709471109e+00 + 2.554934660124372e+00 + 2.548928723658388e+00 + 2.542936866838776e+00 + 2.536959057454498e+00 + 2.530995262744329e+00 + 2.525045449466102e+00 + 2.519109584724339e+00 + 2.513187636248425e+00 + 2.507279571988880e+00 + 2.501385358975255e+00 + 2.495504964649074e+00 + 2.489638357011514e+00 + 2.483785503877440e+00 + 2.477946372993181e+00 + 2.472120932153270e+00 + 2.466309149445497e+00 + 2.460510993028623e+00 + 2.454726431004756e+00 + 2.448955431279121e+00 + 2.443197962222979e+00 + 2.437453992749473e+00 + 2.431723490873920e+00 + 2.426006424702946e+00 + 2.420302763016215e+00 + 2.414612474849150e+00 + 2.408935529060812e+00 + 2.403271894158963e+00 + 2.397621539164549e+00 + 2.391984432987164e+00 + 2.386360544006294e+00 + 2.380749841635097e+00 + 2.375152295549876e+00 + 2.369567874852914e+00 + 2.363996548806182e+00 + 2.358438286639424e+00 + 2.352893057314327e+00 + 2.347360830870365e+00 + 2.341841577523073e+00 + 2.336335266205249e+00 + 2.330841866367291e+00 + 2.325361348038680e+00 + 2.319893681430557e+00 + 2.314438836350213e+00 + 2.308996782525400e+00 + 2.303567490482044e+00 + 2.298150930506590e+00 + 2.292747072570897e+00 + 2.287355886755380e+00 + 2.281977343203661e+00 + 2.276611412298537e+00 + 2.271258065216148e+00 + 2.265917272602903e+00 + 2.260589004572198e+00 + 2.255273232290290e+00 + 2.249969926489583e+00 + 2.244679057245269e+00 + 2.239400595715412e+00 + 2.234134513404378e+00 + 2.228880781619142e+00 + 2.223639370658599e+00 + 2.218410251587366e+00 + 2.213193396784469e+00 + 2.207988776668006e+00 + 2.202796362022329e+00 + 2.197616125439223e+00 + 2.192448038696543e+00 + 2.187292072961057e+00 + 2.182148199232985e+00 + 2.177016389630072e+00 + 2.171896616273634e+00 + 2.166788850281173e+00 + 2.161693063906081e+00 + 2.156609229682959e+00 + 2.151537319229026e+00 + 2.146477304323076e+00 + 2.141429157241363e+00 + 2.136392850965788e+00 + 2.131368357474257e+00 + 2.126355648357996e+00 + 2.121354696293322e+00 + 2.116365474412895e+00 + 2.111387955644702e+00 + 2.106422111701483e+00 + 2.101467915161587e+00 + 2.096525339256323e+00 + 2.091594356269759e+00 + 2.086674939419586e+00 + 2.081767062556874e+00 + 2.076870697792070e+00 + 2.071985817585338e+00 + 2.067112395385072e+00 + 2.062250405333861e+00 + 2.057399820685575e+00 + 2.052560613690088e+00 + 2.047732757766644e+00 + 2.042916226752948e+00 + 2.038110994473384e+00 + 2.033317034456518e+00 + 2.028534320143143e+00 + 2.023762825026766e+00 + 2.019002522786289e+00 + 2.014253387371010e+00 + 2.009515392980107e+00 + 2.004788513366984e+00 + 2.000072722331970e+00 + 1.995367994084281e+00 + 1.990674302528751e+00 + 1.985991621875054e+00 + 1.981319927135009e+00 + 1.976659192144684e+00 + 1.972009390577169e+00 + 1.967370497271926e+00 + 1.962742487042732e+00 + 1.958125334423246e+00 + 1.953519013586483e+00 + 1.948923499307053e+00 + 1.944338766657414e+00 + 1.939764790288013e+00 + 1.935201544608222e+00 + 1.930649004305472e+00 + 1.926107145248571e+00 + 1.921575942429786e+00 + 1.917055370205972e+00 + 1.912545403920843e+00 + 1.908046018818900e+00 + 1.903557190023903e+00 + 1.899078893306255e+00 + 1.894611103753838e+00 + 1.890153795996532e+00 + 1.885706946418488e+00 + 1.881270530851602e+00 + 1.876844524133795e+00 + 1.872428902057808e+00 + 1.868023640555410e+00 + 1.863628715350075e+00 + 1.859244102235784e+00 + 1.854869776995271e+00 + 1.850505715382929e+00 + 1.846151893336109e+00 + 1.841808287113597e+00 + 1.837474873250078e+00 + 1.833151627500553e+00 + 1.828838525627810e+00 + 1.824535544058799e+00 + 1.820242659570220e+00 + 1.815959848693184e+00 + 1.811687087278560e+00 + 1.807424351697017e+00 + 1.803171618832717e+00 + 1.798928865872756e+00 + 1.794696068965575e+00 + 1.790473204047284e+00 + 1.786260248130819e+00 + 1.782057178544553e+00 + 1.777863972462110e+00 + 1.773680606381003e+00 + 1.769507057021776e+00 + 1.765343301422396e+00 + 1.761189316819223e+00 + 1.757045080431908e+00 + 1.752910569457795e+00 + 1.748785761158555e+00 + 1.744670632638435e+00 + 1.740565161064786e+00 + 1.736469324420783e+00 + 1.732383100261040e+00 + 1.728306465595185e+00 + 1.724239397824929e+00 + 1.720181874717039e+00 + 1.716133874165168e+00 + 1.712095373335323e+00 + 1.708066350172305e+00 + 1.704046783495335e+00 + 1.700036650108550e+00 + 1.696035927584721e+00 + 1.692044595228792e+00 + 1.688062630443701e+00 + 1.684090010658294e+00 + 1.680126714513943e+00 + 1.676172720475745e+00 + 1.672228006742128e+00 + 1.668292551293985e+00 + 1.664366332353613e+00 + 1.660449328407859e+00 + 1.656541518177628e+00 + 1.652642880144113e+00 + 1.648753392536404e+00 + 1.644873033459887e+00 + 1.641001782122826e+00 + 1.637139617814172e+00 + 1.633286518510287e+00 + 1.629442462986198e+00 + 1.625607430332010e+00 + 1.621781398816242e+00 + 1.617964347766720e+00 + 1.614156256896447e+00 + 1.610357104501046e+00 + 1.606566869528149e+00 + 1.602785531410984e+00 + 1.599013068753329e+00 + 1.595249460964023e+00 + 1.591494688099418e+00 + 1.587748729316499e+00 + 1.584011563661260e+00 + 1.580283170366302e+00 + 1.576563528988570e+00 + 1.572852618951484e+00 + 1.569150419635379e+00 + 1.565456911345500e+00 + 1.561772073862832e+00 + 1.558095886092752e+00 + 1.554428327545355e+00 + 1.550769378486889e+00 + 1.547119019605569e+00 + 1.543477229774924e+00 + 1.539843988467248e+00 + 1.536219276909035e+00 + 1.532603074921508e+00 + 1.528995362108325e+00 + 1.525396118748853e+00 + 1.521805324624735e+00 + 1.518222960029189e+00 + 1.514649006474386e+00 + 1.511083443428444e+00 + 1.507526250258141e+00 + 1.503977408416516e+00 + 1.500436898413773e+00 + 1.496904700394939e+00 + 1.493380795245864e+00 + 1.489865163193006e+00 + 1.486357784560608e+00 + 1.482858641108495e+00 + 1.479367713055452e+00 + 1.475884980161266e+00 + 1.472410424507970e+00 + 1.468944026943859e+00 + 1.465485767394345e+00 + 1.462035627142284e+00 + 1.458593587684006e+00 + 1.455159630175685e+00 + 1.451733734815932e+00 + 1.448315882934573e+00 + 1.444906056735074e+00 + 1.441504236540235e+00 + 1.438110403156542e+00 + 1.434724538391589e+00 + 1.431346623689521e+00 + 1.427976640530080e+00 + 1.424614570482120e+00 + 1.421260394583763e+00 + 1.417914094214416e+00 + 1.414575651360096e+00 + 1.411245047472556e+00 + 1.407922264020345e+00 + 1.404607282835409e+00 + 1.401300085863839e+00 + 1.398000654669480e+00 + 1.394708970355802e+00 + 1.391425015768138e+00 + 1.388148773421020e+00 + 1.384880223867722e+00 + 1.381619349200565e+00 + 1.378366132070768e+00 + 1.375120554406545e+00 + 1.371882598139520e+00 + 1.368652245409923e+00 + 1.365429478679455e+00 + 1.362214280063096e+00 + 1.359006631544336e+00 + 1.355806515436745e+00 + 1.352613914349998e+00 + 1.349428810943258e+00 + 1.346251187518860e+00 + 1.343081026346772e+00 + 1.339918309785753e+00 + 1.336763020411036e+00 + 1.333615141050152e+00 + 1.330474654669161e+00 + 1.327341544046787e+00 + 1.324215791384803e+00 + 1.321097378844990e+00 + 1.317986290481943e+00 + 1.314882509034005e+00 + 1.311786015892630e+00 + 1.308696795174431e+00 + 1.305614830505543e+00 + 1.302540104058677e+00 + 1.299472598550161e+00 + 1.296412297281467e+00 + 1.293359183920592e+00 + 1.290313241355424e+00 + 1.287274452437336e+00 + 1.284242800425524e+00 + 1.281218268758570e+00 + 1.278200840971845e+00 + 1.275190500607243e+00 + 1.272187230813499e+00 + 1.269191014801793e+00 + 1.266201836184906e+00 + 1.263219678244913e+00 + 1.260244524601725e+00 + 1.257276359774731e+00 + 1.254315166659556e+00 + 1.251360928048363e+00 + 1.248413628680403e+00 + 1.245473252809453e+00 + 1.242539783866816e+00 + 1.239613204552696e+00 + 1.236693499248504e+00 + 1.233780652886690e+00 + 1.230874648458874e+00 + 1.227975469917747e+00 + 1.225083101914005e+00 + 1.222197528080320e+00 + 1.219318732110022e+00 + 1.216446698153004e+00 + 1.213581411119631e+00 + 1.210722855169846e+00 + 1.207871013783067e+00 + 1.205025871252235e+00 + 1.202187412198626e+00 + 1.199355621285526e+00 + 1.196530482820713e+00 + 1.193711981112556e+00 + 1.190900100540550e+00 + 1.188094825240966e+00 + 1.185296139829988e+00 + 1.182504029475636e+00 + 1.179718478336080e+00 + 1.176939470878376e+00 + 1.174166992368249e+00 + 1.171401026812090e+00 + 1.168641558834780e+00 + 1.165888574598542e+00 + 1.163142057955574e+00 + 1.160401992993886e+00 + 1.157668365957760e+00 + 1.154941161055026e+00 + 1.152220362585152e+00 + 1.149505957086653e+00 + 1.146797929075238e+00 + 1.144096262593592e+00 + 1.141400943461258e+00 + 1.138711957183691e+00 + 1.136029288721929e+00 + 1.133352922646698e+00 + 1.130682844729473e+00 + 1.128019040871752e+00 + 1.125361495026299e+00 + 1.122710192813904e+00 + 1.120065120718956e+00 + 1.117426263156838e+00 + 1.114793605478939e+00 + 1.112167133798269e+00 + 1.109546832911627e+00 + 1.106932688501257e+00 + 1.104324686965964e+00 + 1.101722813149718e+00 + 1.099127052563592e+00 + 1.096537391606932e+00 + 1.093953815681381e+00 + 1.091376310184498e+00 + 1.088804860939953e+00 + 1.086239454210304e+00 + 1.083680075729654e+00 + 1.081126710640132e+00 + 1.078579345517792e+00 + 1.076037966489376e+00 + 1.073502558540405e+00 + 1.070973108232868e+00 + 1.068449602380304e+00 + 1.065932027008114e+00 + 1.063420367175204e+00 + 1.060914608734753e+00 + 1.058414739347230e+00 + 1.055920744292059e+00 + 1.053432609053344e+00 + 1.050950321750202e+00 + 1.048473868029229e+00 + 1.046003233045439e+00 + 1.043538404185668e+00 + 1.041079368203120e+00 + 1.038626111099421e+00 + 1.036178618557632e+00 + 1.033736877638818e+00 + 1.031300875738098e+00 + 1.028870598541307e+00 + 1.026446032279889e+00 + 1.024027163804816e+00 + 1.021613979927458e+00 + 1.019206467749757e+00 + 1.016804614201984e+00 + 1.014408404816817e+00 + 1.012017826403038e+00 + 1.009632866898133e+00 + 1.007253512894913e+00 + 1.004879750675442e+00 + 1.002511566831996e+00 + 1.000148949144661e+00 + 9.977918847145494e-01 + 9.954403597206510e-01 + 9.930943614528360e-01 + 9.907538771638876e-01 + 9.884188936740620e-01 + 9.860893982148410e-01 + 9.837653780132122e-01 + 9.814468200970017e-01 + 9.791337116433633e-01 + 9.768260399168340e-01 + 9.745237922184445e-01 + 9.722269558574113e-01 + 9.699355181124353e-01 + 9.676494662336389e-01 + 9.653687876978898e-01 + 9.630934699744002e-01 + 9.608235003289306e-01 + 9.585588662885656e-01 + 9.562995554410040e-01 + 9.540455551859868e-01 + 9.517968530931009e-01 + 9.495534368089285e-01 + 9.473152938858177e-01 + 9.450824119343849e-01 + 9.428547786515334e-01 + 9.406323818265262e-01 + 9.384152091378831e-01 + 9.362032482272097e-01 + 9.339964869112888e-01 + 9.317949130473703e-01 + 9.295985145023118e-01 + 9.274072791539573e-01 + 9.252211948367504e-01 + 9.230402494034374e-01 + 9.208644309329761e-01 + 9.186937274260252e-01 + 9.165281267878318e-01 + 9.143676170947915e-01 + 9.122121864230248e-01 + 9.100618228160294e-01 + 9.079165144228375e-01 + 9.057762494101416e-01 + 9.036410159434681e-01 + 9.015108022852656e-01 + 8.993855966397851e-01 + 8.972653871281971e-01 + 8.951501621417270e-01 + 8.930399100640233e-01 + 8.909346191274636e-01 + 8.888342777366450e-01 + 8.867388743114113e-01 + 8.846483971822027e-01 + 8.825628348501290e-01 + 8.804821758633001e-01 + 8.784064086961259e-01 + 8.763355218399448e-01 + 8.742695038377348e-01 + 8.722083433110044e-01 + 8.701520288632569e-01 + 8.681005490981059e-01 + 8.660538926659707e-01 + 8.640120483003465e-01 + 8.619750047630993e-01 + 8.599427507590320e-01 + 8.579152750196485e-01 + 8.558925663402844e-01 + 8.538746136199826e-01 + 8.518614056598554e-01 + 8.498529312415619e-01 + 8.478491793952468e-01 + 8.458501390477595e-01 + 8.438557990450950e-01 + 8.418661484572341e-01 + 8.398811762963594e-01 + 8.379008715007146e-01 + 8.359252231704962e-01 + 8.339542204095658e-01 + 8.319878522912720e-01 + 8.300261079597043e-01 + 8.280689765769632e-01 + 8.261164473121154e-01 + 8.241685093974929e-01 + 8.222251520716921e-01 + 8.202863645582569e-01 + 8.183521361157176e-01 + 8.164224560764958e-01 + 8.144973138478017e-01 + 8.125766987264839e-01 + 8.106606000443977e-01 + 8.087490072689457e-01 + 8.068419098143643e-01 + 8.049392971238847e-01 + 8.030411587461223e-01 + 8.011474841608780e-01 + 7.992582628576694e-01 + 7.973734844327619e-01 + 7.954931384582111e-01 + 7.936172145055983e-01 + 7.917457022017643e-01 + 7.898785912282000e-01 + 7.880158712799969e-01 + 7.861575320154112e-01 + 7.843035631939634e-01 + 7.824539546080472e-01 + 7.806086959262468e-01 + 7.787677769423338e-01 + 7.769311875560446e-01 + 7.750989176225404e-01 + 7.732709569908990e-01 + 7.714472955211897e-01 + 7.696279231052234e-01 + 7.678128297211314e-01 + 7.660020053954494e-01 + 7.641954400346342e-01 + 7.623931236302374e-01 + 7.605950462809293e-01 + 7.588011979868697e-01 + 7.570115688113509e-01 + 7.552261489288551e-01 + 7.534449284973839e-01 + 7.516678976340316e-01 + 7.498950464317325e-01 + 7.481263651057625e-01 + 7.463618439154999e-01 + 7.446014731170009e-01 + 7.428452429690515e-01 + 7.410931437272748e-01 + 7.393451656574506e-01 + 7.376012991597445e-01 + 7.358615346418724e-01 + 7.341258624174279e-01 + 7.323942728636854e-01 + 7.306667564033948e-01 + 7.289433034811010e-01 + 7.272239046205010e-01 + 7.255085503342084e-01 + 7.237972310231134e-01 + 7.220899372662432e-01 + 7.203866596915919e-01 + 7.186873887451811e-01 + 7.169921150530395e-01 + 7.153008293485266e-01 + 7.136135222426403e-01 + 7.119301843519804e-01 + 7.102508063464710e-01 + 7.085753789925818e-01 + 7.069038930034782e-01 + 7.052363390742027e-01 + 7.035727080638011e-01 + 7.019129907750439e-01 + 7.002571779481277e-01 + 6.986052604437083e-01 + 6.969572291357313e-01 + 6.953130748988353e-01 + 6.936727887020229e-01 + 6.920363614522840e-01 + 6.904037839913317e-01 + 6.887750473773369e-01 + 6.871501426160253e-01 + 6.855290605871678e-01 + 6.839117924040222e-01 + 6.822983292013320e-01 + 6.806886620123715e-01 + 6.790827818618205e-01 + 6.774806798407976e-01 + 6.758823471448717e-01 + 6.742877749381793e-01 + 6.726969543916550e-01 + 6.711098767181217e-01 + 6.695265330883416e-01 + 6.679469147191758e-01 + 6.663710129607012e-01 + 6.647988190816884e-01 + 6.632303243195694e-01 + 6.616655199754741e-01 + 6.601043974442724e-01 + 6.585469481315596e-01 + 6.569931633409668e-01 + 6.554430344698831e-01 + 6.538965529680326e-01 + 6.523537102216029e-01 + 6.508144977390269e-01 + 6.492789070902356e-01 + 6.477469297088367e-01 + 6.462185570471877e-01 + 6.446937806333431e-01 + 6.431725921186934e-01 + 6.416549831028872e-01 + 6.401409451291091e-01 + 6.386304698205402e-01 + 6.371235488250266e-01 + 6.356201737964903e-01 + 6.341203364004929e-01 + 6.326240283504814e-01 + 6.311312414023209e-01 + 6.296419672600960e-01 + 6.281561976359686e-01 + 6.266739242956670e-01 + 6.251951391194286e-01 + 6.237198339322255e-01 + 6.222480004492102e-01 + 6.207796305731890e-01 + 6.193147162378443e-01 + 6.178532493027741e-01 + 6.163952216645050e-01 + 6.149406252489860e-01 + 6.134894519984210e-01 + 6.120416938491114e-01 + 6.105973427932988e-01 + 6.091563909278429e-01 + 6.077188302116784e-01 + 6.062846526061169e-01 + 6.048538502614210e-01 + 6.034264152733694e-01 + 6.020023397068369e-01 + 6.005816156757759e-01 + 5.991642353027361e-01 + 5.977501907311656e-01 + 5.963394741611191e-01 + 5.949320777981851e-01 + 5.935279938331314e-01 + 5.921272144271874e-01 + 5.907297318514260e-01 + 5.893355384452119e-01 + 5.879446264415406e-01 + 5.865569881004723e-01 + 5.851726157460215e-01 + 5.837915017610071e-01 + 5.824136384878618e-01 + 5.810390182425506e-01 + 5.796676334763571e-01 + 5.782994765837256e-01 + 5.769345398883866e-01 + 5.755728159164172e-01 + 5.742142971757567e-01 + 5.728589760824728e-01 + 5.715068451452269e-01 + 5.701578968814059e-01 + 5.688121237820786e-01 + 5.674695184134964e-01 + 5.661300733449661e-01 + 5.647937811172017e-01 + 5.634606343850478e-01 + 5.621306258016221e-01 + 5.608037479323033e-01 + 5.594799934047830e-01 + 5.581593548937616e-01 + 5.568418251014721e-01 + 5.555273967809645e-01 + 5.542160626584934e-01 + 5.529078153599613e-01 + 5.516026476869607e-01 + 5.503005524879265e-01 + 5.490015224489507e-01 + 5.477055503706971e-01 + 5.464126291394721e-01 + 5.451227516109934e-01 + 5.438359105997491e-01 + 5.425520989229732e-01 + 5.412713094911021e-01 + 5.399935352390902e-01 + 5.387187691034094e-01 + 5.374470040087604e-01 + 5.361782329113639e-01 + 5.349124487848638e-01 + 5.336496445514358e-01 + 5.323898132502169e-01 + 5.311329480044003e-01 + 5.298790417206248e-01 + 5.286280874266170e-01 + 5.273800783238198e-01 + 5.261350074387794e-01 + 5.248928678300107e-01 + 5.236536526638225e-01 + 5.224173550711992e-01 + 5.211839681868924e-01 + 5.199534851768893e-01 + 5.187258992380968e-01 + 5.175012035574781e-01 + 5.162793913049719e-01 + 5.150604557657682e-01 + 5.138443902158373e-01 + 5.126311878387027e-01 + 5.114208419243966e-01 + 5.102133457996210e-01 + 5.090086927559088e-01 + 5.078068761246483e-01 + 5.066078892445277e-01 + 5.054117254264059e-01 + 5.042183780707232e-01 + 5.030278406156905e-01 + 5.018401064512819e-01 + 5.006551689399448e-01 + 4.994730214827681e-01 + 4.982936576173500e-01 + 4.971170708196209e-01 + 4.959432545161117e-01 + 4.947722021784954e-01 + 4.936039073158257e-01 + 4.924383634713059e-01 + 4.912755642215030e-01 + 4.901155031029966e-01 + 4.889581736357913e-01 + 4.878035694426762e-01 + 4.866516841617155e-01 + 4.855025114095978e-01 + 4.843560447562852e-01 + 4.832122778219742e-01 + 4.820712043014287e-01 + 4.809328179063641e-01 + 4.797971123302187e-01 + 4.786640812406847e-01 + 4.775337183184862e-01 + 4.764060173050261e-01 + 4.752809720122764e-01 + 4.741585762111217e-01 + 4.730388236459319e-01 + 4.719217080626099e-01 + 4.708072233097221e-01 + 4.696953632391141e-01 + 4.685861216425942e-01 + 4.674794923970907e-01 + 4.663754693817960e-01 + 4.652740464091041e-01 + 4.641752174102053e-01 + 4.630789763262008e-01 + 4.619853169869301e-01 + 4.608942333544824e-01 + 4.598057194461572e-01 + 4.587197692038988e-01 + 4.576363766028796e-01 + 4.565555356433393e-01 + 4.554772403132010e-01 + 4.544014845983438e-01 + 4.533282625263808e-01 + 4.522575682421708e-01 + 4.511893957834002e-01 + 4.501237391381080e-01 + 4.490605924783677e-01 + 4.479999499003729e-01 + 4.469418054464263e-01 + 4.458861533467765e-01 + 4.448329877503518e-01 + 4.437823027182840e-01 + 4.427340924804984e-01 + 4.416883512661691e-01 + 4.406450732493647e-01 + 4.396042525949710e-01 + 4.385658835318161e-01 + 4.375299603591805e-01 + 4.364964773180230e-01 + 4.354654286529535e-01 + 4.344368086449737e-01 + 4.334106115882546e-01 + 4.323868317973252e-01 + 4.313654636098987e-01 + 4.303465013528135e-01 + 4.293299393608014e-01 + 4.283157719943348e-01 + 4.273039936389254e-01 + 4.262945986862648e-01 + 4.252875815207442e-01 + 4.242829365388910e-01 + 4.232806581737543e-01 + 4.222807409163533e-01 + 4.212831792180283e-01 + 4.202879675160039e-01 + 4.192951002821390e-01 + 4.183045720190468e-01 + 4.173163772308877e-01 + 4.163305103895081e-01 + 4.153469660738844e-01 + 4.143657388899517e-01 + 4.133868232899575e-01 + 4.124102138463511e-01 + 4.114359052122226e-01 + 4.104638919136763e-01 + 4.094941685453110e-01 + 4.085267297797091e-01 + 4.075615702608746e-01 + 4.065986846247451e-01 + 4.056380675149961e-01 + 4.046797136100241e-01 + 4.037236175791848e-01 + 4.027697740867438e-01 + 4.018181778668817e-01 + 4.008688236537185e-01 + 3.999217061664900e-01 + 3.989768201804763e-01 + 3.980341604347806e-01 + 3.970937216243767e-01 + 3.961554986257766e-01 + 3.952194862561716e-01 + 3.942856791749246e-01 + 3.933540722797152e-01 + 3.924246604937682e-01 + 3.914974386057962e-01 + 3.905724013991230e-01 + 3.896495437210211e-01 + 3.887288605266558e-01 + 3.878103467384126e-01 + 3.868939972364765e-01 + 3.859798068708922e-01 + 3.850677705778728e-01 + 3.841578833358520e-01 + 3.832501400994028e-01 + 3.823445358203205e-01 + 3.814410654593080e-01 + 3.805397240004402e-01 + 3.796405064500310e-01 + 3.787434078281850e-01 + 3.778484231550217e-01 + 3.769555474293507e-01 + 3.760647756653435e-01 + 3.751761029803938e-01 + 3.742895244689736e-01 + 3.734050351897463e-01 + 3.725226302131253e-01 + 3.716423046174657e-01 + 3.707640535037849e-01 + 3.698878720450124e-01 + 3.690137553825144e-01 + 3.681416986254492e-01 + 3.672716969834960e-01 + 3.664037456169829e-01 + 3.655378396196581e-01 + 3.646739742353085e-01 + 3.638121447254376e-01 + 3.629523462980791e-01 + 3.620945741078652e-01 + 3.612388233782960e-01 + 3.603850894480597e-01 + 3.595333675652969e-01 + 3.586836529581271e-01 + 3.578359408972814e-01 + 3.569902267099870e-01 + 3.561465057284826e-01 + 3.553047732480240e-01 + 3.544650245489661e-01 + 3.536272549462773e-01 + 3.527914598373741e-01 + 3.519576346281267e-01 + 3.511257746893044e-01 + 3.502958753213151e-01 + 3.494679319027064e-01 + 3.486419398734283e-01 + 3.478178946841421e-01 + 3.469957917542031e-01 + 3.461756264906874e-01 + 3.453573943334770e-01 + 3.445410907466465e-01 + 3.437267112053919e-01 + 3.429142511753236e-01 + 3.421037061357636e-01 + 3.412950715914754e-01 + 3.404883430871680e-01 + 3.396835161220511e-01 + 3.388805861778083e-01 + 3.380795488742585e-01 + 3.372803997559251e-01 + 3.364831342820857e-01 + 3.356877480644493e-01 + 3.348942367473924e-01 + 3.341025959445759e-01 + 3.333128211902062e-01 + 3.325249080644813e-01 + 3.317388522349806e-01 + 3.309546493345364e-01 + 3.301722950123494e-01 + 3.293917849546400e-01 + 3.286131147929292e-01 + 3.278362801760134e-01 + 3.270612768173198e-01 + 3.262881004205997e-01 + 3.255167466776044e-01 + 3.247472112772871e-01 + 3.239794899591130e-01 + 3.232135784952971e-01 + 3.224494726613195e-01 + 3.216871681318617e-01 + 3.209266606147330e-01 + 3.201679460077643e-01 + 3.194110200806869e-01 + 3.186558785671545e-01 + 3.179025173467112e-01 + 3.171509322207308e-01 + 3.164011189537259e-01 + 3.156530734030461e-01 + 3.149067914068442e-01 + 3.141622688005587e-01 + 3.134195014964591e-01 + 3.126784853846941e-01 + 3.119392163234083e-01 + 3.112016901727622e-01 + 3.104659028247113e-01 + 3.097318502149898e-01 + 3.089995283319454e-01 + 3.082689330721005e-01 + 3.075400602663161e-01 + 3.068129059444455e-01 + 3.060874661234649e-01 + 3.053637367362504e-01 + 3.046417137079979e-01 + 3.039213930418579e-01 + 3.032027708160949e-01 + 3.024858429572295e-01 + 3.017706054435225e-01 + 3.010570543910775e-01 + 3.003451858060657e-01 + 2.996349956970866e-01 + 2.989264801519126e-01 + 2.982196352437723e-01 + 2.975144570351090e-01 + 2.968109415899733e-01 + 2.961090850010212e-01 + 2.954088833712375e-01 + 2.947103327960572e-01 + 2.940134293733013e-01 + 2.933181692489545e-01 + 2.926245486550200e-01 + 2.919325636534309e-01 + 2.912422103009782e-01 + 2.905534849067459e-01 + 2.898663836341845e-01 + 2.891809025716850e-01 + 2.884970379499296e-01 + 2.878147859946116e-01 + 2.871341429080666e-01 + 2.864551048929091e-01 + 2.857776681836698e-01 + 2.851018290276999e-01 + 2.844275836247271e-01 + 2.837549281975342e-01 + 2.830838590248334e-01 + 2.824143724756855e-01 + 2.817464648036653e-01 + 2.810801321714598e-01 + 2.804153709678348e-01 + 2.797521775256636e-01 + 2.790905480675208e-01 + 2.784304789557904e-01 + 2.777719665515425e-01 + 2.771150071618110e-01 + 2.764595971293084e-01 + 2.758057328280932e-01 + 2.751534106502780e-01 + 2.745026269461392e-01 + 2.738533780697315e-01 + 2.732056604090851e-01 + 2.725594703611444e-01 + 2.719148043516682e-01 + 2.712716588414341e-01 + 2.706300302048149e-01 + 2.699899148252890e-01 + 2.693513091897057e-01 + 2.687142097803046e-01 + 2.680786130525787e-01 + 2.674445154290628e-01 + 2.668119133864937e-01 + 2.661808034370396e-01 + 2.655511820876730e-01 + 2.649230457933240e-01 + 2.642963910212357e-01 + 2.636712143601993e-01 + 2.630475123173707e-01 + 2.624252813696634e-01 + 2.618045181320832e-01 + 2.611852191534089e-01 + 2.605673809346502e-01 + 2.599510000768786e-01 + 2.593360731482656e-01 + 2.587225966857610e-01 + 2.581105673020023e-01 + 2.574999816218594e-01 + 2.568908362541323e-01 + 2.562831277681717e-01 + 2.556768527608984e-01 + 2.550720078808991e-01 + 2.544685898158210e-01 + 2.538665952029344e-01 + 2.532660206235198e-01 + 2.526668627769018e-01 + 2.520691183671673e-01 + 2.514727840477828e-01 + 2.508778564832650e-01 + 2.502843323450471e-01 + 2.496922083184503e-01 + 2.491014811741685e-01 + 2.485121476502302e-01 + 2.479242043768022e-01 + 2.473376480750936e-01 + 2.467524755330193e-01 + 2.461686835577825e-01 + 2.455862688342147e-01 + 2.450052280577099e-01 + 2.444255580815592e-01 + 2.438472556710649e-01 + 2.432703175621061e-01 + 2.426947405763887e-01 + 2.421205215374063e-01 + 2.415476572469543e-01 + 2.409761444715131e-01 + 2.404059800179439e-01 + 2.398371607406130e-01 + 2.392696835287547e-01 + 2.387035451998669e-01 + 2.381387425460481e-01 + 2.375752724876774e-01 + 2.370131319015487e-01 + 2.364523176042576e-01 + 2.358928264341899e-01 + 2.353346553264918e-01 + 2.347778012643444e-01 + 2.342222610470114e-01 + 2.336680315481888e-01 + 2.331151097630285e-01 + 2.325634926002070e-01 + 2.320131769932555e-01 + 2.314641599195968e-01 + 2.309164382391606e-01 + 2.303700088852327e-01 + 2.298248689205450e-01 + 2.292810152383314e-01 + 2.287384447851956e-01 + 2.281971546639490e-01 + 2.276571417710518e-01 + 2.271184030444375e-01 + 2.265809356255705e-01 + 2.260447364485637e-01 + 2.255098024678978e-01 + 2.249761308577043e-01 + 2.244437185775247e-01 + 2.239125625576918e-01 + 2.233826599324974e-01 + 2.228540078061592e-01 + 2.223266032159731e-01 + 2.218004431318401e-01 + 2.212755246546400e-01 + 2.207518449320089e-01 + 2.202294009962271e-01 + 2.197081899542086e-01 + 2.191882089472320e-01 + 2.186694550184439e-01 + 2.181519252528444e-01 + 2.176356167922184e-01 + 2.171205268020538e-01 + 2.166066524055817e-01 + 2.160939907132310e-01 + 2.155825389406823e-01 + 2.150722942170351e-01 + 2.145632535999474e-01 + 2.140554143128364e-01 + 2.135487735713488e-01 + 2.130433285376680e-01 + 2.125390763961598e-01 + 2.120360143225986e-01 + 2.115341394866721e-01 + 2.110334491301104e-01 + 2.105339404911263e-01 + 2.100356107692428e-01 + 2.095384571593845e-01 + 2.090424768875718e-01 + 2.085476672286868e-01 + 2.080540254649787e-01 + 2.075615488420417e-01 + 2.070702345483775e-01 + 2.065800798220710e-01 + 2.060910819584345e-01 + 2.056032383022174e-01 + 2.051165461242101e-01 + 2.046310026800104e-01 + 2.041466052775261e-01 + 2.036633511978056e-01 + 2.031812377402102e-01 + 2.027002622847228e-01 + 2.022204221329609e-01 + 2.017417145657832e-01 + 2.012641369577989e-01 + 2.007876866366623e-01 + 2.003123609141739e-01 + 1.998381571847571e-01 + 1.993650728223044e-01 + 1.988931051658790e-01 + 1.984222515391579e-01 + 1.979525093566432e-01 + 1.974838760798574e-01 + 1.970163490394448e-01 + 1.965499255924087e-01 + 1.960846031634822e-01 + 1.956203792216227e-01 + 1.951572511456189e-01 + 1.946952162469976e-01 + 1.942342720345011e-01 + 1.937744160087473e-01 + 1.933156455855021e-01 + 1.928579581617409e-01 + 1.924013511601302e-01 + 1.919458220512063e-01 + 1.914913683490730e-01 + 1.910379875218400e-01 + 1.905856769690048e-01 + 1.901344342343178e-01 + 1.896842568245793e-01 + 1.892351421122624e-01 + 1.887870876905098e-01 + 1.883400911382459e-01 + 1.878941498265747e-01 + 1.874492612709913e-01 + 1.870054230608876e-01 + 1.865626327533365e-01 + 1.861208878413260e-01 + 1.856801858136446e-01 + 1.852405242290795e-01 + 1.848019006561455e-01 + 1.843643126785636e-01 + 1.839277579090688e-01 + 1.834922338318696e-01 + 1.830577379260484e-01 + 1.826242679175190e-01 + 1.821918214082343e-01 + 1.817603959062191e-01 + 1.813299890419420e-01 + 1.809005984096844e-01 + 1.804722215715290e-01 + 1.800448561676805e-01 + 1.796184998471616e-01 + 1.791931502428031e-01 + 1.787688049585256e-01 + 1.783454616010165e-01 + 1.779231178061076e-01 + 1.775017712916434e-01 + 1.770814197185338e-01 + 1.766620606643876e-01 + 1.762436917707559e-01 + 1.758263107475443e-01 + 1.754099153420786e-01 + 1.749945031691721e-01 + 1.745800718793870e-01 + 1.741666192327871e-01 + 1.737541428912765e-01 + 1.733426405128906e-01 + 1.729321098263452e-01 + 1.725225486037993e-01 + 1.721139545737653e-01 + 1.717063253659770e-01 + 1.712996586978482e-01 + 1.708939523524782e-01 + 1.704892041326627e-01 + 1.700854117308264e-01 + 1.696825728315613e-01 + 1.692806852413719e-01 + 1.688797467224855e-01 + 1.684797550119726e-01 + 1.680807078836749e-01 + 1.676826031516292e-01 + 1.672854386236076e-01 + 1.668892120257625e-01 + 1.664939211364053e-01 + 1.660995637803862e-01 + 1.657061377653481e-01 + 1.653136408872399e-01 + 1.649220709518862e-01 + 1.645314258210089e-01 + 1.641417032838678e-01 + 1.637529010922642e-01 + 1.633650171697226e-01 + 1.629780493848234e-01 + 1.625919955117979e-01 + 1.622068533722165e-01 + 1.618226208359533e-01 + 1.614392958070308e-01 + 1.610568761629498e-01 + 1.606753597398519e-01 + 1.602947443525945e-01 + 1.599150279328092e-01 + 1.595362083976419e-01 + 1.591582835876941e-01 + 1.587812513806259e-01 + 1.584051096895935e-01 + 1.580298564486488e-01 + 1.576554895336735e-01 + 1.572820068494865e-01 + 1.569094063844064e-01 + 1.565376859935812e-01 + 1.561668435389086e-01 + 1.557968770322644e-01 + 1.554277844430587e-01 + 1.550595637013555e-01 + 1.546922127252644e-01 + 1.543257294352004e-01 + 1.539601117962810e-01 + 1.535953578647676e-01 + 1.532314655336160e-01 + 1.528684326732572e-01 + 1.525062574192351e-01 + 1.521449377506904e-01 + 1.517844715391133e-01 + 1.514248568043021e-01 + 1.510660915767141e-01 + 1.507081738546517e-01 + 1.503511015850067e-01 + 1.499948728018673e-01 + 1.496394855976185e-01 + 1.492849379312514e-01 + 1.489312277844460e-01 + 1.485783532079666e-01 + 1.482263122946834e-01 + 1.478751030581660e-01 + 1.475247234388987e-01 + 1.471751715323068e-01 + 1.468264454469171e-01 + 1.464785432366154e-01 + 1.461314629165984e-01 + 1.457852025452889e-01 + 1.454397602433564e-01 + 1.450951340257908e-01 + 1.447513219420110e-01 + 1.444083221508569e-01 + 1.440661327109938e-01 + 1.437247516822454e-01 + 1.433841772072484e-01 + 1.430444073901374e-01 + 1.427054403077353e-01 + 1.423672740371242e-01 + 1.420299067307786e-01 + 1.416933365389871e-01 + 1.413575615237460e-01 + 1.410225798197031e-01 + 1.406883895901710e-01 + 1.403549889435149e-01 + 1.400223760032770e-01 + 1.396905489310608e-01 + 1.393595059398338e-01 + 1.390292451224472e-01 + 1.386997645496112e-01 + 1.383710625136722e-01 + 1.380431371824567e-01 + 1.377159866326167e-01 + 1.373896091055998e-01 + 1.370640027752214e-01 + 1.367391657592925e-01 + 1.364150963290927e-01 + 1.360917926783734e-01 + 1.357692529162330e-01 + 1.354474752994454e-01 + 1.351264580487222e-01 + 1.348061993122180e-01 + 1.344866973181728e-01 + 1.341679503229942e-01 + 1.338499565682958e-01 + 1.335327142013654e-01 + 1.332162214320106e-01 + 1.329004765902875e-01 + 1.325854779258548e-01 + 1.322712236439755e-01 + 1.319577119415115e-01 + 1.316449410589970e-01 + 1.313329092858229e-01 + 1.310216149439378e-01 + 1.307110562100912e-01 + 1.304012313067827e-01 + 1.300921386594473e-01 + 1.297837764741680e-01 + 1.294761429396481e-01 + 1.291692364743583e-01 + 1.288630553353233e-01 + 1.285575977268189e-01 + 1.282528620001082e-01 + 1.279488464604472e-01 + 1.276455493846629e-01 + 1.273429690961771e-01 + 1.270411039265308e-01 + 1.267399521912149e-01 + 1.264395121606617e-01 + 1.261397821624623e-01 + 1.258407605612416e-01 + 1.255424456647032e-01 + 1.252448358003494e-01 + 1.249479293256652e-01 + 1.246517245985584e-01 + 1.243562199440227e-01 + 1.240614136699035e-01 + 1.237673041530004e-01 + 1.234738897833225e-01 + 1.231811689369122e-01 + 1.228891399541342e-01 + 1.225978011897444e-01 + 1.223071510285007e-01 + 1.220171878485729e-01 + 1.217279100221359e-01 + 1.214393159216827e-01 + 1.211514039479249e-01 + 1.208641725134118e-01 + 1.205776200272462e-01 + 1.202917448643591e-01 + 1.200065453999493e-01 + 1.197220200388544e-01 + 1.194381672203820e-01 + 1.191549853738593e-01 + 1.188724728867774e-01 + 1.185906281982059e-01 + 1.183094497403516e-01 + 1.180289358802264e-01 + 1.177490851172322e-01 + 1.174698959467804e-01 + 1.171913666738082e-01 + 1.169134957966049e-01 + 1.166362818664215e-01 + 1.163597231976005e-01 + 1.160838182295854e-01 + 1.158085654926154e-01 + 1.155339634301463e-01 + 1.152600105152270e-01 + 1.149867052503320e-01 + 1.147140461081585e-01 + 1.144420315115333e-01 + 1.141706598827868e-01 + 1.138999297876022e-01 + 1.136298397724347e-01 + 1.133603883121320e-01 + 1.130915738073908e-01 + 1.128233947404400e-01 + 1.125558496925181e-01 + 1.122889371862514e-01 + 1.120226557224089e-01 + 1.117570037925019e-01 + 1.114919798574519e-01 + 1.112275824488763e-01 + 1.109638101830490e-01 + 1.107006615250319e-01 + 1.104381349837027e-01 + 1.101762291935612e-01 + 1.099149425817940e-01 + 1.096542736337556e-01 + 1.093942210611990e-01 + 1.091347833750166e-01 + 1.088759590309823e-01 + 1.086177465849946e-01 + 1.083601446468375e-01 + 1.081031518187680e-01 + 1.078467666394599e-01 + 1.075909876365943e-01 + 1.073358133695979e-01 + 1.070812424691808e-01 + 1.068272734935852e-01 + 1.065739049762412e-01 + 1.063211355273724e-01 + 1.060689637753818e-01 + 1.058173883167701e-01 + 1.055664076522099e-01 + 1.053160204177178e-01 + 1.050662253083730e-01 + 1.048170208178578e-01 + 1.045684055544151e-01 + 1.043203782231123e-01 + 1.040729373745031e-01 + 1.038260815947437e-01 + 1.035798095388146e-01 + 1.033341198496129e-01 + 1.030890111353506e-01 + 1.028444819874746e-01 + 1.026005310752415e-01 + 1.023571570589419e-01 + 1.021143585605686e-01 + 1.018721342063349e-01 + 1.016304826397723e-01 + 1.013894025261397e-01 + 1.011488925387934e-01 + 1.009089513225976e-01 + 1.006695774897203e-01 + 1.004307697412456e-01 + 1.001925267601519e-01 + 9.995484714928184e-02 + 9.971772961029543e-02 + 9.948117285600222e-02 + 9.924517553207431e-02 + 9.900973631553432e-02 + 9.877485389836264e-02 + 9.854052696321766e-02 + 9.830675420513860e-02 + 9.807353432053509e-02 + 9.784086599274402e-02 + 9.760874792461041e-02 + 9.737717882679013e-02 + 9.714615739564664e-02 + 9.691568233864475e-02 + 9.668575237168565e-02 + 9.645636620785530e-02 + 9.622752256709627e-02 + 9.599922017186617e-02 + 9.577145773476621e-02 + 9.554423398421420e-02 + 9.531754766036228e-02 + 9.509139748951084e-02 + 9.486578220406576e-02 + 9.464070054491446e-02 + 9.441615125061270e-02 + 9.419213306609350e-02 + 9.396864474309072e-02 + 9.374568502973517e-02 + 9.352325267700241e-02 + 9.330134644110695e-02 + 9.307996508088016e-02 + 9.285910736057421e-02 + 9.263877204899980e-02 + 9.241895790580944e-02 + 9.219966369924239e-02 + 9.198088821288498e-02 + 9.176263021506216e-02 + 9.154488848131032e-02 + 9.132766180778470e-02 + 9.111094897456264e-02 + 9.089474876186458e-02 + 9.067905996494766e-02 + 9.046388137755315e-02 + 9.024921179550818e-02 + 9.003505002157913e-02 + 8.982139485394958e-02 + 8.960824509461397e-02 + 8.939559956013639e-02 + 8.918345705664489e-02 + 8.897181638949078e-02 + 8.876067638185769e-02 + 8.855003585659860e-02 + 8.833989363307079e-02 + 8.813024852794749e-02 + 8.792109937217552e-02 + 8.771244500411417e-02 + 8.750428424959714e-02 + 8.729661594151820e-02 + 8.708943892038122e-02 + 8.688275202450782e-02 + 8.667655409894154e-02 + 8.647084399500883e-02 + 8.626562056133152e-02 + 8.606088264727801e-02 + 8.585662910508592e-02 + 8.565285879275303e-02 + 8.544957056834111e-02 + 8.524676329099723e-02 + 8.504443583456130e-02 + 8.484258706983651e-02 + 8.464121585953427e-02 + 8.444032107793580e-02 + 8.423990160173883e-02 + 8.403995630660985e-02 + 8.384048408000297e-02 + 8.364148380739438e-02 + 8.344295436518863e-02 + 8.324489464319758e-02 + 8.304730353631981e-02 + 8.285017993778075e-02 + 8.265352274723796e-02 + 8.245733086610936e-02 + 8.226160319325253e-02 + 8.206633863307464e-02 + 8.187153609323948e-02 + 8.167719448173898e-02 + 8.148331271120264e-02 + 8.128988969775824e-02 + 8.109692435907066e-02 + 8.090441561287741e-02 + 8.071236238052074e-02 + 8.052076359329230e-02 + 8.032961817777003e-02 + 8.013892505875637e-02 + 7.994868317110082e-02 + 7.975889145271257e-02 + 7.956954884117817e-02 + 7.938065427030368e-02 + 7.919220668490153e-02 + 7.900420503735811e-02 + 7.881664826626886e-02 + 7.862953532075569e-02 + 7.844286516277023e-02 + 7.825663674598571e-02 + 7.807084902471831e-02 + 7.788550095723919e-02 + 7.770059150356423e-02 + 7.751611963202924e-02 + 7.733208431808952e-02 + 7.714848452080215e-02 + 7.696531920714285e-02 + 7.678258736267791e-02 + 7.660028796508990e-02 + 7.641841999039113e-02 + 7.623698241865759e-02 + 7.605597423593993e-02 + 7.587539442823407e-02 + 7.569524197859213e-02 + 7.551551588287067e-02 + 7.533621514035872e-02 + 7.515733874432233e-02 + 7.497888568830134e-02 + 7.480085497230264e-02 + 7.462324560850588e-02 + 7.444605660139321e-02 + 7.426928695221670e-02 + 7.409293566891996e-02 + 7.391700176673981e-02 + 7.374148426439617e-02 + 7.356638217848553e-02 + 7.339169452681224e-02 + 7.321742033027905e-02 + 7.304355861552074e-02 + 7.287010840815517e-02 + 7.269706873346385e-02 + 7.252443862192341e-02 + 7.235221710900047e-02 + 7.218040323296335e-02 + 7.200899602923243e-02 + 7.183799453782064e-02 + 7.166739780328910e-02 + 7.149720486628935e-02 + 7.132741477295296e-02 + 7.115802657678706e-02 + 7.098903933028558e-02 + 7.082045208266741e-02 + 7.065226388294948e-02 + 7.048447379884398e-02 + 7.031708089501212e-02 + 7.015008422428966e-02 + 6.998348285005500e-02 + 6.981727584178653e-02 + 6.965146227138255e-02 + 6.948604121024822e-02 + 6.932101172952453e-02 + 6.915637290177927e-02 + 6.899212381049466e-02 + 6.882826353831664e-02 + 6.866479115768206e-02 + 6.850170575553913e-02 + 6.833900642241270e-02 + 6.817669223943085e-02 + 6.801476230171261e-02 + 6.785321570853013e-02 + 6.769205154760372e-02 + 6.753126891476798e-02 + 6.737086691265436e-02 + 6.721084464422118e-02 + 6.705120121123045e-02 + 6.689193571639933e-02 + 6.673304726820108e-02 + 6.657453497707803e-02 + 6.641639795452120e-02 + 6.625863531334618e-02 + 6.610124617104705e-02 + 6.594422964866814e-02 + 6.578758486543299e-02 + 6.563131094165708e-02 + 6.547540700031643e-02 + 6.531987216906585e-02 + 6.516470557837332e-02 + 6.500990635997374e-02 + 6.485547364331327e-02 + 6.470140656204018e-02 + 6.454770425495959e-02 + 6.439436585565442e-02 + 6.424139050518185e-02 + 6.408877735587415e-02 + 6.393652554954243e-02 + 6.378463422860429e-02 + 6.363310254304622e-02 + 6.348192964216136e-02 + 6.333111467891862e-02 + 6.318065681259971e-02 + 6.303055519506789e-02 + 6.288080898167979e-02 + 6.273141734118348e-02 + 6.258237943694549e-02 + 6.243369443027962e-02 + 6.228536148636779e-02 + 6.213737977420020e-02 + 6.198974846629291e-02 + 6.184246673830131e-02 + 6.169553375986874e-02 + 6.154894870195880e-02 + 6.140271075002260e-02 + 6.125681908485271e-02 + 6.111127288458502e-02 + 6.096607133498923e-02 + 6.082121362330186e-02 + 6.067669893778977e-02 + 6.053252647002724e-02 + 6.038869541057570e-02 + 6.024520494995262e-02 + 6.010205428441878e-02 + 5.995924261312039e-02 + 5.981676913606451e-02 + 5.967463305068377e-02 + 5.953283356000520e-02 + 5.939136987405440e-02 + 5.925024120274393e-02 + 5.910944675075109e-02 + 5.896898572024133e-02 + 5.882885733338872e-02 + 5.868906080796419e-02 + 5.854959534943303e-02 + 5.841046018167366e-02 + 5.827165452667381e-02 + 5.813317759547484e-02 + 5.799502862000658e-02 + 5.785720683145133e-02 + 5.771971144629215e-02 + 5.758254169802959e-02 + 5.744569682168989e-02 + 5.730917604038396e-02 + 5.717297859181381e-02 + 5.703710371771920e-02 + 5.690155065150715e-02 + 5.676631863514842e-02 + 5.663140691176988e-02 + 5.649681471550019e-02 + 5.636254129818796e-02 + 5.622858591623749e-02 + 5.609494780637568e-02 + 5.596162621719655e-02 + 5.582862040612746e-02 + 5.569592962469259e-02 + 5.556355312969728e-02 + 5.543149018219354e-02 + 5.529974004038473e-02 + 5.516830196534387e-02 + 5.503717522021812e-02 + 5.490635906464716e-02 + 5.477585276550550e-02 + 5.464565559606692e-02 + 5.451576682310098e-02 + 5.438618571514530e-02 + 5.425691154653305e-02 + 5.412794360021059e-02 + 5.399928115030668e-02 + 5.387092346220070e-02 + 5.374286982722595e-02 + 5.361511953114147e-02 + 5.348767184376742e-02 + 5.336052605560348e-02 + 5.323368145967647e-02 + 5.310713734132736e-02 + 5.298089299097181e-02 + 5.285494769998652e-02 + 5.272930075794034e-02 + 5.260395145882699e-02 + 5.247889910164562e-02 + 5.235414299012888e-02 + 5.222968242359068e-02 + 5.210551670083303e-02 + 5.198164512479836e-02 + 5.185806699751586e-02 + 5.173478162567831e-02 + 5.161178832737659e-02 + 5.148908640731603e-02 + 5.136667516904735e-02 + 5.124455393713079e-02 + 5.112272202541198e-02 + 5.100117874145336e-02 + 5.087992340415878e-02 + 5.075895533879782e-02 + 5.063827387090040e-02 + 5.051787831631814e-02 + 5.039776799903027e-02 + 5.027794224956293e-02 + 5.015840039018700e-02 + 5.003914174911826e-02 + 4.992016566217095e-02 + 4.980147146451835e-02 + 4.968305848565534e-02 + 4.956492605368701e-02 + 4.944707351707827e-02 + 4.932950021762891e-02 + 4.921220548403189e-02 + 4.909518865563611e-02 + 4.897844908008390e-02 + 4.886198610990230e-02 + 4.874579909103274e-02 + 4.862988736593407e-02 + 4.851425027765261e-02 + 4.839888718215429e-02 + 4.828379743688660e-02 + 4.816898039344981e-02 + 4.805443540666063e-02 + 4.794016183433197e-02 + 4.782615903628512e-02 + 4.771242637003347e-02 + 4.759896319572905e-02 + 4.748576888074151e-02 + 4.737284278529100e-02 + 4.726018427180961e-02 + 4.714779271742187e-02 + 4.703566749191604e-02 + 4.692380796083007e-02 + 4.681221349433169e-02 + 4.670088346648602e-02 + 4.658981725459431e-02 + 4.647901423861556e-02 + 4.636847379587147e-02 + 4.625819530165282e-02 + 4.614817813294777e-02 + 4.603842167462240e-02 + 4.592892531626435e-02 + 4.581968844113481e-02 + 4.571071043517037e-02 + 4.560199068716930e-02 + 4.549352858172214e-02 + 4.538532351139604e-02 + 4.527737487551017e-02 + 4.516968206047602e-02 + 4.506224446147816e-02 + 4.495506148526871e-02 + 4.484813252205796e-02 + 4.474145696749865e-02 + 4.463503423038843e-02 + 4.452886371233129e-02 + 4.442294481444441e-02 + 4.431727694184067e-02 + 4.421185950249874e-02 + 4.410669190344264e-02 + 4.400177354999718e-02 + 4.389710385863231e-02 + 4.379268224561385e-02 + 4.368850811842072e-02 + 4.358458089102907e-02 + 4.348089998166713e-02 + 4.337746480946211e-02 + 4.327427479290449e-02 + 4.317132935104961e-02 + 4.306862790538591e-02 + 4.296616988021409e-02 + 4.286395470044128e-02 + 4.276198178905291e-02 + 4.266025057462805e-02 + 4.255876048879523e-02 + 4.245751096050290e-02 + 4.235650142249979e-02 + 4.225573130868781e-02 + 4.215520004666404e-02 + 4.205490707118502e-02 + 4.195485182346907e-02 + 4.185503374212908e-02 + 4.175545226547943e-02 + 4.165610683241490e-02 + 4.155699688308426e-02 + 4.145812186261490e-02 + 4.135948121993713e-02 + 4.126107439911159e-02 + 4.116290084444992e-02 + 4.106496000347922e-02 + 4.096725133001662e-02 + 4.086977427671171e-02 + 4.077252829257245e-02 + 4.067551282880393e-02 + 4.057872734137224e-02 + 4.048217129089726e-02 + 4.038584413229004e-02 + 4.028974532214836e-02 + 4.019387432351806e-02 + 4.009823059884492e-02 + 4.000281360871143e-02 + 3.990762281263916e-02 + 3.981265768084040e-02 + 3.971791768404272e-02 + 3.962340228400397e-02 + 3.952911094818634e-02 + 3.943504314911767e-02 + 3.934119836154264e-02 + 3.924757605343093e-02 + 3.915417569553582e-02 + 3.906099677494528e-02 + 3.896803876462888e-02 + 3.887530113159737e-02 + 3.878278335807980e-02 + 3.869048493136602e-02 + 3.859840533540703e-02 + 3.850654403935494e-02 + 3.841490053099362e-02 + 3.832347430867683e-02 + 3.823226484521238e-02 + 3.814127162485314e-02 + 3.805049414568009e-02 + 3.795993189727254e-02 + 3.786958436838804e-02 + 3.777945104950908e-02 + 3.768953143238456e-02 + 3.759982501241896e-02 + 3.751033128919119e-02 + 3.742104976390229e-02 + 3.733197993101921e-02 + 3.724312127956968e-02 + 3.715447331892652e-02 + 3.706603555538170e-02 + 3.697780748276271e-02 + 3.688978860838169e-02 + 3.680197844047203e-02 + 3.671437648083043e-02 + 3.662698223859462e-02 + 3.653979522528233e-02 + 3.645281495050149e-02 + 3.636604092401768e-02 + 3.627947265609178e-02 + 3.619310965841749e-02 + 3.610695145022991e-02 + 3.602099755007137e-02 + 3.593524746722872e-02 + 3.584970072102484e-02 + 3.576435683502647e-02 + 3.567921532660211e-02 + 3.559427571435374e-02 + 3.550953752116312e-02 + 3.542500027693078e-02 + 3.534066350384148e-02 + 3.525652672170999e-02 + 3.517258946309138e-02 + 3.508885125628003e-02 + 3.500531162678291e-02 + 3.492197010940873e-02 + 3.483882623525817e-02 + 3.475587953263527e-02 + 3.467312953919303e-02 + 3.459057578876406e-02 + 3.450821781198862e-02 + 3.442605515251815e-02 + 3.434408734999198e-02 + 3.426231393723483e-02 + 3.418073445581654e-02 + 3.409934844816886e-02 + 3.401815545480084e-02 + 3.393715501912345e-02 + 3.385634668688416e-02 + 3.377573000557660e-02 + 3.369530452244349e-02 + 3.361506978424043e-02 + 3.353502533757853e-02 + 3.345517073173793e-02 + 3.337550551749302e-02 + 3.329602924688495e-02 + 3.321674147756795e-02 + 3.313764176353938e-02 + 3.305872964905672e-02 + 3.298000469915312e-02 + 3.290146647870388e-02 + 3.282311452928582e-02 + 3.274494841443989e-02 + 3.266696770524984e-02 + 3.258917195579586e-02 + 3.251156072835543e-02 + 3.243413358906379e-02 + 3.235689009597382e-02 + 3.227982981604406e-02 + 3.220295232152501e-02 + 3.212625717796667e-02 + 3.204974395335523e-02 + 3.197341221860803e-02 + 3.189726154412311e-02 + 3.182129149811844e-02 + 3.174550164996338e-02 + 3.166989158019418e-02 + 3.159446086552403e-02 + 3.151920907660396e-02 + 3.144413578635114e-02 + 3.136924057603768e-02 + 3.129452303273958e-02 + 3.121998273049806e-02 + 3.114561924523454e-02 + 3.107143216011856e-02 + 3.099742105799270e-02 + 3.092358552474080e-02 + 3.084992514925186e-02 + 3.077643951122557e-02 + 3.070312819305190e-02 + 3.062999078663620e-02 + 3.055702688363699e-02 + 3.048423607160382e-02 + 3.041161793396379e-02 + 3.033917207015753e-02 + 3.026689807846873e-02 + 3.019479554128978e-02 + 3.012286404982057e-02 + 3.005110320361238e-02 + 2.997951260681279e-02 + 2.990809185255966e-02 + 2.983684053226568e-02 + 2.976575824919644e-02 + 2.969484460522792e-02 + 2.962409920087884e-02 + 2.955352163824170e-02 + 2.948311151647172e-02 + 2.941286843696999e-02 + 2.934279201383451e-02 + 2.927288185012002e-02 + 2.920313754405053e-02 + 2.913355871301240e-02 + 2.906414496442064e-02 + 2.899489589842786e-02 + 2.892581113440367e-02 + 2.885689028447655e-02 + 2.878813295173805e-02 + 2.871953875071393e-02 + 2.865110730129668e-02 + 2.858283822298987e-02 + 2.851473112218837e-02 + 2.844678561226344e-02 + 2.837900131834860e-02 + 2.831137785774055e-02 + 2.824391484910513e-02 + 2.817661191561155e-02 + 2.810946867322990e-02 + 2.804248474261467e-02 + 2.797565975421473e-02 + 2.790899332866750e-02 + 2.784248508684310e-02 + 2.777613465729838e-02 + 2.770994166752542e-02 + 2.764390574354870e-02 + 2.757802651096095e-02 + 2.751230360164656e-02 + 2.744673664902718e-02 + 2.738132528259552e-02 + 2.731606912935157e-02 + 2.725096782054994e-02 + 2.718602099881922e-02 + 2.712122829441404e-02 + 2.705658933533107e-02 + 2.699210376654040e-02 + 2.692777122629619e-02 + 2.686359134766074e-02 + 2.679956376855050e-02 + 2.673568813034148e-02 + 2.667196407449100e-02 + 2.660839123653040e-02 + 2.654496925932325e-02 + 2.648169779209303e-02 + 2.641857647953893e-02 + 2.635560496230991e-02 + 2.629278288047702e-02 + 2.623010988365326e-02 + 2.616758562366618e-02 + 2.610520975031933e-02 + 2.604298190560967e-02 + 2.598090173551063e-02 + 2.591896889422694e-02 + 2.585718303968204e-02 + 2.579554382343284e-02 + 2.573405088890056e-02 + 2.567270388987125e-02 + 2.561150248485746e-02 + 2.555044633271165e-02 + 2.548953508798725e-02 + 2.542876840653832e-02 + 2.536814594843316e-02 + 2.530766736730480e-02 + 2.524733231969847e-02 + 2.518714047250964e-02 + 2.512709148651639e-02 + 2.506718502244662e-02 + 2.500742074738894e-02 + 2.494779831751132e-02 + 2.488831739145854e-02 + 2.482897764751662e-02 + 2.476977875068464e-02 + 2.471072035836960e-02 + 2.465180213557270e-02 + 2.459302375909131e-02 + 2.453438490494015e-02 + 2.447588522684439e-02 + 2.441752439632618e-02 + 2.435930209686822e-02 + 2.430121799563831e-02 + 2.424327176191257e-02 + 2.418546306971884e-02 + 2.412779159320392e-02 + 2.407025701015483e-02 + 2.401285899981317e-02 + 2.395559723318545e-02 + 2.389847138352894e-02 + 2.384148113010493e-02 + 2.378462615809818e-02 + 2.372790614977233e-02 + 2.367132078182184e-02 + 2.361486973012364e-02 + 2.355855267628842e-02 + 2.350236930912776e-02 + 2.344631931433045e-02 + 2.339040237260871e-02 + 2.333461816188979e-02 + 2.327896637661412e-02 + 2.322344670789084e-02 + 2.316805883183489e-02 + 2.311280243515378e-02 + 2.305767721100430e-02 + 2.300268285332459e-02 + 2.294781904779433e-02 + 2.289308548135576e-02 + 2.283848185163481e-02 + 2.278400785322520e-02 + 2.272966317657758e-02 + 2.267544750906289e-02 + 2.262136054576819e-02 + 2.256740198705395e-02 + 2.251357153295969e-02 + 2.245986887460843e-02 + 2.240629370372433e-02 + 2.235284572997816e-02 + 2.229952464835474e-02 + 2.224633014749699e-02 + 2.219326193897085e-02 + 2.214031972536749e-02 + 2.208750320104361e-02 + 2.203481207216736e-02 + 2.198224604054712e-02 + 2.192980480490446e-02 + 2.187748807809197e-02 + 2.182529556587840e-02 + 2.177322696546572e-02 + 2.172128198614426e-02 + 2.166946034002218e-02 + 2.161776173674180e-02 + 2.156618587797776e-02 + 2.151473246970182e-02 + 2.146340122648612e-02 + 2.141219186176581e-02 + 2.136110408595206e-02 + 2.131013760699968e-02 + 2.125929213929720e-02 + 2.120856739943827e-02 + 2.115796310282077e-02 + 2.110747896148574e-02 + 2.105711468773744e-02 + 2.100686999708853e-02 + 2.095674460640470e-02 + 2.090673823470377e-02 + 2.085685060390323e-02 + 2.080708143284909e-02 + 2.075743043865024e-02 + 2.070789733896461e-02 + 2.065848185338519e-02 + 2.060918370358626e-02 + 2.056000261352416e-02 + 2.051093830731156e-02 + 2.046199050841276e-02 + 2.041315893918832e-02 + 2.036444332670061e-02 + 2.031584339811113e-02 + 2.026735887003067e-02 + 2.021898947028043e-02 + 2.017073493461869e-02 + 2.012259498752183e-02 + 2.007456935661649e-02 + 2.002665777297167e-02 + 1.997885996051357e-02 + 1.993117564943755e-02 + 1.988360457741823e-02 + 1.983614647798445e-02 + 1.978880107994216e-02 + 1.974156811085027e-02 + 1.969444731166336e-02 + 1.964743841723802e-02 + 1.960054115227784e-02 + 1.955375525837354e-02 + 1.950708047707909e-02 + 1.946051654129042e-02 + 1.941406318497287e-02 + 1.936772014474174e-02 + 1.932148716119990e-02 + 1.927536398140690e-02 + 1.922935034564512e-02 + 1.918344598004621e-02 + 1.913765063001423e-02 + 1.909196404425976e-02 + 1.904638595925336e-02 + 1.900091612254435e-02 + 1.895555428061772e-02 + 1.891030016469744e-02 + 1.886515352637163e-02 + 1.882011411999479e-02 + 1.877518167521970e-02 + 1.873035594232305e-02 + 1.868563668110157e-02 + 1.864102363246424e-02 + 1.859651653826993e-02 + 1.855211514697215e-02 + 1.850781921692858e-02 + 1.846362849811174e-02 + 1.841954273425713e-02 + 1.837556167722399e-02 + 1.833168508274460e-02 + 1.828791270638760e-02 + 1.824424429587613e-02 + 1.820067960186819e-02 + 1.815721838079814e-02 + 1.811386039242822e-02 + 1.807060539155448e-02 + 1.802745312809132e-02 + 1.798440336027523e-02 + 1.794145584667328e-02 + 1.789861034396235e-02 + 1.785586661445831e-02 + 1.781322441713358e-02 + 1.777068350457842e-02 + 1.772824363606736e-02 + 1.768590457480606e-02 + 1.764366608520215e-02 + 1.760152792777148e-02 + 1.755948986314423e-02 + 1.751755165530868e-02 + 1.747571306959304e-02 + 1.743397386795286e-02 + 1.739233380686453e-02 + 1.735079266012027e-02 + 1.730935019969591e-02 + 1.726800617515195e-02 + 1.722676035861175e-02 + 1.718561252878033e-02 + 1.714456244423200e-02 + 1.710360987245781e-02 + 1.706275458478830e-02 + 1.702199634263978e-02 + 1.698133492251816e-02 + 1.694077010510486e-02 + 1.690030164730331e-02 + 1.685992932068962e-02 + 1.681965290804600e-02 + 1.677947217681714e-02 + 1.673938689521877e-02 + 1.669939683666028e-02 + 1.665950177991986e-02 + 1.661970150205939e-02 + 1.657999577711447e-02 + 1.654038437906895e-02 + 1.650086708478969e-02 + 1.646144367321789e-02 + 1.642211391798679e-02 + 1.638287759543946e-02 + 1.634373448754158e-02 + 1.630468437732129e-02 + 1.626572704186934e-02 + 1.622686225251448e-02 + 1.618808979718675e-02 + 1.614940946170872e-02 + 1.611082101996680e-02 + 1.607232425440962e-02 + 1.603391895041710e-02 + 1.599560489122978e-02 + 1.595738185903658e-02 + 1.591924963852954e-02 + 1.588120801872957e-02 + 1.584325677844977e-02 + 1.580539569909259e-02 + 1.576762457813194e-02 + 1.572994319938502e-02 + 1.569235134492561e-02 + 1.565484881311209e-02 + 1.561743538476679e-02 + 1.558011083839993e-02 + 1.554287497958433e-02 + 1.550572759639263e-02 + 1.546866846769255e-02 + 1.543169739218311e-02 + 1.539481416304957e-02 + 1.535801856691179e-02 + 1.532131039389481e-02 + 1.528468944222785e-02 + 1.524815551153465e-02 + 1.521170838270258e-02 + 1.517534784896265e-02 + 1.513907371629201e-02 + 1.510288577422132e-02 + 1.506678381600347e-02 + 1.503076764230081e-02 + 1.499483704594779e-02 + 1.495899182406707e-02 + 1.492323178010325e-02 + 1.488755670734373e-02 + 1.485196640180961e-02 + 1.481646066682016e-02 + 1.478103929846549e-02 + 1.474570209596410e-02 + 1.471044886603309e-02 + 1.467527940456368e-02 + 1.464019351058384e-02 + 1.460519099481427e-02 + 1.457027165550810e-02 + 1.453543528903122e-02 + 1.450068170024789e-02 + 1.446601069487559e-02 + 1.443142207965322e-02 + 1.439691566271563e-02 + 1.436249124410753e-02 + 1.432814862414793e-02 + 1.429388761459182e-02 + 1.425970802314729e-02 + 1.422560965483165e-02 + 1.419159231701311e-02 + 1.415765581387266e-02 + 1.412379995194592e-02 + 1.409002455116132e-02 + 1.405632941792580e-02 + 1.402271435207008e-02 + 1.398917917303820e-02 + 1.395572369124169e-02 + 1.392234770932106e-02 + 1.388905104392205e-02 + 1.385583350997071e-02 + 1.382269491788422e-02 + 1.378963507987753e-02 + 1.375665380791922e-02 + 1.372375091366015e-02 + 1.369092621121959e-02 + 1.365817951715469e-02 + 1.362551064978645e-02 + 1.359291942581090e-02 + 1.356040565686591e-02 + 1.352796915179558e-02 + 1.349560973812908e-02 + 1.346332723540702e-02 + 1.343112144483463e-02 + 1.339899219360082e-02 + 1.336693930891053e-02 + 1.333496260043193e-02 + 1.330306188734133e-02 + 1.327123699224855e-02 + 1.323948773476910e-02 + 1.320781393642416e-02 + 1.317621541705235e-02 + 1.314469199166208e-02 + 1.311324348463446e-02 + 1.308186972415766e-02 + 1.305057053367403e-02 + 1.301934573289238e-02 + 1.298819514247371e-02 + 1.295711859027024e-02 + 1.292611590142988e-02 + 1.289518689817567e-02 + 1.286433140253433e-02 + 1.283354924175516e-02 + 1.280284024626999e-02 + 1.277220424320462e-02 + 1.274164105512193e-02 + 1.271115050454398e-02 + 1.268073242580564e-02 + 1.265038664885281e-02 + 1.262011299771964e-02 + 1.258991130045622e-02 + 1.255978138928737e-02 + 1.252972309744999e-02 + 1.249973624798444e-02 + 1.246982067106488e-02 + 1.243997620568067e-02 + 1.241020267461013e-02 + 1.238049990730756e-02 + 1.235086774662455e-02 + 1.232130602051964e-02 + 1.229181455798258e-02 + 1.226239319685087e-02 + 1.223304176770951e-02 + 1.220376010347082e-02 + 1.217454804486660e-02 + 1.214540542341299e-02 + 1.211633207079968e-02 + 1.208732782681391e-02 + 1.205839252771904e-02 + 1.202952600673308e-02 + 1.200072809652295e-02 + 1.197199864009422e-02 + 1.194333748171596e-02 + 1.191474445503102e-02 + 1.188621939116683e-02 + 1.185776212675846e-02 + 1.182937251288597e-02 + 1.180105038815069e-02 + 1.177279558400601e-02 + 1.174460794107161e-02 + 1.171648730396891e-02 + 1.168843351609301e-02 + 1.166044641171630e-02 + 1.163252583370262e-02 + 1.160467163102319e-02 + 1.157688364263099e-02 + 1.154916170933044e-02 + 1.152150567570712e-02 + 1.149391538555893e-02 + 1.146639068095789e-02 + 1.143893140447911e-02 + 1.141153740781534e-02 + 1.138420853673168e-02 + 1.135694462854312e-02 + 1.132974552839056e-02 + 1.130261108533809e-02 + 1.127554114921167e-02 + 1.124853556496432e-02 + 1.122159417997002e-02 + 1.119471684585664e-02 + 1.116790340283271e-02 + 1.114115369710312e-02 + 1.111446758944324e-02 + 1.108784492177717e-02 + 1.106128553754357e-02 + 1.103478929728668e-02 + 1.100835604711145e-02 + 1.098198563223047e-02 + 1.095567791133747e-02 + 1.092943273475230e-02 + 1.090324994987072e-02 + 1.087712940981543e-02 + 1.085107096608238e-02 + 1.082507447053145e-02 + 1.079913977931768e-02 + 1.077326674497864e-02 + 1.074745521859533e-02 + 1.072170505538783e-02 + 1.069601611066200e-02 + 1.067038823886727e-02 + 1.064482129354728e-02 + 1.061931513219808e-02 + 1.059386961330933e-02 + 1.056848458681972e-02 + 1.054315990640127e-02 + 1.051789543133304e-02 + 1.049269102209236e-02 + 1.046754653920045e-02 + 1.044246184044743e-02 + 1.041743677288729e-02 + 1.039247119540387e-02 + 1.036756497988909e-02 + 1.034271797805630e-02 + 1.031793004647734e-02 + 1.029320105343422e-02 + 1.026853085073464e-02 + 1.024391929539164e-02 + 1.021936625852857e-02 + 1.019487159885800e-02 + 1.017043517132599e-02 + 1.014605683522427e-02 + 1.012173646293816e-02 + 1.009747392058842e-02 + 1.007326905588179e-02 + 1.004912173629082e-02 + 1.002503183346208e-02 + 1.000099920653610e-02 + 9.977023717087764e-03 + 9.953105229386335e-03 + 9.929243608872406e-03 + 9.905438720973011e-03 + 9.881690430827046e-03 + 9.857998603258316e-03 + 9.834363104786413e-03 + 9.810783802355094e-03 + 9.787260560990621e-03 + 9.763793247942843e-03 + 9.740381731591760e-03 + 9.717025877967826e-03 + 9.693725554510663e-03 + 9.670480630152000e-03 + 9.647290973567844e-03 + 9.624156453317901e-03 + 9.601076938062841e-03 + 9.578052297039983e-03 + 9.555082399653962e-03 + 9.532167115690399e-03 + 9.509306316445141e-03 + 9.486499872512266e-03 + 9.463747653553711e-03 + 9.441049530896879e-03 + 9.418405376662433e-03 + 9.395815063104617e-03 + 9.373278461427175e-03 + 9.350795443499088e-03 + 9.328365882617906e-03 + 9.305989651970981e-03 + 9.283666624824176e-03 + 9.261396674723778e-03 + 9.239179675408710e-03 + 9.217015500757821e-03 + 9.194904024947497e-03 + 9.172845123632957e-03 + 9.150838672277813e-03 + 9.128884544858834e-03 + 9.106982617538152e-03 + 9.085132766987650e-03 + 9.063334868396023e-03 + 9.041588798714301e-03 + 9.019894435479832e-03 + 8.998251654894885e-03 + 8.976660334271958e-03 + 8.955120351755344e-03 + 8.933631585369030e-03 + 8.912193913264732e-03 + 8.890807213885918e-03 + 8.869471366214972e-03 + 8.848186249259440e-03 + 8.826951742251321e-03 + 8.805767725383001e-03 + 8.784634078744540e-03 + 8.763550682317645e-03 + 8.742517416743529e-03 + 8.721534162832835e-03 + 8.700600801502468e-03 + 8.679717214101214e-03 + 8.658883282492529e-03 + 8.638098888885700e-03 + 8.617363915025602e-03 + 8.596678243443276e-03 + 8.576041757693640e-03 + 8.555454340593486e-03 + 8.534915875110577e-03 + 8.514426244844865e-03 + 8.493985333674233e-03 + 8.473593025909724e-03 + 8.453249206335058e-03 + 8.432953759374154e-03 + 8.412706569850796e-03 + 8.392507523503252e-03 + 8.372356505624580e-03 + 8.352253401838657e-03 + 8.332198098800467e-03 + 8.312190482450988e-03 + 8.292230438871008e-03 + 8.272317855399050e-03 + 8.252452619663047e-03 + 8.232634619050869e-03 + 8.212863740331295e-03 + 8.193139871755427e-03 + 8.173462902270600e-03 + 8.153832719956143e-03 + 8.134249213438031e-03 + 8.114712271802299e-03 + 8.095221784035302e-03 + 8.075777639824947e-03 + 8.056379729417138e-03 + 8.037027942938956e-03 + 8.017722170154111e-03 + 7.998462301028743e-03 + 7.979248227381455e-03 + 7.960079840401237e-03 + 7.940957030626293e-03 + 7.921879690329333e-03 + 7.902847711453821e-03 + 7.883860985421759e-03 + 7.864919405368889e-03 + 7.846022864037674e-03 + 7.827171253341911e-03 + 7.808364466777525e-03 + 7.789602397948123e-03 + 7.770884940051978e-03 + 7.752211987488857e-03 + 7.733583434567862e-03 + 7.714999174963997e-03 + 7.696459103766686e-03 + 7.677963116338556e-03 + 7.659511107448910e-03 + 7.641102971817726e-03 + 7.622738605046243e-03 + 7.604417904226960e-03 + 7.586140765207791e-03 + 7.567907083543369e-03 + 7.549716755813157e-03 + 7.531569679511053e-03 + 7.513465752109027e-03 + 7.495404870064185e-03 + 7.477386931039745e-03 + 7.459411833357052e-03 + 7.441479474804043e-03 + 7.423589753808636e-03 + 7.405742569108545e-03 + 7.387937818890154e-03 + 7.370175402616828e-03 + 7.352455220325825e-03 + 7.334777170378787e-03 + 7.317141152312192e-03 + 7.299547066866589e-03 + 7.281994814230038e-03 + 7.264484294760120e-03 + 7.247015409129196e-03 + 7.229588058070396e-03 + 7.212202142721502e-03 + 7.194857564639825e-03 + 7.177554225359851e-03 + 7.160292026920549e-03 + 7.143070871828701e-03 + 7.125890661781023e-03 + 7.108751299151369e-03 + 7.091652687463649e-03 + 7.074594729473165e-03 + 7.057577328092346e-03 + 7.040600386920140e-03 + 7.023663809440293e-03 + 7.006767499621119e-03 + 6.989911362283671e-03 + 6.973095301761098e-03 + 6.956319222246642e-03 + 6.939583028242014e-03 + 6.922886624933184e-03 + 6.906229917977133e-03 + 6.889612813237578e-03 + 6.873035215956658e-03 + 6.856497031627250e-03 + 6.839998167124653e-03 + 6.823538528849983e-03 + 6.807118022950776e-03 + 6.790736556102037e-03 + 6.774394036012678e-03 + 6.758090370503938e-03 + 6.741825465874382e-03 + 6.725599229831710e-03 + 6.709411571076730e-03 + 6.693262397262002e-03 + 6.677151616823099e-03 + 6.661079138883049e-03 + 6.645044871881905e-03 + 6.629048724424302e-03 + 6.613090605727208e-03 + 6.597170426169362e-03 + 6.581288095176429e-03 + 6.565443521393368e-03 + 6.549636616015259e-03 + 6.533867289690503e-03 + 6.518135451870410e-03 + 6.502441013826720e-03 + 6.486783886693610e-03 + 6.471163980906190e-03 + 6.455581208648475e-03 + 6.440035481784445e-03 + 6.424526710952946e-03 + 6.409054808194274e-03 + 6.393619686148648e-03 + 6.378221257389647e-03 + 6.362859434216829e-03 + 6.347534129405275e-03 + 6.332245256702231e-03 + 6.316992728926398e-03 + 6.301776458848883e-03 + 6.286596360249587e-03 + 6.271452347303485e-03 + 6.256344334335297e-03 + 6.241272235637497e-03 + 6.226235965289892e-03 + 6.211235437617355e-03 + 6.196270567881593e-03 + 6.181341271239764e-03 + 6.166447462747857e-03 + 6.151589057705613e-03 + 6.136765971896623e-03 + 6.121978121401951e-03 + 6.107225422158738e-03 + 6.092507790132517e-03 + 6.077825141615326e-03 + 6.063177393866219e-03 + 6.048564463915550e-03 + 6.033986268442907e-03 + 6.019442724444331e-03 + 6.004933749322879e-03 + 5.990459260846786e-03 + 5.976019176895193e-03 + 5.961613415583682e-03 + 5.947241895231883e-03 + 5.932904533953020e-03 + 5.918601250164555e-03 + 5.904331962789744e-03 + 5.890096590861083e-03 + 5.875895053571317e-03 + 5.861727270328322e-03 + 5.847593160694864e-03 + 5.833492644473149e-03 + 5.819425641715960e-03 + 5.805392072163253e-03 + 5.791391855962741e-03 + 5.777424914193254e-03 + 5.763491167389796e-03 + 5.749590536018355e-03 + 5.735722941127108e-03 + 5.721888304505554e-03 + 5.708086547766434e-03 + 5.694317591457392e-03 + 5.680581357894303e-03 + 5.666877769868901e-03 + 5.653206748498332e-03 + 5.639568216215936e-03 + 5.625962096156960e-03 + 5.612388310444156e-03 + 5.598846782244085e-03 + 5.585337435180374e-03 + 5.571860191464977e-03 + 5.558414974887664e-03 + 5.545001710146245e-03 + 5.531620319651646e-03 + 5.518270727342278e-03 + 5.504952858629092e-03 + 5.491666637149790e-03 + 5.478411987013346e-03 + 5.465188833301630e-03 + 5.451997101098349e-03 + 5.438836715460566e-03 + 5.425707601454353e-03 + 5.412609684284744e-03 + 5.399542889597323e-03 + 5.386507143481766e-03 + 5.373502371672819e-03 + 5.360528500347250e-03 + 5.347585456320476e-03 + 5.334673165238440e-03 + 5.321791553290573e-03 + 5.308940548233661e-03 + 5.296120077396591e-03 + 5.283330067535706e-03 + 5.270570445063373e-03 + 5.257841138170363e-03 + 5.245142075198895e-03 + 5.232473183108559e-03 + 5.219834389656078e-03 + 5.207225623450342e-03 + 5.194646813662877e-03 + 5.182097888019903e-03 + 5.169578774433769e-03 + 5.157089403379224e-03 + 5.144629703265777e-03 + 5.132199601938705e-03 + 5.119799030212023e-03 + 5.107427917986021e-03 + 5.095086194230444e-03 + 5.082773788422456e-03 + 5.070490630851877e-03 + 5.058236652191947e-03 + 5.046011782411770e-03 + 5.033815952050060e-03 + 5.021649092096963e-03 + 5.009511132708167e-03 + 4.997402004872952e-03 + 4.985321640554899e-03 + 4.973269971256597e-03 + 4.961246927915243e-03 + 4.949252441362114e-03 + 4.937286444284467e-03 + 4.925348869112448e-03 + 4.913439647407981e-03 + 4.901558711796497e-03 + 4.889705994683389e-03 + 4.877881427891544e-03 + 4.866084944974813e-03 + 4.854316479151838e-03 + 4.842575962313763e-03 + 4.830863328342665e-03 + 4.819178511323663e-03 + 4.807521444209243e-03 + 4.795892060902922e-03 + 4.784290295384500e-03 + 4.772716080911616e-03 + 4.761169351999607e-03 + 4.749650043687660e-03 + 4.738158090505544e-03 + 4.726693426328077e-03 + 4.715255985490762e-03 + 4.703845704177584e-03 + 4.692462517255530e-03 + 4.681106359266913e-03 + 4.669777166671382e-03 + 4.658474874679097e-03 + 4.647199418051047e-03 + 4.635950733693824e-03 + 4.624728757814554e-03 + 4.613533426035275e-03 + 4.602364675085555e-03 + 4.591222441193854e-03 + 4.580106660340010e-03 + 4.569017270133631e-03 + 4.557954207729684e-03 + 4.546917409606468e-03 + 4.535906813063030e-03 + 4.524922355662980e-03 + 4.513963975024341e-03 + 4.503031608847450e-03 + 4.492125194604706e-03 + 4.481244669721014e-03 + 4.470389973075004e-03 + 4.459561043403103e-03 + 4.448757818627942e-03 + 4.437980237157132e-03 + 4.427228237584914e-03 + 4.416501758569245e-03 + 4.405800739475310e-03 + 4.395125119625431e-03 + 4.384474837838549e-03 + 4.373849833628650e-03 + 4.363250046735974e-03 + 4.352675416676511e-03 + 4.342125883231542e-03 + 4.331601386396421e-03 + 4.321101866314548e-03 + 4.310627263408189e-03 + 4.300177518182486e-03 + 4.289752570999768e-03 + 4.279352362569397e-03 + 4.268976833692708e-03 + 4.258625924837488e-03 + 4.248299577436886e-03 + 4.237997733292803e-03 + 4.227720333115902e-03 + 4.217467318344088e-03 + 4.207238631130676e-03 + 4.197034213451204e-03 + 4.186854006892898e-03 + 4.176697952964437e-03 + 4.166565994071401e-03 + 4.156458073034852e-03 + 4.146374132669977e-03 + 4.136314114934178e-03 + 4.126277962231265e-03 + 4.116265617785051e-03 + 4.106277025130100e-03 + 4.096312127225507e-03 + 4.086370866498099e-03 + 4.076453186851077e-03 + 4.066559032318193e-03 + 4.056688346384167e-03 + 4.046841072375552e-03 + 4.037017154234894e-03 + 4.027216536753351e-03 + 4.017439163691562e-03 + 4.007684978984143e-03 + 3.997953927616382e-03 + 3.988245954341580e-03 + 3.978561003853811e-03 + 3.968899020987378e-03 + 3.959259950174110e-03 + 3.949643736363211e-03 + 3.940050325917084e-03 + 3.930479663940112e-03 + 3.920931695287086e-03 + 3.911406366153988e-03 + 3.901903622192779e-03 + 3.892423408886956e-03 + 3.882965672529350e-03 + 3.873530359308857e-03 + 3.864117415340676e-03 + 3.854726787040829e-03 + 3.845358421069152e-03 + 3.836012264203419e-03 + 3.826688263119290e-03 + 3.817386364639579e-03 + 3.808106515834866e-03 + 3.798848664119608e-03 + 3.789612756624806e-03 + 3.780398740246358e-03 + 3.771206562411011e-03 + 3.762036171140922e-03 + 3.752887514765301e-03 + 3.743760540740341e-03 + 3.734655196719529e-03 + 3.725571430980986e-03 + 3.716509192135922e-03 + 3.707468428387338e-03 + 3.698449087559120e-03 + 3.689451119263604e-03 + 3.680474472533827e-03 + 3.671519094850921e-03 + 3.662584935894309e-03 + 3.653671945297336e-03 + 3.644780071212910e-03 + 3.635909263417586e-03 + 3.627059471971036e-03 + 3.618230645951014e-03 + 3.609422734605362e-03 + 3.600635687799489e-03 + 3.591869456282116e-03 + 3.583123989955615e-03 + 3.574399238584414e-03 + 3.565695152845392e-03 + 3.557011682930304e-03 + 3.548348778903569e-03 + 3.539706391532652e-03 + 3.531084471778191e-03 + 3.522482970666699e-03 + 3.513901839239252e-03 + 3.505341028388093e-03 + 3.496800489111114e-03 + 3.488280173119797e-03 + 3.479780031892506e-03 + 3.471300016604386e-03 + 3.462840078578430e-03 + 3.454400170006921e-03 + 3.445980243580072e-03 + 3.437580250715228e-03 + 3.429200142992534e-03 + 3.420839872683964e-03 + 3.412499392748279e-03 + 3.404178655562702e-03 + 3.395877612913789e-03 + 3.387596218304602e-03 + 3.379334424898988e-03 + 3.371092184764359e-03 + 3.362869450661228e-03 + 3.354666175991870e-03 + 3.346482314543412e-03 + 3.338317819175880e-03 + 3.330172642972068e-03 + 3.322046740007612e-03 + 3.313940064235304e-03 + 3.305852569386344e-03 + 3.297784208976364e-03 + 3.289734936574093e-03 + 3.281704706272533e-03 + 3.273693473070924e-03 + 3.265701191257636e-03 + 3.257727814803984e-03 + 3.249773298074863e-03 + 3.241837595979994e-03 + 3.233920663596386e-03 + 3.226022455622204e-03 + 3.218142926555998e-03 + 3.210282031163023e-03 + 3.202439725215123e-03 + 3.194615964134223e-03 + 3.186810703067027e-03 + 3.179023897602838e-03 + 3.171255503398923e-03 + 3.163505475998486e-03 + 3.155773770652277e-03 + 3.148060343553685e-03 + 3.140365151430291e-03 + 3.132688149313277e-03 + 3.125029293443853e-03 + 3.117388541383308e-03 + 3.109765848736372e-03 + 3.102161171408776e-03 + 3.094574466373093e-03 + 3.087005690728782e-03 + 3.079454801280003e-03 + 3.071921754555924e-03 + 3.064406507880629e-03 + 3.056909018530570e-03 + 3.049429243338136e-03 + 3.041967139485151e-03 + 3.034522664377966e-03 + 3.027095775562892e-03 + 3.019686430816400e-03 + 3.012294588055786e-03 + 3.004920205212209e-03 + 2.997563239705405e-03 + 2.990223649173174e-03 + 2.982901392198357e-03 + 2.975596427311395e-03 + 2.968308712822513e-03 + 2.961038206776656e-03 + 2.953784867392937e-03 + 2.946548653201694e-03 + 2.939329523194637e-03 + 2.932127436582907e-03 + 2.924942352269057e-03 + 2.917774228121468e-03 + 2.910623023643702e-03 + 2.903488699056754e-03 + 2.896371212514376e-03 + 2.889270523304112e-03 + 2.882186591679052e-03 + 2.875119376658924e-03 + 2.868068837676741e-03 + 2.861034934776638e-03 + 2.854017627809473e-03 + 2.847016876513213e-03 + 2.840032640686938e-03 + 2.833064880681456e-03 + 2.826113557103426e-03 + 2.819178630449384e-03 + 2.812260060105674e-03 + 2.805357806481411e-03 + 2.798471831428214e-03 + 2.791602094948384e-03 + 2.784748557109701e-03 + 2.777911179108391e-03 + 2.771089922400815e-03 + 2.764284748007964e-03 + 2.757495616304825e-03 + 2.750722488782177e-03 + 2.743965327115693e-03 + 2.737224092409578e-03 + 2.730498746008289e-03 + 2.723789249648372e-03 + 2.717095565426717e-03 + 2.710417654358784e-03 + 2.703755477800617e-03 + 2.697108999073383e-03 + 2.690478179939920e-03 + 2.683862981660020e-03 + 2.677263366970269e-03 + 2.670679298051182e-03 + 2.664110736894467e-03 + 2.657557646411518e-03 + 2.651019989061974e-03 + 2.644497727126834e-03 + 2.637990823748049e-03 + 2.631499241776077e-03 + 2.625022943767310e-03 + 2.618561892619121e-03 + 2.612116051581085e-03 + 2.605685383984310e-03 + 2.599269852507916e-03 + 2.592869420565223e-03 + 2.586484052278204e-03 + 2.580113710814501e-03 + 2.573758359383719e-03 + 2.567417961609419e-03 + 2.561092481362380e-03 + 2.554781882413309e-03 + 2.548486128464411e-03 + 2.542205184036424e-03 + 2.535939013422343e-03 + 2.529687580238901e-03 + 2.523450848436006e-03 + 2.517228782352248e-03 + 2.511021346720505e-03 + 2.504828506444104e-03 + 2.498650225958320e-03 + 2.492486468967693e-03 + 2.486337200680032e-03 + 2.480202386472821e-03 + 2.474081990653096e-03 + 2.467975978460781e-03 + 2.461884315215007e-03 + 2.455806965243642e-03 + 2.449743893777428e-03 + 2.443695066608286e-03 + 2.437660449328877e-03 + 2.431640007119714e-03 + 2.425633705211946e-03 + 2.419641509634817e-03 + 2.413663385966202e-03 + 2.407699299632079e-03 + 2.401749216909934e-03 + 2.395813103777745e-03 + 2.389890925926915e-03 + 2.383982649399740e-03 + 2.378088240561300e-03 + 2.372207665865560e-03 + 2.366340891258303e-03 + 2.360487883037754e-03 + 2.354648608018378e-03 + 2.348823033118129e-03 + 2.343011124539103e-03 + 2.337212848054228e-03 + 2.331428171390765e-03 + 2.325657062040943e-03 + 2.319899486412222e-03 + 2.314155410784699e-03 + 2.308424802407836e-03 + 2.302707629600105e-03 + 2.297003858807791e-03 + 2.291313456815245e-03 + 2.285636391934661e-03 + 2.279972631361444e-03 + 2.274322142294874e-03 + 2.268684892803898e-03 + 2.263060850445441e-03 + 2.257449982675230e-03 + 2.251852257331842e-03 + 2.246267642469826e-03 + 2.240696106224612e-03 + 2.235137616698782e-03 + 2.229592141925271e-03 + 2.224059650030012e-03 + 2.218540109447705e-03 + 2.213033488566449e-03 + 2.207539755750870e-03 + 2.202058879448589e-03 + 2.196590827868759e-03 + 2.191135569406817e-03 + 2.185693073538250e-03 + 2.180263309295804e-03 + 2.174846245146934e-03 + 2.169441849486111e-03 + 2.164050091629877e-03 + 2.158670941358884e-03 + 2.153304367034930e-03 + 2.147950337590323e-03 + 2.142608822828710e-03 + 2.137279792340324e-03 + 2.131963215445541e-03 + 2.126659061301170e-03 + 2.121367299363035e-03 + 2.116087899310018e-03 + 2.110820830985273e-03 + 2.105566064252215e-03 + 2.100323568822542e-03 + 2.095093314356113e-03 + 2.089875271479949e-03 + 2.084669410319118e-03 + 2.079475699908646e-03 + 2.074294110902819e-03 + 2.069124614088778e-03 + 2.063967179294363e-03 + 2.058821776747180e-03 + 2.053688376991423e-03 + 2.048566950718572e-03 + 2.043457468582260e-03 + 2.038359901240009e-03 + 2.033274219398007e-03 + 2.028200393563254e-03 + 2.023138394473247e-03 + 2.018088193625467e-03 + 2.013049761808302e-03 + 2.008023069533293e-03 + 2.003008088023720e-03 + 1.998004788922133e-03 + 1.993013143739461e-03 + 1.988033123025423e-03 + 1.983064698401174e-03 + 1.978107841939004e-03 + 1.973162523959716e-03 + 1.968228716337500e-03 + 1.963306391971906e-03 + 1.958395521239848e-03 + 1.953496075620802e-03 + 1.948608028019931e-03 + 1.943731350228688e-03 + 1.938866013789267e-03 + 1.934011990485791e-03 + 1.929169252901841e-03 + 1.924337773155020e-03 + 1.919517522737516e-03 + 1.914708474346116e-03 + 1.909910600724645e-03 + 1.905123874118434e-03 + 1.900348266781407e-03 + 1.895583751238830e-03 + 1.890830300301618e-03 + 1.886087886140893e-03 + 1.881356481302734e-03 + 1.876636059328014e-03 + 1.871926593048255e-03 + 1.867228054886164e-03 + 1.862540417322874e-03 + 1.857863653843670e-03 + 1.853197738035545e-03 + 1.848542642643349e-03 + 1.843898340659710e-03 + 1.839264805482605e-03 + 1.834642010884087e-03 + 1.830029929675155e-03 + 1.825428534756660e-03 + 1.820837800826725e-03 + 1.816257700994342e-03 + 1.811688207883416e-03 + 1.807129296515363e-03 + 1.802580940478343e-03 + 1.798043112569654e-03 + 1.793515787775185e-03 + 1.788998939681262e-03 + 1.784492541016137e-03 + 1.779996567290557e-03 + 1.775510992946437e-03 + 1.771035790955467e-03 + 1.766570935188881e-03 + 1.762116400368900e-03 + 1.757672161650980e-03 + 1.753238193179788e-03 + 1.748814468721301e-03 + 1.744400962234815e-03 + 1.739997649147064e-03 + 1.735604504535336e-03 + 1.731221502449809e-03 + 1.726848617398671e-03 + 1.722485824273258e-03 + 1.718133098140551e-03 + 1.713790413411143e-03 + 1.709457744960287e-03 + 1.705135068710917e-03 + 1.700822359316501e-03 + 1.696519591326764e-03 + 1.692226740206531e-03 + 1.687943780998710e-03 + 1.683670689031842e-03 + 1.679407440586236e-03 + 1.675154010255336e-03 + 1.670910372590760e-03 + 1.666676504276520e-03 + 1.662452380680008e-03 + 1.658237976786625e-03 + 1.654033269138933e-03 + 1.649838233189823e-03 + 1.645652843894769e-03 + 1.641477077483747e-03 + 1.637310910181093e-03 + 1.633154317979306e-03 + 1.629007276710043e-03 + 1.624869761977525e-03 + 1.620741749539434e-03 + 1.616623216289868e-03 + 1.612514138626602e-03 + 1.608414492386588e-03 + 1.604324254066237e-03 + 1.600243399893093e-03 + 1.596171905775712e-03 + 1.592109748332882e-03 + 1.588056904380057e-03 + 1.584013350591343e-03 + 1.579979063015354e-03 + 1.575954018417919e-03 + 1.571938194458784e-03 + 1.567931567017785e-03 + 1.563934112240304e-03 + 1.559945807641685e-03 + 1.555966630779868e-03 + 1.551996558434039e-03 + 1.548035566441649e-03 + 1.544083632647983e-03 + 1.540140734902817e-03 + 1.536206849423688e-03 + 1.532281953055403e-03 + 1.528366023373284e-03 + 1.524459038476067e-03 + 1.520560975283225e-03 + 1.516671810636623e-03 + 1.512791522697568e-03 + 1.508920088807869e-03 + 1.505057486015793e-03 + 1.501203692209861e-03 + 1.497358685322736e-03 + 1.493522443062933e-03 + 1.489694942724099e-03 + 1.485876162194853e-03 + 1.482066079697174e-03 + 1.478264672890618e-03 + 1.474471919644568e-03 + 1.470687798121700e-03 + 1.466912286519041e-03 + 1.463145362594138e-03 + 1.459387003990816e-03 + 1.455637189563730e-03 + 1.451895897787254e-03 + 1.448163106500405e-03 + 1.444438793947980e-03 + 1.440722938760434e-03 + 1.437015519703072e-03 + 1.433316514691718e-03 + 1.429625901910583e-03 + 1.425943660193772e-03 + 1.422269768088744e-03 + 1.418604204318803e-03 + 1.414946947952337e-03 + 1.411297977567125e-03 + 1.407657271824714e-03 + 1.404024809794961e-03 + 1.400400570139807e-03 + 1.396784531434119e-03 + 1.393176672533590e-03 + 1.389576972970703e-03 + 1.385985412235932e-03 + 1.382401969107074e-03 + 1.378826622443783e-03 + 1.375259351219489e-03 + 1.371700134514839e-03 + 1.368148952079336e-03 + 1.364605783757036e-03 + 1.361070608632290e-03 + 1.357543405835987e-03 + 1.354024154621594e-03 + 1.350512834317403e-03 + 1.347009424934256e-03 + 1.343513906652181e-03 + 1.340026258589547e-03 + 1.336546459739890e-03 + 1.333074489680093e-03 + 1.329610329681042e-03 + 1.326153959375566e-03 + 1.322705357081586e-03 + 1.319264503327096e-03 + 1.315831378649060e-03 + 1.312405962942001e-03 + 1.308988235813027e-03 + 1.305578177302038e-03 + 1.302175767936687e-03 + 1.298780987783750e-03 + 1.295393816813521e-03 + 1.292014235092867e-03 + 1.288642222828844e-03 + 1.285277760581296e-03 + 1.281920829182064e-03 + 1.278571408463430e-03 + 1.275229478684235e-03 + 1.271895021211065e-03 + 1.268568016022571e-03 + 1.265248443161927e-03 + 1.261936283866222e-03 + 1.258631518851812e-03 + 1.255334128626978e-03 + 1.252044093890807e-03 + 1.248761395151665e-03 + 1.245486013113075e-03 + 1.242217929129610e-03 + 1.238957124307338e-03 + 1.235703579418827e-03 + 1.232457274959363e-03 + 1.229218191838193e-03 + 1.225986311430220e-03 + 1.222761615473843e-03 + 1.219544084856006e-03 + 1.216333699988417e-03 + 1.213130441964228e-03 + 1.209934292565758e-03 + 1.206745233676102e-03 + 1.203563245963205e-03 + 1.200388310525061e-03 + 1.197220409053557e-03 + 1.194059523052293e-03 + 1.190905633902454e-03 + 1.187758723052301e-03 + 1.184618772531461e-03 + 1.181485763720917e-03 + 1.178359677471923e-03 + 1.175240496495629e-03 + 1.172128202790389e-03 + 1.169022777016151e-03 + 1.165924201427151e-03 + 1.162832457974330e-03 + 1.159747527667428e-03 + 1.156669393342115e-03 + 1.153598037513975e-03 + 1.150533441195656e-03 + 1.147475586343000e-03 + 1.144424455192698e-03 + 1.141380029693999e-03 + 1.138342292070108e-03 + 1.135311224784675e-03 + 1.132286810371483e-03 + 1.129269030382949e-03 + 1.126257866687504e-03 + 1.123253302806332e-03 + 1.120255320585528e-03 + 1.117263901594027e-03 + 1.114279029333705e-03 + 1.111300685926414e-03 + 1.108328853214485e-03 + 1.105363515126287e-03 + 1.102404653695880e-03 + 1.099452250276090e-03 + 1.096506289065756e-03 + 1.093566752790314e-03 + 1.090633622975802e-03 + 1.087706882683815e-03 + 1.084786514992603e-03 + 1.081872502714538e-03 + 1.078964828755093e-03 + 1.076063476054719e-03 + 1.073168427526746e-03 + 1.070279665938792e-03 + 1.067397174442816e-03 + 1.064520936417689e-03 + 1.061650934087830e-03 + 1.058787150577005e-03 + 1.055929570192933e-03 + 1.053078175257412e-03 + 1.050232948596978e-03 + 1.047393874491941e-03 + 1.044560935952646e-03 + 1.041734115621369e-03 + 1.038913396581877e-03 + 1.036098763274602e-03 + 1.033290199394467e-03 + 1.030487686748966e-03 + 1.027691209748195e-03 + 1.024900752824401e-03 + 1.022116298137994e-03 + 1.019337829638554e-03 + 1.016565331581354e-03 + 1.013798786681050e-03 + 1.011038178744547e-03 + 1.008283492069452e-03 + 1.005534710312413e-03 + 1.002791817167788e-03 + 1.000054796401326e-03 + 9.973236317525795e-04 + 9.945983071579347e-04 + 9.918788066535795e-04 + 9.891651141399331e-04 + 9.864572135795289e-04 + 9.837550890187182e-04 + 9.810587245440681e-04 + 9.783681042165289e-04 + 9.756832121036981e-04 + 9.730040324093254e-04 + 9.703305493295432e-04 + 9.676627470403841e-04 + 9.650006098016114e-04 + 9.623441219373007e-04 + 9.596932677964791e-04 + 9.570480316324495e-04 + 9.544083978205768e-04 + 9.517743508934125e-04 + 9.491458752396962e-04 + 9.465229553012878e-04 + 9.439055756501852e-04 + 9.412937207895038e-04 + 9.386873752809525e-04 + 9.360865238076440e-04 + 9.334911509791107e-04 + 9.309012414140627e-04 + 9.283167798171848e-04 + 9.257377509839119e-04 + 9.231641396993918e-04 + 9.205959306774516e-04 + 9.180331088080401e-04 + 9.154756590138127e-04 + 9.129235660962607e-04 + 9.103768149701878e-04 + 9.078353906321772e-04 + 9.052992780955343e-04 + 9.027684623946166e-04 + 9.002429285800444e-04 + 8.977226617159664e-04 + 8.952076469091944e-04 + 8.926978693026045e-04 + 8.901933140640126e-04 + 8.876939664445818e-04 + 8.851998117353022e-04 + 8.827108351548252e-04 + 8.802270219825557e-04 + 8.777483575705740e-04 + 8.752748272927118e-04 + 8.728064165680979e-04 + 8.703431108522175e-04 + 8.678848955854838e-04 + 8.654317562117358e-04 + 8.629836782258149e-04 + 8.605406473168484e-04 + 8.581026490766350e-04 + 8.556696689660546e-04 + 8.532416927295966e-04 + 8.508187061075220e-04 + 8.484006947265324e-04 + 8.459876443379812e-04 + 8.435795407180884e-04 + 8.411763696276702e-04 + 8.387781169670006e-04 + 8.363847686436173e-04 + 8.339963104952486e-04 + 8.316127284703911e-04 + 8.292340085393665e-04 + 8.268601366269382e-04 + 8.244910987818749e-04 + 8.221268810991789e-04 + 8.197674696351961e-04 + 8.174128504795555e-04 + 8.150630097863373e-04 + 8.127179337987174e-04 + 8.103776086785577e-04 + 8.080420206116735e-04 + 8.057111559859151e-04 + 8.033850010442538e-04 + 8.010635420033148e-04 + 7.987467653572807e-04 + 7.964346575211490e-04 + 7.941272048473775e-04 + 7.918243938145987e-04 + 7.895262109139402e-04 + 7.872326426395918e-04 + 7.849436755404454e-04 + 7.826592961986925e-04 + 7.803794912312032e-04 + 7.781042473133076e-04 + 7.758335511109105e-04 + 7.735673892815593e-04 + 7.713057485643111e-04 + 7.690486157536814e-04 + 7.667959776727836e-04 + 7.645478211004869e-04 + 7.623041328912646e-04 + 7.600649000053854e-04 + 7.578301093264215e-04 + 7.555997477583347e-04 + 7.533738022910566e-04 + 7.511522599897612e-04 + 7.489351079282785e-04 + 7.467223331578882e-04 + 7.445139227962291e-04 + 7.423098639775247e-04 + 7.401101438263767e-04 + 7.379147496119391e-04 + 7.357236686066337e-04 + 7.335368879604144e-04 + 7.313543950177981e-04 + 7.291761771981462e-04 + 7.270022218267318e-04 + 7.248325162821272e-04 + 7.226670479949025e-04 + 7.205058044241288e-04 + 7.183487730730944e-04 + 7.161959414766152e-04 + 7.140472971809699e-04 + 7.119028277561113e-04 + 7.097625208231147e-04 + 7.076263641063239e-04 + 7.054943452576004e-04 + 7.033664519218169e-04 + 7.012426719956601e-04 + 6.991229932510109e-04 + 6.970074033540157e-04 + 6.948958902400331e-04 + 6.927884418689321e-04 + 6.906850461485944e-04 + 6.885856909689247e-04 + 6.864903643067344e-04 + 6.843990542306164e-04 + 6.823117487354689e-04 + 6.802284359113009e-04 + 6.781491039842488e-04 + 6.760737410664455e-04 + 6.740023352773578e-04 + 6.719348748282754e-04 + 6.698713480434448e-04 + 6.678117432465053e-04 + 6.657560487034911e-04 + 6.637042527655789e-04 + 6.616563438432147e-04 + 6.596123103782944e-04 + 6.575721408025738e-04 + 6.555358236048545e-04 + 6.535033473936704e-04 + 6.514747007198682e-04 + 6.494498721244171e-04 + 6.474288502266283e-04 + 6.454116237722280e-04 + 6.433981815196293e-04 + 6.413885120963034e-04 + 6.393826043225120e-04 + 6.373804471016686e-04 + 6.353820291728463e-04 + 6.333873394523912e-04 + 6.313963669651093e-04 + 6.294091005879305e-04 + 6.274255292885601e-04 + 6.254456421348147e-04 + 6.234694281911996e-04 + 6.214968766042923e-04 + 6.195279765738893e-04 + 6.175627172048796e-04 + 6.156010877068991e-04 + 6.136430774102346e-04 + 6.116886755862946e-04 + 6.097378715435904e-04 + 6.077906546697107e-04 + 6.058470144286753e-04 + 6.039069402835228e-04 + 6.019704216802507e-04 + 6.000374481409778e-04 + 5.981080092530985e-04 + 5.961820946588096e-04 + 5.942596940083503e-04 + 5.923407969721301e-04 + 5.904253932616448e-04 + 5.885134726736782e-04 + 5.866050250202793e-04 + 5.847000400980857e-04 + 5.827985078791194e-04 + 5.809004183470366e-04 + 5.790057613579966e-04 + 5.771145269606196e-04 + 5.752267052829598e-04 + 5.733422863841403e-04 + 5.714612604000550e-04 + 5.695836175305099e-04 + 5.677093480037018e-04 + 5.658384420678564e-04 + 5.639708900266142e-04 + 5.621066823022896e-04 + 5.602458092869138e-04 + 5.583882613744714e-04 + 5.565340290827777e-04 + 5.546831029664671e-04 + 5.528354736013446e-04 + 5.509911316015094e-04 + 5.491500676402182e-04 + 5.473122724461832e-04 + 5.454777367810608e-04 + 5.436464514412940e-04 + 5.418184072613321e-04 + 5.399935951226492e-04 + 5.381720060069473e-04 + 5.363536309736959e-04 + 5.345384609776805e-04 + 5.327264870731968e-04 + 5.309177004752346e-04 + 5.291120923593077e-04 + 5.273096539494221e-04 + 5.255103765544823e-04 + 5.237142514407170e-04 + 5.219212699733492e-04 + 5.201314236865307e-04 + 5.183447040146093e-04 + 5.165611024246747e-04 + 5.147806105331874e-04 + 5.130032200114061e-04 + 5.112289225530487e-04 + 5.094577098620987e-04 + 5.076895736854020e-04 + 5.059245058583464e-04 + 5.041624983517820e-04 + 5.024035430839573e-04 + 5.006476319990773e-04 + 4.988947572035529e-04 + 4.971449107942239e-04 + 4.953980848818115e-04 + 4.936542716805855e-04 + 4.919134634956966e-04 + 4.901756526846954e-04 + 4.884408316006080e-04 + 4.867089926403434e-04 + 4.849801282731475e-04 + 4.832542310892493e-04 + 4.815312937309208e-04 + 4.798113088560132e-04 + 4.780942690954599e-04 + 4.763801672259719e-04 + 4.746689961731605e-04 + 4.729607488395965e-04 + 4.712554181333483e-04 + 4.695529970142063e-04 + 4.678534786182833e-04 + 4.661568561056742e-04 + 4.644631226216267e-04 + 4.627722714285577e-04 + 4.610842958399921e-04 + 4.593991892076899e-04 + 4.577169450300335e-04 + 4.560375568336446e-04 + 4.543610181249096e-04 + 4.526873225594637e-04 + 4.510164638467393e-04 + 4.493484356950898e-04 + 4.476832319569833e-04 + 4.460208465382815e-04 + 4.443612733350173e-04 + 4.427045064285867e-04 + 4.410505399510805e-04 + 4.393993679504339e-04 + 4.377509846774214e-04 + 4.361053844834223e-04 + 4.344625616662297e-04 + 4.328225106365193e-04 + 4.311852259097922e-04 + 4.295507020701359e-04 + 4.279189337521271e-04 + 4.262899156467061e-04 + 4.246636425301464e-04 + 4.230401092565931e-04 + 4.214193107478168e-04 + 4.198012419839599e-04 + 4.181858979868060e-04 + 4.165732738692198e-04 + 4.149633649793705e-04 + 4.133561665739344e-04 + 4.117516738453971e-04 + 4.101498823326724e-04 + 4.085507876181731e-04 + 4.069543852399522e-04 + 4.053606707914893e-04 + 4.037696400086752e-04 + 4.021812887746215e-04 + 4.005956129436248e-04 + 3.990126084760660e-04 + 3.974322714846641e-04 + 3.958545980293214e-04 + 3.942795842513983e-04 + 3.927072264688036e-04 + 3.911375211195356e-04 + 3.895704646516968e-04 + 3.880060534869329e-04 + 3.864442842922655e-04 + 3.848851538066537e-04 + 3.833286587063809e-04 + 3.817747958651508e-04 + 3.802235622593628e-04 + 3.786749548688228e-04 + 3.771289708205180e-04 + 3.755856073392733e-04 + 3.740448616816441e-04 + 3.725067311649692e-04 + 3.709712132043737e-04 + 3.694383053790672e-04 + 3.679080053579325e-04 + 3.663803108503057e-04 + 3.648552195606523e-04 + 3.633327293646475e-04 + 3.618128382932607e-04 + 3.602955444344858e-04 + 3.587808459078391e-04 + 3.572687408998931e-04 + 3.557592277792442e-04 + 3.542523049927096e-04 + 3.527479710408145e-04 + 3.512462245019904e-04 + 3.497470640500036e-04 + 3.482504884770882e-04 + 3.467564967437580e-04 + 3.452650877985240e-04 + 3.437762605764859e-04 + 3.422900143250168e-04 + 3.408063483640538e-04 + 3.393252619896345e-04 + 3.378467546103840e-04 + 3.363708257653936e-04 + 3.348974751341996e-04 + 3.334267024660963e-04 + 3.319585075646510e-04 + 3.304928903038992e-04 + 3.290298507212495e-04 + 3.275693889604049e-04 + 3.261115052402690e-04 + 3.246561999078441e-04 + 3.232034733603981e-04 + 3.217533259984899e-04 + 3.203057585048498e-04 + 3.188607716794152e-04 + 3.174183662346357e-04 + 3.159785430822741e-04 + 3.145413032833238e-04 + 3.131066479369465e-04 + 3.116745782136557e-04 + 3.102450954058189e-04 + 3.088182010197684e-04 + 3.073938965650610e-04 + 3.059721835802734e-04 + 3.045530637916020e-04 + 3.031365390386899e-04 + 3.017226112552691e-04 + 3.003112824793533e-04 + 2.989025548234045e-04 + 2.974964304955006e-04 + 2.960929118913988e-04 + 2.946920014422941e-04 + 2.932937016150965e-04 + 2.918980151023185e-04 + 2.905049446767149e-04 + 2.891144931550728e-04 + 2.877266634923031e-04 + 2.863414587568181e-04 + 2.849588821248430e-04 + 2.835789368717610e-04 + 2.822016263526234e-04 + 2.808269540128114e-04 + 2.794549234525530e-04 + 2.780855383858511e-04 + 2.767188026160350e-04 + 2.753547199733251e-04 + 2.739932944362576e-04 + 2.726345302253411e-04 + 2.712784314751994e-04 + 2.699250024118205e-04 + 2.685742475595017e-04 + 2.672261714415124e-04 + 2.658807786371782e-04 + 2.645380739000219e-04 + 2.631980619957597e-04 + 2.618607478073471e-04 + 2.605261365239288e-04 + 2.591942332540738e-04 + 2.578650431168888e-04 + 2.565385715181067e-04 + 2.552148239188000e-04 + 2.538938058269625e-04 + 2.525755228940477e-04 + 2.512599808919393e-04 + 2.499471856899045e-04 + 2.486371432244064e-04 + 2.473298595107682e-04 + 2.460253406646765e-04 + 2.447235929605613e-04 + 2.434246227773826e-04 + 2.421284365723855e-04 + 2.408350408491505e-04 + 2.395444422583646e-04 + 2.382566475978551e-04 + 2.369716636126905e-04 + 2.356894972066169e-04 + 2.344101555067014e-04 + 2.331336455922197e-04 + 2.318599746393059e-04 + 2.305891500038487e-04 + 2.293211790903458e-04 + 2.280560693749351e-04 + 2.267938284406659e-04 + 2.255344639796373e-04 + 2.242779837481179e-04 + 2.230243955584571e-04 + 2.217737074088738e-04 + 2.205259273750273e-04 + 2.192810635194828e-04 + 2.180391240377155e-04 + 2.168001172473318e-04 + 2.155640515683406e-04 + 2.143309354202995e-04 + 2.131007772856537e-04 + 2.118735858306198e-04 + 2.106493698084787e-04 + 2.094281380089542e-04 + 2.082098992208740e-04 + 2.069946624015073e-04 + 2.057824366286796e-04 + 2.045732309577213e-04 + 2.033670545521807e-04 + 2.021639166762670e-04 + 2.009638266139778e-04 + 1.997667937615743e-04 + 1.985728276099416e-04 + 1.973819376249420e-04 + 1.961941334186818e-04 + 1.950094247288422e-04 + 1.938278211445808e-04 + 1.926493324216112e-04 + 1.914739685443153e-04 + 1.903017394244089e-04 + 1.891326549660665e-04 + 1.879667251315557e-04 + 1.868039600691931e-04 + 1.856443699312781e-04 + 1.844879648170397e-04 + 1.833347550402995e-04 + 1.821847509305592e-04 + 1.810379627269367e-04 + 1.798944008330572e-04 + 1.787540757117569e-04 + 1.776169978023503e-04 + 1.764831776083264e-04 + 1.753526257075039e-04 + 1.742253527510645e-04 + 1.731013693254543e-04 + 1.719806860311682e-04 + 1.708633135911149e-04 + 1.697492627671219e-04 + 1.686385443331759e-04 + 1.675311690584120e-04 + 1.664271477052953e-04 + 1.653264910845736e-04 + 1.642292101438652e-04 + 1.631353157524277e-04 + 1.620448187436031e-04 + 1.609577300754615e-04 + 1.598740606888992e-04 + 1.587938214812250e-04 + 1.577170233258566e-04 + 1.566436772168694e-04 + 1.555737942159133e-04 + 1.545073851968179e-04 + 1.534444610998845e-04 + 1.523850329588589e-04 + 1.513291117056277e-04 + 1.502767082718022e-04 + 1.492278336091406e-04 + 1.481824986061289e-04 + 1.471407142013266e-04 + 1.461024913960903e-04 + 1.450678410475480e-04 + 1.440367739871709e-04 + 1.430093010797408e-04 + 1.419854331666308e-04 + 1.409651810501559e-04 + 1.399485554884026e-04 + 1.389355672434459e-04 + 1.379262270311977e-04 + 1.369205454852166e-04 + 1.359185332657421e-04 + 1.349202010239166e-04 + 1.339255593445051e-04 + 1.329346186642912e-04 + 1.319473893926901e-04 + 1.309638820305585e-04 + 1.299841069283569e-04 + 1.290080743627276e-04 + 1.280357946577360e-04 + 1.270672779908064e-04 + 1.261025344295187e-04 + 1.251415740258344e-04 + 1.241844068414831e-04 + 1.232310428585276e-04 + 1.222814917974582e-04 + 1.213357634573326e-04 + 1.203938676644921e-04 + 1.194558139769228e-04 + 1.185216119071049e-04 + 1.175912709437735e-04 + 1.166648004395201e-04 + 1.157422096820540e-04 + 1.148235078941047e-04 + 1.139087041565015e-04 + 1.129978074560730e-04 + 1.120908266963163e-04 + 1.111877706801771e-04 + 1.102886481165339e-04 + 1.093934676008822e-04 + 1.085022375478793e-04 + 1.076149663238430e-04 + 1.067316622735248e-04 + 1.058523335072715e-04 + 1.049769880021785e-04 + 1.041056336506420e-04 + 1.032382782134568e-04 + 1.023749293560659e-04 + 1.015155946454073e-04 + 1.006602814254268e-04 + 9.980899691604087e-05 + 9.896174827822380e-05 + 9.811854249648769e-05 + 9.727938640362722e-05 + 9.644428669167931e-05 + 9.561324990949941e-05 + 9.478628247235907e-05 + 9.396339064407664e-05 + 9.314458052650697e-05 + 9.232985807344924e-05 + 9.151922908266137e-05 + 9.071269919463392e-05 + 8.991027389908417e-05 + 8.911195851702552e-05 + 8.831775819982511e-05 + 8.752767793866873e-05 + 8.674172256023528e-05 + 8.595989672073575e-05 + 8.518220490590348e-05 + 8.440865141920385e-05 + 8.363924039733809e-05 + 8.287397580722622e-05 + 8.211286142104977e-05 + 8.135590084051729e-05 + 8.060309749193613e-05 + 7.985445459658365e-05 + 7.910997520302950e-05 + 7.836966218298747e-05 + 7.763351820320998e-05 + 7.690154574559998e-05 + 7.617374709914362e-05 + 7.545012436145964e-05 + 7.473067944430294e-05 + 7.401541405207780e-05 + 7.330432968888297e-05 + 7.259742767376570e-05 + 7.189470911942744e-05 + 7.119617492698978e-05 + 7.050182580949816e-05 + 6.981166227552030e-05 + 6.912568461899398e-05 + 6.844389294104750e-05 + 6.776628713300132e-05 + 6.709286687130013e-05 + 6.642363163388969e-05 + 6.575858067871575e-05 + 6.509771305493562e-05 + 6.444102761503343e-05 + 6.378852298738557e-05 + 6.314019758583242e-05 + 6.249604962295056e-05 + 6.185607708979005e-05 + 6.122027775903875e-05 + 6.058864919749181e-05 + 5.996118875684596e-05 + 5.933789356597989e-05 + 5.871876053888907e-05 + 5.810378637807827e-05 + 5.749296757228591e-05 + 5.688630039241354e-05 + 5.628378087841946e-05 + 5.568540486517338e-05 + 5.509116797959273e-05 + 5.450106561933422e-05 + 5.391509296529475e-05 + 5.333324497665832e-05 + 5.275551640753726e-05 + 5.218190179953343e-05 + 5.161239545977182e-05 + 5.104699148146081e-05 + 5.048568375913861e-05 + 4.992846596763329e-05 + 4.937533154817934e-05 + 4.882627374147808e-05 + 4.828128557792042e-05 + 4.774035986717692e-05 + 4.720348921279551e-05 + 4.667066599018748e-05 + 4.614188237688400e-05 + 4.561713035194944e-05 + 4.509640165438987e-05 + 4.457968783032909e-05 + 4.406698023190481e-05 + 4.355826996891768e-05 + 4.305354796420506e-05 + 4.255280495560636e-05 + 4.205603144616347e-05 + 4.156321774032957e-05 + 4.107435395613752e-05 + 4.058942999665175e-05 + 4.010843556869507e-05 + 3.963136019085871e-05 + 3.915819316952545e-05 + 3.868892361719641e-05 + 3.822354046226270e-05 + 3.776203244031561e-05 + 3.730438809005640e-05 + 3.685059575689182e-05 + 3.640064360230198e-05 + 3.595451960373741e-05 + 3.551221155134442e-05 + 3.507370705450421e-05 + 3.463899354241650e-05 + 3.420805825588243e-05 + 3.378088825986720e-05 + 3.335747045272196e-05 + 3.293779155463552e-05 + 3.252183811026268e-05 + 3.210959649877579e-05 + 3.170105292073684e-05 + 3.129619341367949e-05 + 3.089500386055728e-05 + 3.049746997239612e-05 + 3.010357730585059e-05 + 2.971331126487755e-05 + 2.932665707619367e-05 + 2.894359982344077e-05 + 2.856412445132613e-05 + 2.818821573877814e-05 + 2.781585831987375e-05 + 2.744703669454755e-05 + 2.708173520948728e-05 + 2.671993807124836e-05 + 2.636162935767626e-05 + 2.600679300301263e-05 + 2.565541280257304e-05 + 2.530747242805460e-05 + 2.496295542635910e-05 + 2.462184521206604e-05 + 2.428412507785163e-05 + 2.394977819771233e-05 + 2.361878762181457e-05 + 2.329113628448998e-05 + 2.296680701293541e-05 + 2.264578251757474e-05 + 2.232804539557172e-05 + 2.201357814530585e-05 + 2.170236315904943e-05 + 2.139438272306276e-05 + 2.108961903561866e-05 + 2.078805419745760e-05 + 2.048967020582218e-05 + 2.019444897050880e-05 + 1.990237231614296e-05 + 1.961342197777572e-05 + 1.932757961462368e-05 + 1.904482680353188e-05 + 1.876514502948172e-05 + 1.848851571515060e-05 + 1.821492021640795e-05 + 1.794433980831370e-05 + 1.767675569461099e-05 + 1.741214902258527e-05 + 1.715050088166273e-05 + 1.689179228451716e-05 + 1.663600419203502e-05 + 1.638311752659296e-05 + 1.613311314360082e-05 + 1.588597184706167e-05 + 1.564167440067926e-05 + 1.540020151829502e-05 + 1.516153387453479e-05 + 1.492565211244357e-05 + 1.469253682958783e-05 + 1.446216858599219e-05 + 1.423452791421707e-05 + 1.400959531801727e-05 + 1.378735127470914e-05 + 1.356777623611474e-05 + 1.335085063337062e-05 + 1.313655487876277e-05 + 1.292486936494610e-05 + 1.271577446639333e-05 + 1.250925054517824e-05 + 1.230527796605363e-05 + 1.210383708165599e-05 + 1.190490823144681e-05 + 1.170847175649016e-05 + 1.151450799717159e-05 + 1.132299729345123e-05 + 1.113391999104486e-05 + 1.094725644586371e-05 + 1.076298702423590e-05 + 1.058109209097755e-05 + 1.040155202830912e-05 + 1.022434724511555e-05 + 1.004945815519835e-05 + 9.876865191434875e-06 + 9.706548817576976e-06 + 9.538489515829050e-06 + 9.372667791395497e-06 + 9.209064181113714e-06 + 9.047659251528417e-06 + 8.888433598428113e-06 + 8.731367852817854e-06 + 8.576442682615395e-06 + 8.423638791088146e-06 + 8.272936922653283e-06 + 8.124317863654203e-06 + 7.977762440936675e-06 + 7.833251528895281e-06 + 7.690766049713759e-06 + 7.550286971453687e-06 + 7.411795314787993e-06 + 7.275272152381047e-06 + 7.140698608042407e-06 + 7.008055865069554e-06 + 6.877325162999093e-06 + 6.748487797069182e-06 + 6.621525126924672e-06 + 6.496418573295794e-06 + 6.373149617974847e-06 + 6.251699810845691e-06 + 6.132050767139283e-06 + 6.014184168887862e-06 + 5.898081770597224e-06 + 5.783725394973654e-06 + 5.671096936050662e-06 + 5.560178365641361e-06 + 5.450951726966341e-06 + 5.343399138917566e-06 + 5.237502802602254e-06 + 5.133244993336976e-06 + 5.030608065950253e-06 + 4.929574461382145e-06 + 4.830126698086031e-06 + 4.732247378331277e-06 + 4.635919192753656e-06 + 4.541124912789582e-06 + 4.447847398958468e-06 + 4.356069602328335e-06 + 4.265774557056776e-06 + 4.176945389932598e-06 + 4.089565320281935e-06 + 4.003617653697645e-06 + 3.919085793399028e-06 + 3.835953236362346e-06 + 3.754203567594348e-06 + 3.673820472678765e-06 + 3.594787733829004e-06 + 3.517089224074456e-06 + 3.440708918621393e-06 + 3.365630891050476e-06 + 3.291839308866544e-06 + 3.219318444550688e-06 + 3.148052669614368e-06 + 3.078026451876596e-06 + 3.009224366838337e-06 + 2.941631088840265e-06 + 2.875231390541817e-06 + 2.810010156134118e-06 + 2.745952369256776e-06 + 2.683043113446972e-06 + 2.621267584374269e-06 + 2.560611078236376e-06 + 2.501058994098514e-06 + 2.442596843868032e-06 + 2.385210240033010e-06 + 2.328884899906223e-06 + 2.273606655048426e-06 + 2.219361438179688e-06 + 2.166135288678882e-06 + 2.113914360085187e-06 + 2.062684907096663e-06 + 2.012433292648363e-06 + 1.963145994708785e-06 + 1.914809593193956e-06 + 1.867410777738679e-06 + 1.820936351687259e-06 + 1.775373220609589e-06 + 1.730708401732302e-06 + 1.686929024947410e-06 + 1.644022323067757e-06 + 1.601975642837270e-06 + 1.560776441656341e-06 + 1.520412279298016e-06 + 1.480870830315665e-06 + 1.442139880698395e-06 + 1.404207319625165e-06 + 1.367061149777226e-06 + 1.330689483155573e-06 + 1.295080535963615e-06 + 1.260222640873472e-06 + 1.226104237141460e-06 + 1.192713867093549e-06 + 1.160040189500492e-06 + 1.128071969424555e-06 + 1.096798076008267e-06 + 1.066207493417804e-06 + 1.036289310659725e-06 + 1.007032721493895e-06 + 9.784270338592895e-07 + 9.504616588874799e-07 + 9.231261128777845e-07 + 8.964100256766087e-07 + 8.703031286825232e-07 + 8.447952585150839e-07 + 8.198763641874936e-07 + 7.955364948286083e-07 + 7.717658049346632e-07 + 7.485545598914059e-07 + 7.258931238786546e-07 + 7.037719665410901e-07 + 6.821816664988581e-07 + 6.611128998791421e-07 + 6.405564483055948e-07 + 6.205032003278225e-07 + 6.009441408121614e-07 + 5.818703600152187e-07 + 5.632730528626383e-07 + 5.451435094097432e-07 + 5.274731247677268e-07 + 5.102533965000488e-07 + 4.934759162480692e-07 + 4.771323800991179e-07 + 4.612145841662162e-07 + 4.457144176881476e-07 + 4.306238737382497e-07 + 4.159350428538716e-07 + 4.016401077051372e-07 + 3.877313539402903e-07 + 3.742011622647174e-07 + 3.610420046499709e-07 + 3.482464547968944e-07 + 3.358071790150475e-07 + 3.237169340794661e-07 + 3.119685770955318e-07 + 3.005550554375413e-07 + 2.894694062090586e-07 + 2.787047650882795e-07 + 2.682543557659438e-07 + 2.581114909968296e-07 + 2.482695801165890e-07 + 2.387221182727572e-07 + 2.294626890008751e-07 + 2.204849703818711e-07 + 2.117827241548630e-07 + 2.033497996067267e-07 + 1.951801384624956e-07 + 1.872677641420447e-07 + 1.796067867093452e-07 + 1.721914062527642e-07 + 1.650159027896018e-07 + 1.580746421942365e-07 + 1.513620776690125e-07 + 1.448727406655565e-07 + 1.386012477684331e-07 + 1.325423000867771e-07 + 1.266906752868051e-07 + 1.210412352836648e-07 + 1.155889239131470e-07 + 1.103287600176973e-07 + 1.052558455547364e-07 + 1.003653617115032e-07 + 9.565256321109978e-08 + 9.111278671292873e-08 + 8.674144556415159e-08 + 8.253402527034383e-08 + 7.848609177536523e-08 + 7.459328520671134e-08 + 7.085131661573615e-08 + 6.725597593242166e-08 + 6.380312478868793e-08 + 6.048869454868003e-08 + 5.730869376312298e-08 + 5.425920030057914e-08 + 5.133636064443935e-08 + 4.853639660958214e-08 + 4.585559704969457e-08 + 4.329031836991616e-08 + 4.083699031045398e-08 + 3.849210751108346e-08 + 3.625223113812888e-08 + 3.411399355038575e-08 + 3.207408998824199e-08 + 3.012928121511996e-08 + 2.827639696078167e-08 + 2.651232793166280e-08 + 2.483402934338396e-08 + 2.323852311474514e-08 + 2.172289034680298e-08 + 2.028427559044146e-08 + 1.891988778877689e-08 + 1.762699339728036e-08 + 1.640292125402531e-08 + 1.524506224991720e-08 + 1.415086320949184e-08 + 1.311783222446413e-08 + 1.214353712508316e-08 + 1.122560019257377e-08 + 1.036170378687137e-08 + 9.549587722603005e-09 + 8.787044877843281e-09 + 8.071926961605364e-09 + 7.402140903161946e-09 + 6.775645388472587e-09 + 6.190456604129221e-09 + 5.644643792323680e-09 + 5.136326726146161e-09 + 4.663681262833891e-09 + 4.224934232596794e-09 + 3.818361845837701e-09 + 3.442294891302550e-09 + 3.095113141246544e-09 + 2.775244665814773e-09 + 2.481170514027403e-09 + 2.211418834557044e-09 + 1.964565051498559e-09 + 1.739235878927056e-09 + 1.534103359980556e-09 + 1.347885831336700e-09 + 1.179351163642234e-09 + 1.027310894648576e-09 + 8.906218977827446e-10 + 7.681887799049897e-10 + 6.589582548472836e-10 + 5.619214238374888e-10 + 4.761152985830568e-10 + 4.006175343477256e-10 + 3.345492227248809e-10 + 2.770755362128679e-10 + 2.274009179016229e-10 + 1.847722840116674e-10 + 1.484788129161787e-10 + 1.178476648704315e-10 + 9.224748788184573e-11 + 7.108739902161543e-11 + 5.381328706422649e-11 + 3.991151299277479e-11 + 2.890715266694438e-11 + 2.036091648096713e-11 + 1.387293445657948e-11 + 9.080348320515963e-12 + 5.654863833225597e-12 + 3.306507674167889e-12 + 1.780689492983021e-12 + 8.563646514161430e-13 + 3.496037276049943e-13 + 1.104943293034196e-13 + 2.161027650041889e-14 + 5.028049336448720e-16 + 0.000000000000000e+00 + 0.000000000000000e+00 + 7.485272593961791e+04 + 1.493519318697679e+05 + 2.234970831294996e+05 + 2.972876533345640e+05 + 3.707231243774131e+05 + 4.438029864272003e+05 + 5.165267379297798e+05 + 5.888938856077072e+05 + 6.609039444602393e+05 + 7.325564377633340e+05 + 8.038508970696502e+05 + 8.747868622085482e+05 + 9.453638812860892e+05 + 1.015581510685036e+06 + 1.085439315064852e+06 + 1.154936867361703e+06 + 1.224073748788454e+06 + 1.292849548834672e+06 + 1.361263865266626e+06 + 1.429316304127287e+06 + 1.497006479736322e+06 + 1.564334014690106e+06 + 1.631298539861710e+06 + 1.697899694400910e+06 + 1.764137125734180e+06 + 1.830010489564698e+06 + 1.895519449872339e+06 + 1.960663678913685e+06 + 2.025442857222013e+06 + 2.089856673607306e+06 + 2.153904825156247e+06 + 2.217587017232217e+06 + 2.280902963475303e+06 + 2.343852385802290e+06 + 2.406435014406665e+06 + 2.468650587758616e+06 + 2.530498852605031e+06 + 2.591979563969503e+06 + 2.653092485152322e+06 + 2.713837387730482e+06 + 2.774214051557675e+06 + 2.834222264764299e+06 + 2.893861823757447e+06 + 2.953132533220919e+06 + 3.012034206115212e+06 + 3.070566663677527e+06 + 3.128729735421764e+06 + 3.186523259138526e+06 + 3.243947080895115e+06 + 3.301001055035538e+06 + 3.357685044180497e+06 + 3.413998919227402e+06 + 3.469942559350358e+06 + 3.525515852000176e+06 + 3.580718692904367e+06 + 3.635550986067140e+06 + 3.690012643769410e+06 + 3.744103586568788e+06 + 3.797823743299592e+06 + 3.851173051072835e+06 + 3.904151455276238e+06 + 3.956758909574216e+06 + 4.008995375907890e+06 + 4.060860824495079e+06 + 4.112355233830309e+06 + 4.163478590684799e+06 + 4.214230890106475e+06 + 4.264612135419963e+06 + 4.314622338226588e+06 + 4.364261518404378e+06 + 4.413529704108060e+06 + 4.462426931769069e+06 + 4.510953246095533e+06 + 4.559108700072286e+06 + 4.606893354960858e+06 + 4.654307280299488e+06 + 4.701350553903108e+06 + 4.748023261863358e+06 + 4.794325498548575e+06 + 4.840257366603798e+06 + 4.885818976950768e+06 + 4.931010448787929e+06 + 4.975831909590418e+06 + 5.020283495110085e+06 + 5.064365349375471e+06 + 5.108077624691823e+06 + 5.151420481641092e+06 + 5.194394089081923e+06 + 5.236998624149668e+06 + 5.279234272256376e+06 + 5.321101227090800e+06 + 5.362599690618395e+06 + 5.403729873081312e+06 + 5.444491992998408e+06 + 5.484886277165242e+06 + 5.524912960654069e+06 + 5.564572286813851e+06 + 5.603864507270245e+06 + 5.642789881925615e+06 + 5.681348678959022e+06 + 5.719541174826232e+06 + 5.757367654259708e+06 + 5.794828410268617e+06 + 5.831923744138825e+06 + 5.868653965432902e+06 + 5.905019391990117e+06 + 5.941020349926441e+06 + 5.976657173634544e+06 + 6.011930205783804e+06 + 6.046839797320290e+06 + 6.081386307466779e+06 + 6.115570103722748e+06 + 6.149391561864376e+06 + 6.182851065944540e+06 + 6.215949008292822e+06 + 6.248685789515501e+06 + 6.281061818495561e+06 + 6.313077512392686e+06 + 6.344733296643257e+06 + 6.376029604960365e+06 + 6.406966879333793e+06 + 6.437545570030033e+06 + 6.467766135592271e+06 + 6.497629042840399e+06 + 6.527134766871008e+06 + 6.556283791057392e+06 + 6.585076607049545e+06 + 6.613513714774162e+06 + 6.641595622434638e+06 + 6.669322846511072e+06 + 6.696695911760264e+06 + 6.723715351215711e+06 + 6.750381706187615e+06 + 6.776695526262878e+06 + 6.802657369305105e+06 + 6.828267801454599e+06 + 6.853527397128366e+06 + 6.878436739020114e+06 + 6.902996418100249e+06 + 6.927207033615881e+06 + 6.951069193090819e+06 + 6.974583512325577e+06 + 6.997750615397365e+06 + 7.020571134660101e+06 + 7.043045710744397e+06 + 7.065174992557568e+06 + 7.086959637283633e+06 + 7.108400310383311e+06 + 7.129497685594021e+06 + 7.150252444929884e+06 + 7.170665278681723e+06 + 7.190736885417059e+06 + 7.210467971980117e+06 + 7.229859253491821e+06 + 7.248911453349800e+06 + 7.267625303228384e+06 + 7.286001543078595e+06 + 7.304040921128170e+06 + 7.321744193881537e+06 + 7.339112126119829e+06 + 7.356145490900878e+06 + 7.372845069559221e+06 + 7.389211651706094e+06 + 7.405246035229432e+06 + 7.420949026293878e+06 + 7.436321439340766e+06 + 7.451364097088138e+06 + 7.466077830530737e+06 + 7.480463478940005e+06 + 7.494521889864086e+06 + 7.508253919127826e+06 + 7.521660430832772e+06 + 7.534742297357169e+06 + 7.547500399355968e+06 + 7.559935625760819e+06 + 7.572048873780073e+06 + 7.583841048898782e+06 + 7.595313064878696e+06 + 7.606465843758276e+06 + 7.617300315852673e+06 + 7.627817419753745e+06 + 7.638018102330050e+06 + 7.647903318726848e+06 + 7.657474032366097e+06 + 7.666731214946462e+06 + 7.675675846443305e+06 + 7.684308915108687e+06 + 7.692631417471377e+06 + 7.700644358336839e+06 + 7.708348750787240e+06 + 7.715745616181449e+06 + 7.722835984155037e+06 + 7.729620892620270e+06 + 7.736101387766127e+06 + 7.742278524058277e+06 + 7.748153364239094e+06 + 7.753726979327655e+06 + 7.759000448619737e+06 + 7.763974859687817e+06 + 7.768651308381077e+06 + 7.773030898825390e+06 + 7.777114743423343e+06 + 7.780903962854218e+06 + 7.784399686073998e+06 + 7.787603050315368e+06 + 7.790515201087713e+06 + 7.793137292177119e+06 + 7.795470485646380e+06 + 7.797515951834979e+06 + 7.799274869359110e+06 + 7.800748425111664e+06 + 7.801937814262233e+06 + 7.802844240257111e+06 + 7.803468914819297e+06 + 7.803813057948484e+06 + 7.803877897921070e+06 + 7.803664671290156e+06 + 7.803174622885538e+06 + 7.802409005813722e+06 + 7.801369081457905e+06 + 7.800056119477995e+06 + 7.798471397810592e+06 + 7.796616202669007e+06 + 7.794491828543244e+06 + 7.792099578200010e+06 + 7.789440762682715e+06 + 7.786516701311473e+06 + 7.783328721683091e+06 + 7.779878159671083e+06 + 7.776166359425665e+06 + 7.772194673373749e+06 + 7.767964462218953e+06 + 7.763477094941595e+06 + 7.758733948798692e+06 + 7.753736409323964e+06 + 7.748485870327833e+06 + 7.742983733897421e+06 + 7.737231410396548e+06 + 7.731230318465743e+06 + 7.724981885022229e+06 + 7.718487545259933e+06 + 7.711748742649483e+06 + 7.704766928938209e+06 + 7.697543564150140e+06 + 7.690080116586007e+06 + 7.682378062823243e+06 + 7.674438887715982e+06 + 7.666264084395060e+06 + 7.657855154268012e+06 + 7.649213607019073e+06 + 7.640340960609185e+06 + 7.631238741275986e+06 + 7.621908483533815e+06 + 7.612351730173716e+06 + 7.602570032263434e+06 + 7.592564949147409e+06 + 7.582338048446788e+06 + 7.571890906059416e+06 + 7.561225106159844e+06 + 7.550342241199316e+06 + 7.539243911905785e+06 + 7.527931727283904e+06 + 7.516407304615024e+06 + 7.504672269457198e+06 + 7.492728241314434e+06 + 7.480576638153253e+06 + 7.468218723741564e+06 + 7.455655760425863e+06 + 7.442889013018662e+06 + 7.429919748798485e+06 + 7.416749237509867e+06 + 7.403378751363355e+06 + 7.389809565035504e+06 + 7.376042955668893e+06 + 7.362080202872100e+06 + 7.347922588719724e+06 + 7.333571397752370e+06 + 7.319027916976659e+06 + 7.304293435865226e+06 + 7.289369246356713e+06 + 7.274256642855777e+06 + 7.258956922233087e+06 + 7.243471383825325e+06 + 7.227801329435183e+06 + 7.211948063331366e+06 + 7.195912892248592e+06 + 7.179697125387593e+06 + 7.163302074415106e+06 + 7.146729053463890e+06 + 7.129979379132709e+06 + 7.113054370486340e+06 + 7.095955349055575e+06 + 7.078683638837219e+06 + 7.061240566294081e+06 + 7.043627460354994e+06 + 7.025845652414794e+06 + 7.007896476334333e+06 + 6.989781268440476e+06 + 6.971501367526096e+06 + 6.953058114850082e+06 + 6.934452854137335e+06 + 6.915686931578767e+06 + 6.896761695831302e+06 + 6.877678498017876e+06 + 6.858438691727439e+06 + 6.839043633014950e+06 + 6.819494680401385e+06 + 6.799793194873727e+06 + 6.779940539884974e+06 + 6.759938081354136e+06 + 6.739787187666233e+06 + 6.719489229672301e+06 + 6.699045580689385e+06 + 6.678457616500544e+06 + 6.657726715354848e+06 + 6.636854257967380e+06 + 6.615841627519235e+06 + 6.594690209657516e+06 + 6.573401392495350e+06 + 6.551976566611862e+06 + 6.530417125052197e+06 + 6.508724463327511e+06 + 6.486899979414972e+06 + 6.464945073757762e+06 + 6.442861149265066e+06 + 6.420649611312096e+06 + 6.398311867740063e+06 + 6.375849328856199e+06 + 6.353263407433745e+06 + 6.330555518711952e+06 + 6.307727080396090e+06 + 6.284779512657427e+06 + 6.261714238133261e+06 + 6.238532681926889e+06 + 6.215236271607627e+06 + 6.191826437210800e+06 + 6.168304611237747e+06 + 6.144672228655818e+06 + 6.120930726898375e+06 + 6.097081545864792e+06 + 6.073126127920459e+06 + 6.049065917896771e+06 + 6.024902363091142e+06 + 6.000636913266994e+06 + 5.976271020653763e+06 + 5.951806139946896e+06 + 5.927243728307854e+06 + 5.902585245364108e+06 + 5.877832153209144e+06 + 5.852985916402456e+06 + 5.828048001969552e+06 + 5.803019879401957e+06 + 5.777903020657199e+06 + 5.752698900158828e+06 + 5.727408994796395e+06 + 5.702034783925475e+06 + 5.676577749367647e+06 + 5.651039375410506e+06 + 5.625421148807654e+06 + 5.599724558778713e+06 + 5.573951097009313e+06 + 5.548102257651095e+06 + 5.522179537321717e+06 + 5.496184435104840e+06 + 5.470118452550147e+06 + 5.443983093673326e+06 + 5.417779864956085e+06 + 5.391510275346133e+06 + 5.365175836257203e+06 + 5.338778061569033e+06 + 5.312318467627374e+06 + 5.285798573243992e+06 + 5.259219899696662e+06 + 5.232583970729172e+06 + 5.205892312551324e+06 + 5.179146453838928e+06 + 5.152347925733812e+06 + 5.125498261843814e+06 + 5.098598998242779e+06 + 5.071651673470572e+06 + 5.044657828533064e+06 + 5.017619006902143e+06 + 4.990536754515706e+06 + 4.963412619777663e+06 + 4.936248153557934e+06 + 4.909044909192459e+06 + 4.881804442483181e+06 + 4.854528311698059e+06 + 4.827218077571064e+06 + 4.799875303302180e+06 + 4.772501554557402e+06 + 4.745098399468737e+06 + 4.717667408634203e+06 + 4.690210155117837e+06 + 4.662728214449679e+06 + 4.635223164625784e+06 + 4.607696586108224e+06 + 4.580150061825077e+06 + 4.552585177170437e+06 + 4.525003520004408e+06 + 4.497406680653105e+06 + 4.469796251908661e+06 + 4.442173829029216e+06 + 4.414541009738924e+06 + 4.386899394227949e+06 + 4.359250585152469e+06 + 4.331596187634676e+06 + 4.303937809262771e+06 + 4.276277060090966e+06 + 4.248615552639492e+06 + 4.220954901894585e+06 + 4.193296725308497e+06 + 4.165642642799490e+06 + 4.137994276751838e+06 + 4.110353252015833e+06 + 4.082721195907769e+06 + 4.055099738209962e+06 + 4.027490511170732e+06 + 3.999895149504419e+06 + 3.972315290391369e+06 + 3.944752573477943e+06 + 3.917208640876513e+06 + 3.889685137165464e+06 + 3.862183709389194e+06 + 3.834706007058111e+06 + 3.807253682148634e+06 + 3.779828389103201e+06 + 3.752431784830255e+06 + 3.725065528704253e+06 + 3.697731282565667e+06 + 3.670430710720977e+06 + 3.643165479942679e+06 + 3.615937259469279e+06 + 3.588747721005293e+06 + 3.561598538721256e+06 + 3.534491389253709e+06 + 3.507427951705208e+06 + 3.480409907644318e+06 + 3.453438941105620e+06 + 3.426516738589706e+06 + 3.399644989063177e+06 + 3.372825383958653e+06 + 3.346070405300663e+06 + 3.319465764034668e+06 + 3.293075510459727e+06 + 3.266878224028187e+06 + 3.240869303162930e+06 + 3.215052515525467e+06 + 3.189425288821125e+06 + 3.163986116962672e+06 + 3.138734196986282e+06 + 3.113668261011591e+06 + 3.088787111422079e+06 + 3.064089610843659e+06 + 3.039574592322599e+06 + 3.015240898358055e+06 + 2.991087379108502e+06 + 2.967112887579310e+06 + 2.943316282301805e+06 + 2.919696425519270e+06 + 2.896252185185970e+06 + 2.872982434032476e+06 + 2.849886048305633e+06 + 2.826961910405915e+06 + 2.804208907130155e+06 + 2.781625929123051e+06 + 2.759211873644148e+06 + 2.736965641930875e+06 + 2.714886139213983e+06 + 2.692972277509980e+06 + 2.671222972611625e+06 + 2.649637144535781e+06 + 2.628213719738566e+06 + 2.606951628696007e+06 + 2.585849806893141e+06 + 2.564907195391811e+06 + 2.544122739173453e+06 + 2.523495389008342e+06 + 2.503024101022326e+06 + 2.482707834844927e+06 + 2.462545556054412e+06 + 2.442536236120740e+06 + 2.422678849863131e+06 + 2.402972377820649e+06 + 2.383415806334849e+06 + 2.364008125120872e+06 + 2.344748329835404e+06 + 2.325635421794777e+06 + 2.306668405813749e+06 + 2.287846292982958e+06 + 2.269168099757709e+06 + 2.250632846050586e+06 + 2.232239558087812e+06 + 2.213987266957964e+06 + 2.195875007435864e+06 + 2.177901821473463e+06 + 2.160066755143500e+06 + 2.142368857948063e+06 + 2.124807187017495e+06 + 2.107380803567688e+06 + 2.090088772295509e+06 + 2.072930165358991e+06 + 2.055904058912681e+06 + 2.039009532618940e+06 + 2.022245673041180e+06 + 2.005611571515091e+06 + 1.989106323649552e+06 + 1.972729030298793e+06 + 1.956478797095825e+06 + 1.940354734917566e+06 + 1.924355960016194e+06 + 1.908481592593613e+06 + 1.892730757829583e+06 + 1.877102587255698e+06 + 1.861596215902473e+06 + 1.846210783186639e+06 + 1.830945435012332e+06 + 1.815799321323061e+06 + 1.800771597138300e+06 + 1.785861422681660e+06 + 1.771067961851037e+06 + 1.756390384114008e+06 + 1.741827864373811e+06 + 1.727379581118251e+06 + 1.713044718232857e+06 + 1.698822464839031e+06 + 1.684712013805151e+06 + 1.670712563808684e+06 + 1.656823318006428e+06 + 1.643043483055245e+06 + 1.629372271790869e+06 + 1.615808901794204e+06 + 1.602352594044968e+06 + 1.589002574975609e+06 + 1.575758076322615e+06 + 1.562618333634041e+06 + 1.549582587186907e+06 + 1.536650081999105e+06 + 1.523820067124634e+06 + 1.511091797064626e+06 + 1.498464530780005e+06 + 1.485937531189585e+06 + 1.473510066471004e+06 + 1.461181408848471e+06 + 1.448950834625962e+06 + 1.436817625926470e+06 + 1.424781068397768e+06 + 1.412840451407005e+06 + 1.400995070309470e+06 + 1.389244224245752e+06 + 1.377587216393601e+06 + 1.366023355231714e+06 + 1.354551952598923e+06 + 1.343172324427429e+06 + 1.331883792566280e+06 + 1.320685682335073e+06 + 1.309577322956444e+06 + 1.298558048406515e+06 + 1.287627196362487e+06 + 1.276784109496439e+06 + 1.266028134784103e+06 + 1.255358622320226e+06 + 1.244774926866806e+06 + 1.234276408467881e+06 + 1.223862430240984e+06 + 1.213532358913532e+06 + 1.203285566621101e+06 + 1.193121428991948e+06 + 1.183039325468658e+06 + 1.173038640304339e+06 + 1.163118761212139e+06 + 1.153279079987504e+06 + 1.143518992564451e+06 + 1.133837898317260e+06 + 1.124235201394960e+06 + 1.114710310121780e+06 + 1.105262636013577e+06 + 1.095891594215875e+06 + 1.086596604396163e+06 + 1.077377090233802e+06 + 1.068232479242110e+06 + 1.059162202266971e+06 + 1.050165693700918e+06 + 1.041242393088349e+06 + 1.032391743039430e+06 + 1.023613189178869e+06 + 1.014906181731556e+06 + 1.006270174355871e+06 + 9.977046244036794e+05 + 9.892089935677024e+05 + 9.807827465832273e+05 + 9.724253516436865e+05 + 9.641362810046984e+05 + 9.559150102252770e+05 + 9.477610188719215e+05 + 9.396737899139688e+05 + 9.316528091743080e+05 + 9.236975665277143e+05 + 9.158075561735997e+05 + 9.079822751134991e+05 + 9.002212236533747e+05 + 8.925239057014534e+05 + 8.848898281604102e+05 + 8.773185021612709e+05 + 8.698094424648809e+05 + 8.623621663528095e+05 + 8.549761944053203e+05 + 8.476510510446843e+05 + 8.403862637513808e+05 + 8.331813638606276e+05 + 8.260358860837971e+05 + 8.189493675787508e+05 + 8.119213484350282e+05 + 8.049513730504208e+05 + 7.980389891667470e+05 + 7.911837471192866e+05 + 7.843852003599521e+05 + 7.776429053419405e+05 + 7.709564221446526e+05 + 7.643253140173181e+05 + 7.577491471803443e+05 + 7.512274911598503e+05 + 7.447599180231568e+05 + 7.383460026539576e+05 + 7.319853241781880e+05 + 7.256774641850799e+05 + 7.194220067146936e+05 + 7.132185395309449e+05 + 7.070666529520636e+05 + 7.009659400728574e+05 + 6.949159975085063e+05 + 6.889164244086053e+05 + 6.829668228095188e+05 + 6.770667979456050e+05 + 6.712159574726904e+05 + 6.654139120281750e+05 + 6.596602754201142e+05 + 6.539546638052445e+05 + 6.482966962660485e+05 + 6.426859949531512e+05 + 6.371221843431445e+05 + 6.316048918617199e+05 + 6.261337478197705e+05 + 6.207083848228143e+05 + 6.153284385055641e+05 + 6.099935472682050e+05 + 6.047033517354903e+05 + 5.994574954765684e+05 + 5.942556247644721e+05 + 5.890973881315375e+05 + 5.839824370501742e+05 + 5.789104255366600e+05 + 5.738810098259439e+05 + 5.688938490610282e+05 + 5.639486048510919e+05 + 5.590449410545636e+05 + 5.541825244030216e+05 + 5.493610239467514e+05 + 5.445801109611018e+05 + 5.398394595653637e+05 + 5.351387461239359e+05 + 5.304776492529869e+05 + 5.258558503354813e+05 + 5.212730328842620e+05 + 5.167288826663374e+05 + 5.122230881980868e+05 + 5.077553400369428e+05 + 5.033253309731948e+05 + 4.989327564601310e+05 + 4.945773139811803e+05 + 4.902587033143605e+05 + 4.859766266490913e+05 + 4.817307881770549e+05 + 4.775208945372272e+05 + 4.733466546121029e+05 + 4.692077791818720e+05 + 4.651039814988278e+05 + 4.610349770333872e+05 + 4.570004830895544e+05 + 4.530002193970305e+05 + 4.490339078641191e+05 + 4.451012721716975e+05 + 4.412020383482487e+05 + 4.373359346346440e+05 + 4.335026910461138e+05 + 4.297020398291695e+05 + 4.259337153289251e+05 + 4.221974536699117e+05 + 4.184929932504728e+05 + 4.148200744355357e+05 + 4.111784393468680e+05 + 4.075678324287459e+05 + 4.039879999559653e+05 + 4.004386899065928e+05 + 3.969196525836200e+05 + 3.934306400492930e+05 + 3.899714060581240e+05 + 3.865417066557175e+05 + 3.831412995849459e+05 + 3.797699442729113e+05 + 3.764274023566474e+05 + 3.731134371233769e+05 + 3.698278135622827e+05 + 3.665702987827645e+05 + 3.633406615280775e+05 + 3.601386722825769e+05 + 3.569641034535824e+05 + 3.538167290782596e+05 + 3.506963250581825e+05 + 3.476026690576235e+05 + 3.445355402744773e+05 + 3.414947198014101e+05 + 3.384799905288533e+05 + 3.354911367821696e+05 + 3.325279446663284e+05 + 3.295902021531574e+05 + 3.266776986085196e+05 + 3.237902250531212e+05 + 3.209275743946921e+05 + 3.180895409646722e+05 + 3.152759207164211e+05 + 3.124865112943467e+05 + 3.097211117598721e+05 + 3.069795229448014e+05 + 3.042615472339510e+05 + 3.015669883675390e+05 + 2.988956518738279e+05 + 2.962473447339852e+05 + 2.936218752484120e+05 + 2.910190535228683e+05 + 2.884386911235514e+05 + 2.858806009067059e+05 + 2.833445973150213e+05 + 2.808304963224871e+05 + 2.783381153152796e+05 + 2.758672731341289e+05 + 2.734177900017136e+05 + 2.709894875428715e+05 + 2.685821889451269e+05 + 2.661957187314447e+05 + 2.638299027837526e+05 + 2.614845683836235e+05 + 2.591595441417535e+05 + 2.568546601323142e+05 + 2.545697477628685e+05 + 2.523046396706559e+05 + 2.500591699434964e+05 + 2.478331740644226e+05 + 2.456264886409644e+05 + 2.434389516128850e+05 + 2.412704023657298e+05 + 2.391206814330098e+05 + 2.369896306904384e+05 + 2.348770933197599e+05 + 2.327829136173563e+05 + 2.307069372775990e+05 + 2.286490111745211e+05 + 2.266089832623643e+05 + 2.245867029748616e+05 + 2.225820209237310e+05 + 2.205947887183862e+05 + 2.186248592921233e+05 + 2.166720868039439e+05 + 2.147363264605261e+05 + 2.128174347706827e+05 + 2.109152693889452e+05 + 2.090296889843596e+05 + 2.071605535077064e+05 + 2.053077240108720e+05 + 2.034710625599661e+05 + 2.016504325007521e+05 + 1.998456982030229e+05 + 1.980567250370223e+05 + 1.962833797138016e+05 + 1.945255298870997e+05 + 1.927830441323482e+05 + 1.910557922964590e+05 + 1.893436452442803e+05 + 1.876464748512476e+05 + 1.859641540986015e+05 + 1.842965568967865e+05 + 1.826435581738182e+05 + 1.810050340354470e+05 + 1.793808614993985e+05 + 1.777709185575675e+05 + 1.761750842223128e+05 + 1.745932384546507e+05 + 1.730252623268235e+05 + 1.714710377603550e+05 + 1.699304475698536e+05 + 1.684033757560533e+05 + 1.668897071565261e+05 + 1.653893274220018e+05 + 1.639021233234161e+05 + 1.624279825159149e+05 + 1.609667934564489e+05 + 1.595184456811041e+05 + 1.580828296015031e+05 + 1.566598363968120e+05 + 1.552493582649543e+05 + 1.538512882702573e+05 + 1.524655202573447e+05 + 1.510919491064028e+05 + 1.497304705096718e+05 + 1.483809809029638e+05 + 1.470433777326744e+05 + 1.457175592117312e+05 + 1.444034242942686e+05 + 1.431008729933990e+05 + 1.418098060421684e+05 + 1.405301248671864e+05 + 1.392617319094920e+05 + 1.380045303259556e+05 + 1.367584239714278e+05 + 1.355233176631985e+05 + 1.342991169575939e+05 + 1.330857281367400e+05 + 1.318830583220329e+05 + 1.306910153813551e+05 + 1.295095079605118e+05 + 1.283384454277201e+05 + 1.271777378619642e+05 + 1.260272961737778e+05 + 1.248870320778846e+05 + 1.237568578844075e+05 + 1.226366865814357e+05 + 1.215264320654815e+05 + 1.204260088572700e+05 + 1.193353321256488e+05 + 1.182543179024696e+05 + 1.171828828329888e+05 + 1.161209441971572e+05 + 1.150684200368906e+05 + 1.140252290172446e+05 + 1.129912905290012e+05 + 1.119665246989244e+05 + 1.109508522377596e+05 + 1.099441945145749e+05 + 1.089464735474612e+05 + 1.079576119759672e+05 + 1.069775332658437e+05 + 1.060061613940063e+05 + 1.050434208615313e+05 + 1.040892370245828e+05 + 1.031435357647047e+05 + 1.022062434696232e+05 + 1.012772873208916e+05 + 1.003565950684523e+05 + 9.944409498683986e+04 + 9.853971604124359e+04 + 9.764338777276211e+04 + 9.675504027159151e+04 + 9.587460425195689e+04 + 9.500201099602232e+04 + 9.413719236230559e+04 + 9.328008081668093e+04 + 9.243060935253548e+04 + 9.158871152497102e+04 + 9.075432150057999e+04 + 8.992737394814150e+04 + 8.910780408381460e+04 + 8.829554772812854e+04 + 8.749054118346877e+04 + 8.669272128864085e+04 + 8.590202548133289e+04 + 8.511839166524833e+04 + 8.434175826486869e+04 + 8.357206428147563e+04 + 8.280924917713979e+04 + 8.205325293940101e+04 + 8.130401609766987e+04 + 8.056147962909786e+04 + 7.982558503959214e+04 + 7.909627435336930e+04 + 7.837349003289423e+04 + 7.765717506702039e+04 + 7.694727293156023e+04 + 7.624372753229675e+04 + 7.554648331079309e+04 + 7.485548516457589e+04 + 7.417067840840557e+04 + 7.349200889763620e+04 + 7.281942291879337e+04 + 7.215286716185752e+04 + 7.149228885123666e+04 + 7.083763563584826e+04 + 7.018885556144921e+04 + 6.954589717270633e+04 + 6.890870943981725e+04 + 6.827724174191867e+04 + 6.765144392787226e+04 + 6.703126624692857e+04 + 6.641665935636370e+04 + 6.580757439267423e+04 + 6.520396286738058e+04 + 6.460577668030654e+04 + 6.401296820028803e+04 + 6.342549017134596e+04 + 6.284329573351656e+04 + 6.226633845994343e+04 + 6.169457228450359e+04 + 6.112795154311772e+04 + 6.056643099463883e+04 + 6.000996574136701e+04 + 5.945851127624574e+04 + 5.891202351267684e+04 + 5.837045869755570e+04 + 5.783377345558033e+04 + 5.730192482224794e+04 + 5.677487016169035e+04 + 5.625256720967715e+04 + 5.573497408983271e+04 + 5.522204924870504e+04 + 5.471375150828325e+04 + 5.421004006036229e+04 + 5.371087441354901e+04 + 5.321621444975662e+04 + 5.272602039469258e+04 + 5.224025278120609e+04 + 5.175887251825204e+04 + 5.128184085536604e+04 + 5.080911934597862e+04 + 5.034066989608183e+04 + 4.987645473684455e+04 + 4.941643640415772e+04 + 4.896057778639881e+04 + 4.850884207769758e+04 + 4.806119277190277e+04 + 4.761759372195903e+04 + 4.717800906318111e+04 + 4.674240321587012e+04 + 4.631074096205200e+04 + 4.588298735942246e+04 + 4.545910774537743e+04 + 4.503906779825089e+04 + 4.462283347373459e+04 + 4.421037101686000e+04 + 4.380164697651120e+04 + 4.339662816996042e+04 + 4.299528171729217e+04 + 4.259757504220949e+04 + 4.220347581706527e+04 + 4.181295199605238e+04 + 4.142597183433527e+04 + 4.104250383855765e+04 + 4.066251680332360e+04 + 4.028597979815155e+04 + 3.991286213370509e+04 + 3.954313341370015e+04 + 3.917676350485435e+04 + 3.881372250661203e+04 + 3.845398080843414e+04 + 3.809750905736740e+04 + 3.774427812448402e+04 + 3.739425915595785e+04 + 3.704742356207799e+04 + 3.670374297805164e+04 + 3.636318929176292e+04 + 3.602573463927982e+04 + 3.569135138416072e+04 + 3.536001215672983e+04 + 3.503168981923036e+04 + 3.470635744840465e+04 + 3.438398837321134e+04 + 3.406455615393208e+04 + 3.374803457309094e+04 + 3.343439765983423e+04 + 3.312361965186892e+04 + 3.281567499925836e+04 + 3.251053841177231e+04 + 3.220818480242078e+04 + 3.190858928930939e+04 + 3.161172723179306e+04 + 3.131757418548949e+04 + 3.102610591483113e+04 + 3.073729842710719e+04 + 3.045112791454117e+04 + 3.016757076691534e+04 + 2.988660361584138e+04 + 2.960820327321778e+04 + 2.933234674204399e+04 + 2.905901125674980e+04 + 2.878817423326317e+04 + 2.851981328372868e+04 + 2.825390622682930e+04 + 2.799043106146120e+04 + 2.772936599846356e+04 + 2.747068942978120e+04 + 2.721437991599866e+04 + 2.696041623625263e+04 + 2.670877735469368e+04 + 2.645944239579323e+04 + 2.621239068106394e+04 + 2.596760172331024e+04 + 2.572505519859007e+04 + 2.548473096648493e+04 + 2.524660907253227e+04 + 2.501066972455740e+04 + 2.477689330466365e+04 + 2.454526037995217e+04 + 2.431575168300119e+04 + 2.408834810912627e+04 + 2.386303072521571e+04 + 2.363978076395976e+04 + 2.341857962988761e+04 + 2.319940888531031e+04 + 2.298225025100848e+04 + 2.276708562349589e+04 + 2.255389704384009e+04 + 2.234266670372097e+04 + 2.213337697918792e+04 + 2.192601038403303e+04 + 2.172054957464747e+04 + 2.151697738916041e+04 + 2.131527680276242e+04 + 2.111543093252480e+04 + 2.091742306098243e+04 + 2.072123660422193e+04 + 2.052685512482519e+04 + 2.033426234716574e+04 + 2.014344212098682e+04 + 1.995437843426629e+04 + 1.976705544479930e+04 + 1.958145743480243e+04 + 1.939756881213320e+04 + 1.921537413684756e+04 + 1.903485810052154e+04 + 1.885600554387774e+04 + 1.867880143037429e+04 + 1.850323084111479e+04 + 1.832927901452323e+04 + 1.815693131178678e+04 + 1.798617320994377e+04 + 1.781699034075772e+04 + 1.764936845332217e+04 + 1.748329340510924e+04 + 1.731875119617678e+04 + 1.715572795110660e+04 + 1.699420990919751e+04 + 1.683418344358223e+04 + 1.667563503873475e+04 + 1.651855128954979e+04 + 1.636291893365341e+04 + 1.620872481606836e+04 + 1.605595588699932e+04 + 1.590459923143625e+04 + 1.575464203906202e+04 + 1.560607160533683e+04 + 1.545887535681969e+04 + 1.531304082443131e+04 + 1.516855564487381e+04 + 1.502540756956110e+04 + 1.488358445432479e+04 + 1.474307426938193e+04 + 1.460386509288249e+04 + 1.446594509814283e+04 + 1.432930256776952e+04 + 1.419392589719183e+04 + 1.405980357729324e+04 + 1.392692420658459e+04 + 1.379527648729671e+04 + 1.366484921045415e+04 + 1.353563127160237e+04 + 1.340761167272577e+04 + 1.328077950662285e+04 + 1.315512397004589e+04 + 1.303063434970357e+04 + 1.290730001970431e+04 + 1.278511047006237e+04 + 1.266405528001385e+04 + 1.254412410683634e+04 + 1.242530670538782e+04 + 1.230759293022181e+04 + 1.219097272173875e+04 + 1.207543611008671e+04 + 1.196097321598039e+04 + 1.184757424417080e+04 + 1.173522949287449e+04 + 1.162392934478714e+04 + 1.151366426384512e+04 + 1.140442480834694e+04 + 1.129620161634352e+04 + 1.118898540470730e+04 + 1.108276698378667e+04 + 1.097753724013271e+04 + 1.087328713748510e+04 + 1.077000773143687e+04 + 1.066769015148828e+04 + 1.056632560317495e+04 + 1.046590537899618e+04 + 1.036642084361778e+04 + 1.026786343864257e+04 + 1.017022468912003e+04 + 1.007349618928313e+04 + 9.977669609110566e+03 + 9.882736699245763e+03 + 9.788689277747368e+03 + 9.695519238085319e+03 + 9.603218551128393e+03 + 9.511779252802526e+03 + 9.421193453258842e+03 + 9.331453338088948e+03 + 9.242551155902798e+03 + 9.154479227014926e+03 + 9.067229945268125e+03 + 8.980795767109534e+03 + 8.895169220035050e+03 + 8.810342900070031e+03 + 8.726309463870590e+03 + 8.643061639313548e+03 + 8.560592220958239e+03 + 8.478894062727115e+03 + 8.397960086705119e+03 + 8.317783279679375e+03 + 8.238356687487571e+03 + 8.159673421912185e+03 + 8.081726656780256e+03 + 8.004509624589081e+03 + 7.928015622553179e+03 + 7.852238006535134e+03 + 7.777170190090504e+03 + 7.702805651284038e+03 + 7.629137923269694e+03 + 7.556160595291118e+03 + 7.483867321684783e+03 + 7.412251809739373e+03 + 7.341307820993124e+03 + 7.271029178292266e+03 + 7.201409756944273e+03 + 7.132443488123264e+03 + 7.064124361053455e+03 + 6.996446414793790e+03 + 6.929403743407001e+03 + 6.862990497342477e+03 + 6.797200875139910e+03 + 6.732029129721631e+03 + 6.667469570161823e+03 + 6.603516551523545e+03 + 6.540164480186692e+03 + 6.477407817185619e+03 + 6.415241069687017e+03 + 6.353658796883785e+03 + 6.292655608307650e+03 + 6.232226157518487e+03 + 6.172365149982773e+03 + 6.113067340658933e+03 + 6.054327527853383e+03 + 5.996140560115221e+03 + 5.938501333076975e+03 + 5.881404784939355e+03 + 5.824845903828315e+03 + 5.768819722461626e+03 + 5.713321315085574e+03 + 5.658345805472132e+03 + 5.603888359764917e+03 + 5.549944184846402e+03 + 5.496508537213697e+03 + 5.443576713947663e+03 + 5.391144051368102e+03 + 5.339205932358427e+03 + 5.287757780134069e+03 + 5.236795058291703e+03 + 5.186313275265001e+03 + 5.136307977685731e+03 + 5.086774751617681e+03 + 5.037709226449485e+03 + 4.989107068721900e+03 + 4.940963984504954e+03 + 4.893275721702213e+03 + 4.846038063285039e+03 + 4.799246830729600e+03 + 4.752897888312231e+03 + 4.706987133608333e+03 + 4.661510499922751e+03 + 4.616463963202395e+03 + 4.571843532909213e+03 + 4.527645253893021e+03 + 4.483865209302220e+03 + 4.440499515324517e+03 + 4.397544325863069e+03 + 4.354995830474305e+03 + 4.312850249817291e+03 + 4.271103841242844e+03 + 4.229752898292512e+03 + 4.188793745275323e+03 + 4.148222740634546e+03 + 4.108036277499050e+03 + 4.068230779797937e+03 + 4.028802705831530e+03 + 3.989748546349355e+03 + 3.951064821861195e+03 + 3.912748086813885e+03 + 3.874794926585441e+03 + 3.837201956255947e+03 + 3.799965825835211e+03 + 3.763083213042953e+03 + 3.726550823283946e+03 + 3.690365398229224e+03 + 3.654523706502605e+03 + 3.619022543185401e+03 + 3.583858737044133e+03 + 3.549029144619817e+03 + 3.514530650270483e+03 + 3.480360168271582e+03 + 3.446514639490202e+03 + 3.412991033253400e+03 + 3.379786348393921e+03 + 3.346897609451590e+03 + 3.314321868815865e+03 + 3.282056205761040e+03 + 3.250097724763853e+03 + 3.218443559530467e+03 + 3.187090870156345e+03 + 3.156036839918450e+03 + 3.125278679259401e+03 + 3.094813626126841e+03 + 3.064638941053712e+03 + 3.034751909739014e+03 + 3.005149845581890e+03 + 2.975830084543172e+03 + 2.946789986568892e+03 + 2.918026936797055e+03 + 2.889538343339952e+03 + 2.861321641252218e+03 + 2.833374286700636e+03 + 2.805693756659566e+03 + 2.778277556469224e+03 + 2.751123212016022e+03 + 2.724228269242043e+03 + 2.697590301914011e+03 + 2.671206904109025e+03 + 2.645075688971241e+03 + 2.619194294405700e+03 + 2.593560380141823e+03 + 2.568171626715687e+03 + 2.543025737542386e+03 + 2.518120435211838e+03 + 2.493453462030127e+03 + 2.469022585172818e+03 + 2.444825590018873e+03 + 2.420860280374604e+03 + 2.397124484732366e+03 + 2.373616049394018e+03 + 2.350332838436762e+03 + 2.327272738609171e+03 + 2.304433655218562e+03 + 2.281813512846715e+03 + 2.259410255601743e+03 + 2.237221844685107e+03 + 2.215246260811306e+03 + 2.193481506123082e+03 + 2.171925598942706e+03 + 2.150576574458654e+03 + 2.129432488764117e+03 + 2.108491414442484e+03 + 2.087751441300839e+03 + 2.067210678851041e+03 + 2.046867252353801e+03 + 2.026719303565292e+03 + 2.006764993817073e+03 + 1.987002500493615e+03 + 1.967430017215378e+03 + 1.948045754608514e+03 + 1.928847939062915e+03 + 1.909834815497017e+03 + 1.891004644375446e+03 + 1.872355700452609e+03 + 1.853886275802303e+03 + 1.835594678424827e+03 + 1.817479231062563e+03 + 1.799538272810955e+03 + 1.781770158220284e+03 + 1.764173256646331e+03 + 1.746745953278825e+03 + 1.729486647489427e+03 + 1.712393752933919e+03 + 1.695465699568615e+03 + 1.678700930817119e+03 + 1.662097903850073e+03 + 1.645655091966725e+03 + 1.629370981458450e+03 + 1.613244072202941e+03 + 1.597272880432907e+03 + 1.581455933682248e+03 + 1.565791771294578e+03 + 1.550278951289465e+03 + 1.534916043199639e+03 + 1.519701626827566e+03 + 1.504634298568118e+03 + 1.489712666891078e+03 + 1.474935351996483e+03 + 1.460300988758884e+03 + 1.445808223853118e+03 + 1.431455716350942e+03 + 1.417242138357507e+03 + 1.403166173386480e+03 + 1.389226518010158e+03 + 1.375421881007569e+03 + 1.361750982089843e+03 + 1.348212553926219e+03 + 1.334805341072190e+03 + 1.321528098720046e+03 + 1.308379594477261e+03 + 1.295358607457036e+03 + 1.282463927300085e+03 + 1.269694355823267e+03 + 1.257048705895313e+03 + 1.244525800699936e+03 + 1.232124475203576e+03 + 1.219843575042076e+03 + 1.207681956049559e+03 + 1.195638485404256e+03 + 1.183712040621380e+03 + 1.171901509388141e+03 + 1.160205790292893e+03 + 1.148623791762007e+03 + 1.137154432301071e+03 + 1.125796641257775e+03 + 1.114549357263016e+03 + 1.103411528660030e+03 + 1.092382114454342e+03 + 1.081460082700689e+03 + 1.070644411095300e+03 + 1.059934087579793e+03 + 1.049328108720195e+03 + 1.038825480494848e+03 + 1.028425218971371e+03 + 1.018126348582361e+03 + 1.007927902927668e+03 + 9.978289254104482e+02 + 9.878284675679866e+02 + 9.779255898738762e+02 + 9.681193621623704e+02 + 9.584088623012988e+02 + 9.487931771658360e+02 + 9.392714022763922e+02 + 9.298426409505287e+02 + 9.205060057185552e+02 + 9.112606172437117e+02 + 9.021056036681564e+02 + 8.930401022331439e+02 + 8.840632580637879e+02 + 8.751742235821227e+02 + 8.663721600224155e+02 + 8.576562363107178e+02 + 8.490256285516970e+02 + 8.404795212594149e+02 + 8.320171064683727e+02 + 8.236375833541338e+02 + 8.153401590664150e+02 + 8.071240480244585e+02 + 7.989884718119321e+02 + 7.909326598382663e+02 + 7.829558483527250e+02 + 7.750572805483002e+02 + 7.672362073200991e+02 + 7.594918862313154e+02 + 7.518235817392584e+02 + 7.442305655888715e+02 + 7.367121158939042e+02 + 7.292675176279554e+02 + 7.218960630476328e+02 + 7.145970504767349e+02 + 7.073697847820883e+02 + 7.002135778527235e+02 + 6.931277475695276e+02 + 6.861116184218383e+02 + 6.791645215373005e+02 + 6.722857937677298e+02 + 6.654747784472139e+02 + 6.587308253877042e+02 + 6.520532900189463e+02 + 6.454415341582659e+02 + 6.388949257382737e+02 + 6.324128381494658e+02 + 6.259946511571761e+02 + 6.196397502963263e+02 + 6.133475264197438e+02 + 6.071173768094372e+02 + 6.009487042352257e+02 + 5.948409165626937e+02 + 5.887934278801442e+02 + 5.828056577149104e+02 + 5.768770306335107e+02 + 5.710069770500405e+02 + 5.651949326339243e+02 + 5.594403381224611e+02 + 5.537426401234757e+02 + 5.481012901152631e+02 + 5.425157443662293e+02 + 5.369854648776029e+02 + 5.315099185870243e+02 + 5.260885773011631e+02 + 5.207209179649268e+02 + 5.154064223119497e+02 + 5.101445770921985e+02 + 5.049348740937783e+02 + 4.997768095346806e+02 + 4.946698844046663e+02 + 4.896136047862775e+02 + 4.846074811232588e+02 + 4.796510285618583e+02 + 4.747437670096575e+02 + 4.698852205388910e+02 + 4.650749179071526e+02 + 4.603123926691712e+02 + 4.555971823563725e+02 + 4.509288288727789e+02 + 4.463068788207030e+02 + 4.417308827523721e+02 + 4.372003955530256e+02 + 4.327149765501697e+02 + 4.282741889383891e+02 + 4.238776002586995e+02 + 4.195247821802792e+02 + 4.152153101301976e+02 + 4.109487639564401e+02 + 4.067247273397822e+02 + 4.025427875685178e+02 + 3.984025363858318e+02 + 3.943035692477232e+02 + 3.902454851077117e+02 + 3.862278871965347e+02 + 3.822503823673904e+02 + 3.783125809462488e+02 + 3.744140974100913e+02 + 3.705545496319825e+02 + 3.667335588488600e+02 + 3.629507504654170e+02 + 3.592057531920941e+02 + 3.554981990087823e+02 + 3.518277237997781e+02 + 3.481939667588409e+02 + 3.445965704231110e+02 + 3.410351808614133e+02 + 3.375094474036331e+02 + 3.340190228447264e+02 + 3.305635632260179e+02 + 3.271427276872922e+02 + 3.237561788604219e+02 + 3.204035826343936e+02 + 3.170846077594078e+02 + 3.137989262426811e+02 + 3.105462135925898e+02 + 3.073261480789523e+02 + 3.041384108878384e+02 + 3.009826866110996e+02 + 2.978586626197819e+02 + 2.947660292278245e+02 + 2.917044799774912e+02 + 2.886737110898206e+02 + 2.856734216347172e+02 + 2.827033138205502e+02 + 2.797630925185924e+02 + 2.768524653868921e+02 + 2.739711430162750e+02 + 2.711188386172004e+02 + 2.682952682417776e+02 + 2.655001507075566e+02 + 2.627332073441469e+02 + 2.599941621974095e+02 + 2.572827420191948e+02 + 2.545986760992428e+02 + 2.519416963867954e+02 + 2.493115374004747e+02 + 2.467079361520411e+02 + 2.441306322519956e+02 + 2.415793676749632e+02 + 2.390538868162835e+02 + 2.365539368283623e+02 + 2.340792670618049e+02 + 2.316296291092408e+02 + 2.292047772816256e+02 + 2.268044680583002e+02 + 2.244284601550858e+02 + 2.220765149177526e+02 + 2.197483957886078e+02 + 2.174438683808943e+02 + 2.151627007150871e+02 + 2.129046628991759e+02 + 2.106695273325222e+02 + 2.084570687245048e+02 + 2.062670637405841e+02 + 2.040992912166423e+02 + 2.019535322675416e+02 + 1.998295699356443e+02 + 1.977271893831886e+02 + 1.956461779038128e+02 + 1.935863247355415e+02 + 1.915474213798307e+02 + 1.895292611678483e+02 + 1.875316391958052e+02 + 1.855543528482985e+02 + 1.835972014670904e+02 + 1.816599861324650e+02 + 1.797425099308505e+02 + 1.778445779263947e+02 + 1.759659969484732e+02 + 1.741065756739388e+02 + 1.722661246609050e+02 + 1.704444562587039e+02 + 1.686413847511611e+02 + 1.668567261386198e+02 + 1.650902980928235e+02 + 1.633419201885993e+02 + 1.616114136777071e+02 + 1.598986014805750e+02 + 1.582033083619683e+02 + 1.565253606828442e+02 + 1.548645864376032e+02 + 1.532208154238845e+02 + 1.515938789750561e+02 + 1.499836100268175e+02 + 1.483898432531407e+02 + 1.468124148175615e+02 + 1.452511624739094e+02 + 1.437059256375598e+02 + 1.421765451600804e+02 + 1.406628634671130e+02 + 1.391647245894945e+02 + 1.376819739463268e+02 + 1.362144585029153e+02 + 1.347620267668397e+02 + 1.333245285902542e+02 + 1.319018153476916e+02 + 1.304937399101563e+02 + 1.291001564496927e+02 + 1.277209206089626e+02 + 1.263558894821456e+02 + 1.250049214462411e+02 + 1.236678763269726e+02 + 1.223446153162142e+02 + 1.210350008534558e+02 + 1.197388968258891e+02 + 1.184561684202564e+02 + 1.171866820332312e+02 + 1.159303054814129e+02 + 1.146869078350233e+02 + 1.134563593521917e+02 + 1.122385316718855e+02 + 1.110332976232454e+02 + 1.098405311979204e+02 + 1.086601077608261e+02 + 1.074919038117468e+02 + 1.063357969778381e+02 + 1.051916662270612e+02 + 1.040593916259906e+02 + 1.029388543466298e+02 + 1.018299368341955e+02 + 1.007325226151947e+02 + 9.964649633160360e+01 + 9.857174381861689e+01 + 9.750815195741964e+01 + 9.645560874783912e+01 + 9.541400332430685e+01 + 9.438322582683587e+01 + 9.336316750654088e+01 + 9.235372071959520e+01 + 9.135477878367931e+01 + 9.036623609174318e+01 + 8.938798812110527e+01 + 8.841993128541375e+01 + 8.746196304638978e+01 + 8.651398190997742e+01 + 8.557588728936869e+01 + 8.464757961904559e+01 + 8.372896034435935e+01 + 8.281993180539014e+01 + 8.192039735296085e+01 + 8.103026128111058e+01 + 8.014942874495698e+01 + 7.927780590401203e+01 + 7.841529983858037e+01 + 7.756181847617736e+01 + 7.671727071446733e+01 + 7.588156631674119e+01 + 7.505461587372024e+01 + 7.423633095310755e+01 + 7.342662394299153e+01 + 7.262540803009301e+01 + 7.183259736215271e+01 + 7.104810687905130e+01 + 7.027185230277652e+01 + 6.950375028269488e+01 + 6.874371822566854e+01 + 6.799167430460268e+01 + 6.724753760248480e+01 + 6.651122793902957e+01 + 6.578266588744034e+01 + 6.506177289160765e+01 + 6.434847109859102e+01 + 6.364268339115934e+01 + 6.294433351272004e+01 + 6.225334589309644e+01 + 6.156964567793161e+01 + 6.089315882540817e+01 + 6.022381196253272e+01 + 5.956153243192431e+01 + 5.890624835784802e+01 + 5.825788851805937e+01 + 5.761638239540161e+01 + 5.698166019812880e+01 + 5.635365277455941e+01 + 5.573229169643452e+01 + 5.511750921575248e+01 + 5.450923819422894e+01 + 5.390741220139490e+01 + 5.331196548214332e+01 + 5.272283288224639e+01 + 5.213994992103501e+01 + 5.156325275682114e+01 + 5.099267813956413e+01 + 5.042816349744366e+01 + 4.986964685712531e+01 + 4.931706681520628e+01 + 4.877036264381848e+01 + 4.822947419115650e+01 + 4.769434186145065e+01 + 4.716490671386568e+01 + 4.664111036072763e+01 + 4.612289496053356e+01 + 4.561020331071316e+01 + 4.510297873475514e+01 + 4.460116508782800e+01 + 4.410470685618211e+01 + 4.361354903662819e+01 + 4.312763714409483e+01 + 4.264691728541290e+01 + 4.217133607138089e+01 + 4.170084064118655e+01 + 4.123537869187841e+01 + 4.077489840121474e+01 + 4.031934846724150e+01 + 3.986867813088396e+01 + 3.942283710103097e+01 + 3.898177559914080e+01 + 3.854544435810492e+01 + 3.811379455962712e+01 + 3.768677789368684e+01 + 3.726434656726136e+01 + 3.684645322013298e+01 + 3.643305096011807e+01 + 3.602409338288954e+01 + 3.561953452360934e+01 + 3.521932891649128e+01 + 3.482343153306697e+01 + 3.443179775085747e+01 + 3.404438343901749e+01 + 3.366114490993698e+01 + 3.328203887889210e+01 + 3.290702252020257e+01 + 3.253605343510379e+01 + 3.216908962260722e+01 + 3.180608952699355e+01 + 3.144701200592500e+01 + 3.109181631097627e+01 + 3.074046212448961e+01 + 3.039290952642646e+01 + 3.004911898312090e+01 + 2.970905136572861e+01 + 2.937266792505025e+01 + 2.903993030352619e+01 + 2.871080057194543e+01 + 2.838524113862799e+01 + 2.806321475945346e+01 + 2.774468462701056e+01 + 2.742961427426368e+01 + 2.711796757973106e+01 + 2.680970882898042e+01 + 2.650480264209962e+01 + 2.620321398820359e+01 + 2.590490820737408e+01 + 2.560985096325826e+01 + 2.531800827674159e+01 + 2.502934653543958e+01 + 2.474383242809609e+01 + 2.446143296972707e+01 + 2.418211555345684e+01 + 2.390584787621480e+01 + 2.363259794654911e+01 + 2.336233413255323e+01 + 2.309502509558490e+01 + 2.283063979379989e+01 + 2.256914753657127e+01 + 2.231051792982671e+01 + 2.205472087402378e+01 + 2.180172658443576e+01 + 2.155150556919213e+01 + 2.130402866286656e+01 + 2.105926696778129e+01 + 2.081719185319289e+01 + 2.057777503962135e+01 + 2.034098850331043e+01 + 2.010680447006149e+01 + 1.987519550063876e+01 + 1.964613442528577e+01 + 1.941959432580914e+01 + 1.919554855827313e+01 + 1.897397075515799e+01 + 1.875483482712853e+01 + 1.853811494931854e+01 + 1.832378554313712e+01 + 1.811182128985819e+01 + 1.790219714756028e+01 + 1.769488831219516e+01 + 1.748987022919803e+01 + 1.728711861052799e+01 + 1.708660939866258e+01 + 1.688831878332332e+01 + 1.669222321036695e+01 + 1.649829934690638e+01 + 1.630652410217020e+01 + 1.611687463464447e+01 + 1.592932831698975e+01 + 1.574386275792588e+01 + 1.556045580679505e+01 + 1.537908552081595e+01 + 1.519973018767385e+01 + 1.502236832516821e+01 + 1.484697865494502e+01 + 1.467354012841447e+01 + 1.450203191274907e+01 + 1.433243337158686e+01 + 1.416472409674524e+01 + 1.399888388846149e+01 + 1.383489273805109e+01 + 1.367273085728561e+01 + 1.351237865788487e+01 + 1.335381673920349e+01 + 1.319702591751814e+01 + 1.304198719993403e+01 + 1.288868177614133e+01 + 1.273709104714046e+01 + 1.258719659672914e+01 + 1.243898018736778e+01 + 1.229242378719869e+01 + 1.214750954184370e+01 + 1.200421977308728e+01 + 1.186253699764556e+01 + 1.172244390266849e+01 + 1.158392335040491e+01 + 1.144695839482221e+01 + 1.131153225250660e+01 + 1.117762830926809e+01 + 1.104523013501555e+01 + 1.091432145806843e+01 + 1.078488617566414e+01 + 1.065690836231677e+01 + 1.053037224372639e+01 + 1.040526220941685e+01 + 1.028156282144551e+01 + 1.015925878941852e+01 + 1.003833498402321e+01 + 9.918776439618298e+00 + 9.800568334039683e+00 + 9.683696005431957e+00 + 9.568144946174611e+00 + 9.453900787919052e+00 + 9.340949322860192e+00 + 9.229276489267212e+00 + 9.118868359446189e+00 + 9.009711163642070e+00 + 8.901791273796370e+00 + 8.795095191785947e+00 + 8.689609570648638e+00 + 8.585321201853549e+00 + 8.482217005296002e+00 + 8.380284046581689e+00 + 8.279509523576925e+00 + 8.179880760330461e+00 + 8.081385224417300e+00 + 7.984010508389771e+00 + 7.887744326863491e+00 + 7.792574535206823e+00 + 7.698489109519010e+00 + 7.605476146133772e+00 + 7.513523878187234e+00 + 7.422620653280592e+00 + 7.332754935473541e+00 + 7.243915322674125e+00 + 7.156090524556396e+00 + 7.069269365741123e+00 + 6.983440797017198e+00 + 6.898593877074525e+00 + 6.814717779007339e+00 + 6.731801797516630e+00 + 6.649835331100769e+00 + 6.568807890289103e+00 + 6.488709103685009e+00 + 6.409528699865692e+00 + 6.331256515961359e+00 + 6.253882503643640e+00 + 6.177396712290002e+00 + 6.101789297547966e+00 + 6.027050522250886e+00 + 5.953170745278158e+00 + 5.880140434362498e+00 + 5.807950156612827e+00 + 5.736590570385212e+00 + 5.666052441485230e+00 + 5.596326632559173e+00 + 5.527404095182261e+00 + 5.459275884510260e+00 + 5.391933149962507e+00 + 5.325367127621260e+00 + 5.259569151260657e+00 + 5.194530647433734e+00 + 5.130243129823101e+00 + 5.066698206416531e+00 + 5.003887570881058e+00 + 4.941803001453549e+00 + 4.880436373900456e+00 + 4.819779643419130e+00 + 4.759824844738533e+00 + 4.700564110200850e+00 + 4.641989649988083e+00 + 4.584093751736625e+00 + 4.526868793565781e+00 + 4.470307230074105e+00 + 4.414401594729965e+00 + 4.359144507919768e+00 + 4.304528662004969e+00 + 4.250546825086277e+00 + 4.197191851627622e+00 + 4.144456666159979e+00 + 4.092334266478478e+00 + 4.040817732600400e+00 + 3.989900213340690e+00 + 3.939574930757155e+00 + 3.889835183170760e+00 + 3.840674335613269e+00 + 3.792085827136757e+00 + 3.744063169627271e+00 + 3.696599938809659e+00 + 3.649689781941428e+00 + 3.603326418080199e+00 + 3.557503628382808e+00 + 3.512215261374414e+00 + 3.467455236465362e+00 + 3.423217534816642e+00 + 3.379496201686086e+00 + 3.336285348654454e+00 + 3.293579148649655e+00 + 3.251371841226642e+00 + 3.209657725464606e+00 + 3.168431157939940e+00 + 3.127686563504146e+00 + 3.087418425005168e+00 + 3.047621281181530e+00 + 3.008289735457784e+00 + 2.969418446796439e+00 + 2.931002129285814e+00 + 2.893035560502260e+00 + 2.855513571053400e+00 + 2.818431044951714e+00 + 2.781782927380857e+00 + 2.745564215550934e+00 + 2.709769959926351e+00 + 2.674395268211684e+00 + 2.639435297946605e+00 + 2.604885259637663e+00 + 2.570740420206636e+00 + 2.536996093812299e+00 + 2.503647644969884e+00 + 2.470690493614489e+00 + 2.438120105903952e+00 + 2.405931997235836e+00 + 2.374121735683807e+00 + 2.342684934097179e+00 + 2.311617253784086e+00 + 2.280914406216957e+00 + 2.250572146781167e+00 + 2.220586279273893e+00 + 2.190952653478714e+00 + 2.161667162009881e+00 + 2.132725747880213e+00 + 2.104124396422602e+00 + 2.075859132902099e+00 + 2.047926031580743e+00 + 2.020321209266692e+00 + 1.993040822266800e+00 + 1.966081073072867e+00 + 1.939438205166541e+00 + 1.913108500802827e+00 + 1.887088286918805e+00 + 1.861373929947225e+00 + 1.835961834345823e+00 + 1.810848447621857e+00 + 1.786030255213551e+00 + 1.761503779943143e+00 + 1.737265586321442e+00 + 1.713312275115213e+00 + 1.689640483715834e+00 + 1.666246890168493e+00 + 1.643128207113473e+00 + 1.620281182788458e+00 + 1.597702604935450e+00 + 1.575389294699183e+00 + 1.553338108095691e+00 + 1.531545938928566e+00 + 1.510009713407906e+00 + 1.488726392261371e+00 + 1.467692971937850e+00 + 1.446906480207705e+00 + 1.426363979153670e+00 + 1.406062564959602e+00 + 1.385999363940103e+00 + 1.366171536058216e+00 + 1.346576274130787e+00 + 1.327210800124654e+00 + 1.308072368828510e+00 + 1.289158266486426e+00 + 1.270465807665891e+00 + 1.251992339187792e+00 + 1.233735237889272e+00 + 1.215691907984316e+00 + 1.197859785096899e+00 + 1.180236333919823e+00 + 1.162819045897185e+00 + 1.145605442741095e+00 + 1.128593073973588e+00 + 1.111779515284567e+00 + 1.095162371880828e+00 + 1.078739275475473e+00 + 1.062507883334588e+00 + 1.046465881646835e+00 + 1.030610981874106e+00 + 1.014940920295882e+00 + 9.994534611152829e-01 + 9.841463930086627e-01 + 9.690175291593270e-01 + 9.540647093618980e-01 + 9.392857969051628e-01 + 9.246786792729085e-01 + 9.102412696338000e-01 + 8.959715035501604e-01 + 8.818673402027871e-01 + 8.679267639726700e-01 + 8.541477806828601e-01 + 8.405284189410931e-01 + 8.270667316623297e-01 + 8.137607925445277e-01 + 8.006086976874616e-01 + 7.876085666023992e-01 + 7.747585387876328e-01 + 7.620567754307064e-01 + 7.495014602973783e-01 + 7.370907966216457e-01 + 7.248230088485250e-01 + 7.126963429559050e-01 + 7.007090638413013e-01 + 6.888594572109231e-01 + 6.771458292779966e-01 + 6.655665045807759e-01 + 6.541198279036498e-01 + 6.428041637825399e-01 + 6.316178947210964e-01 + 6.205594229105623e-01 + 6.096271692194244e-01 + 5.988195719936049e-01 + 5.881350889221284e-01 + 5.775721954595900e-01 + 5.671293841176136e-01 + 5.568051664208875e-01 + 5.465980707194322e-01 + 5.365066418737608e-01 + 5.265294432641988e-01 + 5.166650544231339e-01 + 5.069120710261563e-01 + 4.972691065208522e-01 + 4.877347899800689e-01 + 4.783077664492044e-01 + 4.689866979017357e-01 + 4.597702612073233e-01 + 4.506571489021905e-01 + 4.416460700276524e-01 + 4.327357479310139e-01 + 4.239249212044141e-01 + 4.152123442913231e-01 + 4.065967853823845e-01 + 3.980770275573000e-01 + 3.896518691214699e-01 + 3.813201217302061e-01 + 3.730806117264635e-01 + 3.649321798861945e-01 + 3.568736798624809e-01 + 3.489039797704173e-01 + 3.410219613244947e-01 + 3.332265187013593e-01 + 3.255165605086018e-01 + 3.178910083898543e-01 + 3.103487958586359e-01 + 3.028888701360887e-01 + 2.955101913786422e-01 + 2.882117315106021e-01 + 2.809924755137371e-01 + 2.738514204626846e-01 + 2.667875748867065e-01 + 2.597999604522775e-01 + 2.528876103054370e-01 + 2.460495685714950e-01 + 2.392848919788540e-01 + 2.325926484518769e-01 + 2.259719167857841e-01 + 2.194217877354759e-01 + 2.129413627175208e-01 + 2.065297538274627e-01 + 2.001860848470379e-01 + 1.939094897086202e-01 + 1.876991127051828e-01 + 1.815541094417414e-01 + 1.754736452756007e-01 + 1.694568956974621e-01 + 1.635030470498157e-01 + 1.576112950508609e-01 + 1.517808453564031e-01 + 1.460109140364644e-01 + 1.403007261766342e-01 + 1.346495165784953e-01 + 1.290565300180611e-01 + 1.235210199704221e-01 + 1.180422494287421e-01 + 1.126194909047226e-01 + 1.072520253105725e-01 + 1.019391428882170e-01 + 9.668014295417104e-02 + 9.147433293860986e-02 + 8.632102939670111e-02 + 8.121955756354367e-02 + 7.616925052308399e-02 + 7.116945022739722e-02 + 6.621950693894064e-02 + 6.131877854698166e-02 + 5.646663155698649e-02 + 5.166244039926220e-02 + 4.690558691983024e-02 + 4.219546135291571e-02 + 3.753146148005052e-02 + 3.291299228995687e-02 + 2.833946692296159e-02 + 2.381030573044091e-02 + 1.932493608562982e-02 + 1.488279323951757e-02 + 1.048331933378767e-02 + 6.125963364112487e-03 + 1.810181928441051e-03 + -2.464561780353438e-03 + -6.698797872311066e-03 + -1.089304914227914e-02 + -1.504783206514415e-02 + -1.916365656771068e-02 + -2.324102555134537e-02 + -2.728043585421590e-02 + -3.128237791440187e-02 + -3.524733543518016e-02 + -3.917578628623598e-02 + -4.306820207473552e-02 + -4.692504798332473e-02 + -5.074678357205704e-02 + -5.453386225593208e-02 + -5.828673132987599e-02 + -6.200583267980977e-02 + -6.569160218738558e-02 + -6.934446987831328e-02 + -7.296486055963237e-02 + -7.655319319650564e-02 + -8.010988114185758e-02 + -8.363533270338043e-02 + -8.712995053780122e-02 + -9.059413193193132e-02 + -9.402826926020177e-02 + -9.743274938790861e-02 + -1.008079541064505e-01 + -1.041542604608216e-01 + -1.074720401131623e-01 + -1.107616598682374e-01 + -1.140234819193764e-01 + -1.172578632830942e-01 + -1.204651563472630e-01 + -1.236457089966172e-01 + -1.267998640735528e-01 + -1.299279600346222e-01 + -1.330303309819349e-01 + -1.361073061299021e-01 + -1.391592104828469e-01 + -1.421863647988072e-01 + -1.451890851855495e-01 + -1.481676836992942e-01 + -1.511224682100277e-01 + -1.540537421399553e-01 + -1.569618050599837e-01 + -1.598469524764272e-01 + -1.627094755828420e-01 + -1.655496618778017e-01 + -1.683677949236447e-01 + -1.711641541517163e-01 + -1.739390154343296e-01 + -1.766926508045859e-01 + -1.794253283544359e-01 + -1.821373127460387e-01 + -1.848288648981078e-01 + -1.875002419786971e-01 + -1.901516978443653e-01 + -1.927834826976386e-01 + -1.953958431846420e-01 + -1.979890227559544e-01 + -2.005632612952037e-01 + -2.031187952951354e-01 + -2.056558581586019e-01 + -2.081746798356756e-01 + -2.106754870586183e-01 + -2.131585035595214e-01 + -2.156239496932611e-01 + -2.180720427808593e-01 + -2.205029972492180e-01 + -2.229170242226580e-01 + -2.253143319488933e-01 + -2.276951258807381e-01 + -2.300596082676421e-01 + -2.324079786235529e-01 + -2.347404337552020e-01 + -2.370571673694448e-01 + -2.393583705568364e-01 + -2.416442317867498e-01 + -2.439149365965413e-01 + -2.461706679522762e-01 + -2.484116061949506e-01 + -2.506379289887288e-01 + -2.528498115997045e-01 + -2.550474267213886e-01 + -2.572309443123204e-01 + -2.594005320626763e-01 + -2.615563552809436e-01 + -2.636985766193694e-01 + -2.658273564814242e-01 + -2.679428529152917e-01 + -2.700452216727706e-01 + -2.721346163277996e-01 + -2.742111879420810e-01 + -2.762750854486131e-01 + -2.783264557785196e-01 + -2.803654434689624e-01 + -2.823921908087395e-01 + -2.844068381756654e-01 + -2.864095239109061e-01 + -2.884003839950212e-01 + -2.903795524955394e-01 + -2.923471617904532e-01 + -2.943033420191942e-01 + -2.962482212763220e-01 + -2.981819256807660e-01 + -3.001045795717013e-01 + -3.020163055275332e-01 + -3.039172239885258e-01 + -3.058074538261057e-01 + -3.076871122705734e-01 + -3.095563141516905e-01 + -3.114151727559051e-01 + -3.132637999546681e-01 + -3.151023057635463e-01 + -3.169307984585886e-01 + -3.187493845723606e-01 + -3.205581689117248e-01 + -3.223572548745565e-01 + -3.241467443749414e-01 + -3.259267374142799e-01 + -3.276973325691862e-01 + -3.294586270053045e-01 + -3.312107160890871e-01 + -3.329536937962613e-01 + -3.346876527428858e-01 + -3.364126840777460e-01 + -3.381288775295205e-01 + -3.398363212268976e-01 + -3.415351019703767e-01 + -3.432253052941175e-01 + -3.449070152459444e-01 + -3.465803145779997e-01 + -3.482452848007116e-01 + -3.499020059501182e-01 + -3.515505568826250e-01 + -3.531910152859590e-01 + -3.548234573752725e-01 + -3.564479582224311e-01 + -3.580645917875246e-01 + -3.596734306319520e-01 + -3.612745463110025e-01 + -3.628680093075062e-01 + -3.644538885605036e-01 + -3.660322520458910e-01 + -3.676031668332329e-01 + -3.691666987516095e-01 + -3.707229124779602e-01 + -3.722718716670178e-01 + -3.738136390608672e-01 + -3.753482761467254e-01 + -3.768758432749062e-01 + -3.783964002465877e-01 + -3.799100056353694e-01 + -3.814167166997948e-01 + -3.829165900177774e-01 + -3.844096813827866e-01 + -3.858960456354959e-01 + -3.873757364088854e-01 + -3.888488064600710e-01 + -3.903153077375676e-01 + -3.917752913452379e-01 + -3.932288075353338e-01 + -3.946759055987133e-01 + -3.961166339473677e-01 + -3.975510402779339e-01 + -3.989791715122148e-01 + -4.004010735521537e-01 + -4.018167915706618e-01 + -4.032263700983349e-01 + -4.046298528561717e-01 + -4.060272827301947e-01 + -4.074187016384220e-01 + -4.088041511045764e-01 + -4.101836719923956e-01 + -4.115573039941210e-01 + -4.129250863351419e-01 + -4.142870577221169e-01 + -4.156432559911067e-01 + -4.169937183587125e-01 + -4.183384813924751e-01 + -4.196775807600287e-01 + -4.210110517890521e-01 + -4.223389293056927e-01 + -4.236612469859990e-01 + -4.249780381358575e-01 + -4.262893357128523e-01 + -4.275951718561576e-01 + -4.288955781548201e-01 + -4.301905856512787e-01 + -4.314802245948859e-01 + -4.327645248125306e-01 + -4.340435157401011e-01 + -4.353172261319986e-01 + -4.365856843065750e-01 + -4.378489181140523e-01 + -4.391069545111200e-01 + -4.403598201489721e-01 + -4.416075414900063e-01 + -4.428501441617211e-01 + -4.440876533447125e-01 + -4.453200938994337e-01 + -4.465474900663732e-01 + -4.477698656375739e-01 + -4.489872440132049e-01 + -4.501996482091757e-01 + -4.514071007293397e-01 + -4.526096234423685e-01 + -4.538072381973440e-01 + -4.549999664346564e-01 + -4.561878287239103e-01 + -4.573708454564815e-01 + -4.585490367840940e-01 + -4.597224223838171e-01 + -4.608910215483196e-01 + -4.620548532229073e-01 + -4.632139359470852e-01 + -4.643682878385313e-01 + -4.655179266829866e-01 + -4.666628700026897e-01 + -4.678031349656974e-01 + -4.689387384105899e-01 + -4.700696968538801e-01 + -4.711960263436956e-01 + -4.723177425902391e-01 + -4.734348612979066e-01 + -4.745473978039702e-01 + -4.756553669920032e-01 + -4.767587833133353e-01 + -4.778576611599107e-01 + -4.789520147944609e-01 + -4.800418577229591e-01 + -4.811272034289876e-01 + -4.822080654661853e-01 + -4.832844566828151e-01 + -4.843563896207272e-01 + -4.854238767122197e-01 + -4.864869303811440e-01 + -4.875455627166793e-01 + -4.885997853037442e-01 + -4.896496095260600e-01 + -4.906950466868944e-01 + -4.917361080077966e-01 + -4.927728043326091e-01 + -4.938051461993963e-01 + -4.948331438840640e-01 + -4.958568075736277e-01 + -4.968761473512240e-01 + -4.978911731225704e-01 + -4.989018945002663e-01 + -4.999083208482176e-01 + -5.009104613021941e-01 + -5.019083248972503e-01 + -5.029019205602130e-01 + -5.038912570358333e-01 + -5.048763428126108e-01 + -5.058571862004539e-01 + -5.068337954285524e-01 + -5.078061785541023e-01 + -5.087743434557772e-01 + -5.097382978215317e-01 + -5.106980492175714e-01 + -5.116536051046018e-01 + -5.126049728172039e-01 + -5.135521595357905e-01 + -5.144951722594635e-01 + -5.154340177423219e-01 + -5.163687027408718e-01 + -5.172992340140365e-01 + -5.182256180631845e-01 + -5.191478612397169e-01 + -5.200659697981208e-01 + -5.209799499565910e-01 + -5.218898077706294e-01 + -5.227955490891694e-01 + -5.236971796844924e-01 + -5.245947053253370e-01 + -5.254881317688337e-01 + -5.263774645512288e-01 + -5.272627090126604e-01 + -5.281438703531137e-01 + -5.290209539977515e-01 + -5.298939652703512e-01 + -5.307629091340913e-01 + -5.316277904778723e-01 + -5.324886142110474e-01 + -5.333453852986519e-01 + -5.341981085648844e-01 + -5.350467886587517e-01 + -5.358914300418783e-01 + -5.367320373706596e-01 + -5.375686152841432e-01 + -5.384011680734792e-01 + -5.392297001047128e-01 + -5.400542157871245e-01 + -5.408747194069761e-01 + -5.416912150785628e-01 + -5.425037068232484e-01 + -5.433121987186342e-01 + -5.441166948984431e-01 + -5.449171994793097e-01 + -5.457137163885653e-01 + -5.465062494211190e-01 + -5.472948023341125e-01 + -5.480793790518950e-01 + -5.488599834110802e-01 + -5.496366191273470e-01 + -5.504092899443518e-01 + -5.511779994928784e-01 + -5.519427513029639e-01 + -5.527035490710489e-01 + -5.534603964642170e-01 + -5.542132970155166e-01 + -5.549622540841348e-01 + -5.557072711574547e-01 + -5.564483519201564e-01 + -5.571854997347668e-01 + -5.579187179919134e-01 + -5.586480102698358e-01 + -5.593733798207278e-01 + -5.600948299343745e-01 + -5.608123641702291e-01 + -5.615259858006734e-01 + -5.622356980349319e-01 + -5.629415042452564e-01 + -5.636434078437447e-01 + -5.643414121719204e-01 + -5.650355204128609e-01 + -5.657257359067179e-01 + -5.664120620074502e-01 + -5.670945018655422e-01 + -5.677730587569899e-01 + -5.684477360210265e-01 + -5.691185368946378e-01 + -5.697854646230838e-01 + -5.704485224954847e-01 + -5.711077138615709e-01 + -5.717630418949810e-01 + -5.724145097236072e-01 + -5.730621208149326e-01 + -5.737058785857441e-01 + -5.743457862811681e-01 + -5.749818469287452e-01 + -5.756140639044424e-01 + -5.762424408221788e-01 + -5.768669806578469e-01 + -5.774876866716016e-01 + -5.781045625480883e-01 + -5.787176116189466e-01 + -5.793268371095267e-01 + -5.799322422747063e-01 + -5.805338304985933e-01 + -5.811316053198213e-01 + -5.817255703521417e-01 + -5.823157287697720e-01 + -5.829020838792335e-01 + -5.834846393919914e-01 + -5.840633987477251e-01 + -5.846383653791880e-01 + -5.852095428966003e-01 + -5.857769347455314e-01 + -5.863405444497650e-01 + -5.869003757905312e-01 + -5.874564321893059e-01 + -5.880087171065591e-01 + -5.885572344676900e-01 + -5.891019879107314e-01 + -5.896429809482692e-01 + -5.901802172691951e-01 + -5.907137006344829e-01 + -5.912434348345698e-01 + -5.917694236578408e-01 + -5.922916709366750e-01 + -5.928101804947136e-01 + -5.933249560457028e-01 + -5.938360014505492e-01 + -5.943433207019595e-01 + -5.948469177828256e-01 + -5.953467966083078e-01 + -5.958429610716830e-01 + -5.963354151930935e-01 + -5.968241629967538e-01 + -5.973092085065336e-01 + -5.977905558475627e-01 + -5.982682091919279e-01 + -5.987421727060626e-01 + -5.992124504335189e-01 + -5.996790465513929e-01 + -6.001419654039183e-01 + -6.006012111567195e-01 + -6.010567880969444e-01 + -6.015087007096906e-01 + -6.019569532209035e-01 + -6.024015499278780e-01 + -6.028424953542454e-01 + -6.032797938560218e-01 + -6.037134498528102e-01 + -6.041434679778884e-01 + -6.045698527611498e-01 + -6.049926086966888e-01 + -6.054117403213235e-01 + -6.058272522301629e-01 + -6.062391491063152e-01 + -6.066474357377917e-01 + -6.070521167750024e-01 + -6.074531969194140e-01 + -6.078506811508277e-01 + -6.082445741313003e-01 + -6.086348804777613e-01 + -6.090216052590219e-01 + -6.094047534940796e-01 + -6.097843300758768e-01 + -6.101603397998798e-01 + -6.105327877567767e-01 + -6.109016791586246e-01 + -6.112670188349342e-01 + -6.116288118244032e-01 + -6.119870634033990e-01 + -6.123417788015699e-01 + -6.126929631816058e-01 + -6.130406216857481e-01 + -6.133847595846266e-01 + -6.137253821544012e-01 + -6.140624946688724e-01 + -6.143961025253998e-01 + -6.147262111225434e-01 + -6.150528258526081e-01 + -6.153759522847098e-01 + -6.156955958775930e-01 + -6.160117619293205e-01 + -6.163244560848270e-01 + -6.166336840291214e-01 + -6.169394513190288e-01 + -6.172417635406579e-01 + -6.175406263458638e-01 + -6.178360454670117e-01 + -6.181280266570051e-01 + -6.184165756494230e-01 + -6.187016981615432e-01 + -6.189834001775048e-01 + -6.192616876391890e-01 + -6.195365661632112e-01 + -6.198080416901866e-01 + -6.200761202902517e-01 + -6.203408078682074e-01 + -6.206021103982113e-01 + -6.208600339819080e-01 + -6.211145848662764e-01 + -6.213657689929696e-01 + -6.216135922609831e-01 + -6.218580610691606e-01 + -6.220991816188346e-01 + -6.223369599996067e-01 + -6.225714026287344e-01 + -6.228025157602012e-01 + -6.230303055354752e-01 + -6.232547783843585e-01 + -6.234759406837302e-01 + -6.236937987287140e-01 + -6.239083589372102e-01 + -6.241196278170006e-01 + -6.243276119301496e-01 + -6.245323178052680e-01 + -6.247337518435782e-01 + -6.249319204108740e-01 + -6.251268303788342e-01 + -6.253184884524475e-01 + -6.255069009766967e-01 + -6.256920746682559e-01 + -6.258740163701917e-01 + -6.260527328726909e-01 + -6.262282307801417e-01 + -6.264005167589047e-01 + -6.265695976839164e-01 + -6.267354804070123e-01 + -6.268981717782514e-01 + -6.270576786775590e-01 + -6.272140080071444e-01 + -6.273671666729116e-01 + -6.275171615794166e-01 + -6.276639997579259e-01 + -6.278076882572314e-01 + -6.279482340166599e-01 + -6.280856440870369e-01 + -6.282199255881125e-01 + -6.283510856176625e-01 + -6.284791313265480e-01 + -6.286040698559974e-01 + -6.287259082244099e-01 + -6.288446536460758e-01 + -6.289603134448759e-01 + -6.290728947784819e-01 + -6.291824049507972e-01 + -6.292888513285516e-01 + -6.293922409846477e-01 + -6.294925812787570e-01 + -6.295898797737625e-01 + -6.296841435561055e-01 + -6.297753800606519e-01 + -6.298635970427606e-01 + -6.299488015475858e-01 + -6.300310009429952e-01 + -6.301102030719377e-01 + -6.301864151575183e-01 + -6.302596446383568e-01 + -6.303298993873763e-01 + -6.303971867419816e-01 + -6.304615141097702e-01 + -6.305228892569528e-01 + -6.305813198299395e-01 + -6.306368133767261e-01 + -6.306893774095105e-01 + -6.307390196514921e-01 + -6.307857478830511e-01 + -6.308295697989852e-01 + -6.308704929504492e-01 + -6.309085250312678e-01 + -6.309436740902202e-01 + -6.309759477467396e-01 + -6.310053535533821e-01 + -6.310318994644450e-01 + -6.310555933584764e-01 + -6.310764430051494e-01 + -6.310944561130111e-01 + -6.311096405770181e-01 + -6.311220043955681e-01 + -6.311315554778594e-01 + -6.311383016234806e-01 + -6.311422506666780e-01 + -6.311434107412031e-01 + -6.311417897164122e-01 + -6.311373953205278e-01 + -6.311302357049288e-01 + -6.311203189243589e-01 + -6.311076528624781e-01 + -6.310922454230109e-01 + -6.310741047155799e-01 + -6.310532389686786e-01 + -6.310296560496979e-01 + -6.310033639455465e-01 + -6.309743708604478e-01 + -6.309426848479184e-01 + -6.309083139781310e-01 + -6.308712663782245e-01 + -6.308315500367087e-01 + -6.307891730465176e-01 + -6.307441436976022e-01 + -6.306964701860184e-01 + -6.306461606431758e-01 + -6.305932231701467e-01 + -6.305376659121696e-01 + -6.304794970062023e-01 + -6.304187245700347e-01 + -6.303553569805046e-01 + -6.302894025962669e-01 + -6.302208694979176e-01 + -6.301497657748981e-01 + -6.300760996409697e-01 + -6.299998795100692e-01 + -6.299211136514613e-01 + -6.298398102240708e-01 + -6.297559773915322e-01 + -6.296696236381925e-01 + -6.295807574013027e-01 + -6.294893864698483e-01 + -6.293955192818200e-01 + -6.292991645521353e-01 + -6.292003301703828e-01 + -6.290990243894826e-01 + -6.289952557946986e-01 + -6.288890326523825e-01 + -6.287803632353285e-01 + -6.286692558944412e-01 + -6.285557190083507e-01 + -6.284397608763040e-01 + -6.283213897612988e-01 + -6.282006141317734e-01 + -6.280774423035318e-01 + -6.279518824496017e-01 + -6.278239431715447e-01 + -6.276936329659792e-01 + -6.275609600561086e-01 + -6.274259327786519e-01 + -6.272885594754783e-01 + -6.271488484695774e-01 + -6.270068083043618e-01 + -6.268624473656752e-01 + -6.267157737459064e-01 + -6.265667960682418e-01 + -6.264155229504741e-01 + -6.262619625962428e-01 + -6.261061232036680e-01 + -6.259480131935671e-01 + -6.257876413121770e-01 + -6.256250157840939e-01 + -6.254601447340036e-01 + -6.252930366972961e-01 + -6.251237002207461e-01 + -6.249521437614608e-01 + -6.247783756272571e-01 + -6.246024042025771e-01 + -6.244242378628240e-01 + -6.242438848129136e-01 + -6.240613535991787e-01 + -6.238766528797192e-01 + -6.236897908519111e-01 + -6.235007757894835e-01 + -6.233096161276966e-01 + -6.231163203935827e-01 + -6.229208970207127e-01 + -6.227233543185018e-01 + -6.225237005131427e-01 + -6.223219439896936e-01 + -6.221180932821515e-01 + -6.219121568092019e-01 + -6.217041429518334e-01 + -6.214940600654550e-01 + -6.212819164164759e-01 + -6.210677203231969e-01 + -6.208514802076882e-01 + -6.206332045396580e-01 + -6.204129016762088e-01 + -6.201905798335827e-01 + -6.199662474146647e-01 + -6.197399127976954e-01 + -6.195115842446862e-01 + -6.192812702436963e-01 + -6.190489791767655e-01 + -6.188147191241502e-01 + -6.185784984898128e-01 + -6.183403256742428e-01 + -6.181002088023494e-01 + -6.178581563548247e-01 + -6.176141767575239e-01 + -6.173682779492388e-01 + -6.171204683326561e-01 + -6.168707564823446e-01 + -6.166191506410910e-01 + -6.163656588303806e-01 + -6.161102891648960e-01 + -6.158530502595377e-01 + -6.155939503803246e-01 + -6.153329975648972e-01 + -6.150702000825419e-01 + -6.148055662334609e-01 + -6.145391042808102e-01 + -6.142708224010276e-01 + -6.140007288452335e-01 + -6.137288318637003e-01 + -6.134551394181680e-01 + -6.131796597624336e-01 + -6.129024013716257e-01 + -6.126233721919256e-01 + -6.123425803085120e-01 + -6.120600340492245e-01 + -6.117757414641359e-01 + -6.114897106263320e-01 + -6.112019497520672e-01 + -6.109124671249474e-01 + -6.106212708396386e-01 + -6.103283687668304e-01 + -6.100337690977098e-01 + -6.097374799911853e-01 + -6.094395093910078e-01 + -6.091398653642833e-01 + -6.088385561024087e-01 + -6.085355898356886e-01 + -6.082309742898534e-01 + -6.079247173286892e-01 + -6.076168274335947e-01 + -6.073073125569597e-01 + -6.069961804304546e-01 + -6.066834389911383e-01 + -6.063690964791788e-01 + -6.060531610171422e-01 + -6.057356401425160e-01 + -6.054165419100932e-01 + -6.050958745176729e-01 + -6.047736455801236e-01 + -6.044498630713394e-01 + -6.041245351487873e-01 + -6.037976695799242e-01 + -6.034692741523835e-01 + -6.031393567367498e-01 + -6.028079252312161e-01 + -6.024749874110954e-01 + -6.021405510415967e-01 + -6.018046242357692e-01 + -6.014672148046522e-01 + -6.011283302871711e-01 + -6.007879786053585e-01 + -6.004461675660895e-01 + -6.001029047914931e-01 + -5.997581981158361e-01 + -5.994120553308485e-01 + -5.990644841299763e-01 + -5.987154923482985e-01 + -5.983650876334137e-01 + -5.980132773866208e-01 + -5.976600693793478e-01 + -5.973054714798371e-01 + -5.969494914365839e-01 + -5.965921366499265e-01 + -5.962334146660923e-01 + -5.958733334161985e-01 + -5.955119002909084e-01 + -5.951491227180785e-01 + -5.947850085981863e-01 + -5.944195652847630e-01 + -5.940528001366523e-01 + -5.936847210858313e-01 + -5.933153356463837e-01 + -5.929446511281147e-01 + -5.925726749957585e-01 + -5.921994146776245e-01 + -5.918248776317639e-01 + -5.914490714685531e-01 + -5.910720036664838e-01 + -5.906936815737919e-01 + -5.903141124838686e-01 + -5.899333037986367e-01 + -5.895512629817115e-01 + -5.891679973989596e-01 + -5.887835143677140e-01 + -5.883978211708401e-01 + -5.880109250637769e-01 + -5.876228334774148e-01 + -5.872335538851382e-01 + -5.868430932520374e-01 + -5.864514587624869e-01 + -5.860586578978172e-01 + -5.856646978273519e-01 + -5.852695857497323e-01 + -5.848733289413443e-01 + -5.844759343592291e-01 + -5.840774091773326e-01 + -5.836777608879199e-01 + -5.832769964405170e-01 + -5.828751228494087e-01 + -5.824721474461222e-01 + -5.820680771414760e-01 + -5.816629188869962e-01 + -5.812566799399100e-01 + -5.808493672421570e-01 + -5.804409877894702e-01 + -5.800315488931496e-01 + -5.796210574409676e-01 + -5.792095201878247e-01 + -5.787969440874288e-01 + -5.783833362006023e-01 + -5.779687035684036e-01 + -5.775530530727403e-01 + -5.771363914738108e-01 + -5.767187255974381e-01 + -5.763000625573806e-01 + -5.758804091701334e-01 + -5.754597721176316e-01 + -5.750381583420940e-01 + -5.746155745735496e-01 + -5.741920274506414e-01 + -5.737675239291555e-01 + -5.733420708457294e-01 + -5.729156748671344e-01 + -5.724883426083383e-01 + -5.720600807791615e-01 + -5.716308961653045e-01 + -5.712007954599810e-01 + -5.707697852850102e-01 + -5.703378722272431e-01 + -5.699050629355835e-01 + -5.694713639828742e-01 + -5.690367818720566e-01 + -5.686013232951076e-01 + -5.681649948578290e-01 + -5.677278029894381e-01 + -5.672897541841998e-01 + -5.668508549514105e-01 + -5.664111117923113e-01 + -5.659705312703284e-01 + -5.655291198708162e-01 + -5.650868839171556e-01 + -5.646438297396282e-01 + -5.641999637601537e-01 + -5.637552925169101e-01 + -5.633098223391846e-01 + -5.628635595080524e-01 + -5.624165104013904e-01 + -5.619686812129524e-01 + -5.615200782186582e-01 + -5.610707080464411e-01 + -5.606205767194389e-01 + -5.601696901727680e-01 + -5.597180550627928e-01 + -5.592656777039943e-01 + -5.588125640834879e-01 + -5.583587201544192e-01 + -5.579041522213387e-01 + -5.574488666805684e-01 + -5.569928694014434e-01 + -5.565361664370690e-01 + -5.560787640439252e-01 + -5.556206683493946e-01 + -5.551618853326125e-01 + -5.547024208857518e-01 + -5.542422809960397e-01 + -5.537814718214656e-01 + -5.533199995563840e-01 + -5.528578699065892e-01 + -5.523950887012330e-01 + -5.519316620139326e-01 + -5.514675956847120e-01 + -5.510028956792654e-01 + -5.505375681122012e-01 + -5.500716185256428e-01 + -5.496050526609592e-01 + -5.491378767346244e-01 + -5.486700964364043e-01 + -5.482017173715276e-01 + -5.477327453596050e-01 + -5.472631862648399e-01 + -5.467930458019109e-01 + -5.463223294669347e-01 + -5.458510432287255e-01 + -5.453791929736209e-01 + -5.449067840000759e-01 + -5.444338218966174e-01 + -5.439603123952597e-01 + -5.434862611282417e-01 + -5.430116738240560e-01 + -5.425365561436022e-01 + -5.420609134355956e-01 + -5.415847511352451e-01 + -5.411080748136494e-01 + -5.406308901525899e-01 + -5.401532026526268e-01 + -5.396750177177082e-01 + -5.391963408895222e-01 + -5.387171774649071e-01 + -5.382375326730170e-01 + -5.377574122460893e-01 + -5.372768215524759e-01 + -5.367957656717409e-01 + -5.363142502106338e-01 + -5.358322806031541e-01 + -5.353498619928253e-01 + -5.348669995911720e-01 + -5.343836987381033e-01 + -5.338999648334884e-01 + -5.334158029976483e-01 + -5.329312184607295e-01 + -5.324462166033185e-01 + -5.319608023124877e-01 + -5.314749807283198e-01 + -5.309887574568507e-01 + -5.305021373834015e-01 + -5.300151254489873e-01 + -5.295277270550899e-01 + -5.290399471460621e-01 + -5.285517906644440e-01 + -5.280632628772480e-01 + -5.275743687930865e-01 + -5.270851133485194e-01 + -5.265955015863814e-01 + -5.261055383771222e-01 + -5.256152286457003e-01 + -5.251245776074701e-01 + -5.246335901741112e-01 + -5.241422711040373e-01 + -5.236506252667474e-01 + -5.231586574906982e-01 + -5.226663726352896e-01 + -5.221737757253558e-01 + -5.216808715970676e-01 + -5.211876649797802e-01 + -5.206941607382560e-01 + -5.202003635069621e-01 + -5.197062778404161e-01 + -5.192119087096243e-01 + -5.187172609937223e-01 + -5.182223393579939e-01 + -5.177271482807863e-01 + -5.172316923981860e-01 + -5.167359765148800e-01 + -5.162400053033168e-01 + -5.157437833544083e-01 + -5.152473152222008e-01 + -5.147506055161414e-01 + -5.142536587718484e-01 + -5.137564794429633e-01 + -5.132590721404230e-01 + -5.127614414610688e-01 + -5.122635918987255e-01 + -5.117655278912282e-01 + -5.112672538418732e-01 + -5.107687741570007e-01 + -5.102700934318621e-01 + -5.097712161511054e-01 + -5.092721464952894e-01 + -5.087728887586368e-01 + -5.082734473837237e-01 + -5.077738269224511e-01 + -5.072740315773641e-01 + -5.067740655070005e-01 + -5.062739331645890e-01 + -5.057736389339202e-01 + -5.052731870571522e-01 + -5.047725816027976e-01 + -5.042718267556677e-01 + -5.037709267723895e-01 + -5.032698858620083e-01 + -5.027687082997551e-01 + -5.022673983228978e-01 + -5.017659599138892e-01 + -5.012643972224838e-01 + -5.007627144504877e-01 + -5.002609154317549e-01 + -4.997590043686060e-01 + -4.992569856498065e-01 + -4.987548629163329e-01 + -4.982526401066572e-01 + -4.977503215591422e-01 + -4.972479113357710e-01 + -4.967454132608579e-01 + -4.962428310876830e-01 + -4.957401690448965e-01 + -4.952374311333114e-01 + -4.947346209581790e-01 + -4.942317423902789e-01 + -4.937287995168010e-01 + -4.932257964768885e-01 + -4.927227367709692e-01 + -4.922196239985463e-01 + -4.917164622446805e-01 + -4.912132554897460e-01 + -4.907100074652099e-01 + -4.902067216434476e-01 + -4.897034018910860e-01 + -4.892000521396699e-01 + -4.886966760659724e-01 + -4.881932772071136e-01 + -4.876898592264887e-01 + -4.871864261030248e-01 + -4.866829812293292e-01 + -4.861795280147214e-01 + -4.856760706607693e-01 + -4.851726126266216e-01 + -4.846691571189501e-01 + -4.841657080652781e-01 + -4.836622690168214e-01 + -4.831588433295234e-01 + -4.826554347812969e-01 + -4.821520468729375e-01 + -4.816486828820545e-01 + -4.811453462713459e-01 + -4.806420406166251e-01 + -4.801387695127853e-01 + -4.796355363816449e-01 + -4.791323444815042e-01 + -4.786291970695780e-01 + -4.781260978727254e-01 + -4.776230503252043e-01 + -4.771200574973888e-01 + -4.766171228253449e-01 + -4.761142497486388e-01 + -4.756114415594656e-01 + -4.751087014547341e-01 + -4.746060327035595e-01 + -4.741034386828772e-01 + -4.736009226286348e-01 + -4.730984877717556e-01 + -4.725961373913972e-01 + -4.720938746127822e-01 + -4.715917026117559e-01 + -4.710896247171748e-01 + -4.705876440070008e-01 + -4.700857635192827e-01 + -4.695839864443119e-01 + -4.690823160197067e-01 + -4.685807553687465e-01 + -4.680793073815264e-01 + -4.675779752270572e-01 + -4.670767621237579e-01 + -4.665756710130068e-01 + -4.660747048828000e-01 + -4.655738667469987e-01 + -4.650731595624170e-01 + -4.645725864569897e-01 + -4.640721505529091e-01 + -4.635718546060059e-01 + -4.630717014390969e-01 + -4.625716940204364e-01 + -4.620718354412200e-01 + -4.615721285948061e-01 + -4.610725762550177e-01 + -4.605731814392849e-01 + -4.600739469669758e-01 + -4.595748755034438e-01 + -4.590759700613133e-01 + -4.585772335293120e-01 + -4.580786685902926e-01 + -4.575802780165928e-01 + -4.570820645542124e-01 + -4.565840309327809e-01 + -4.560861801183225e-01 + -4.555885148408127e-01 + -4.550910375242214e-01 + -4.545937511574892e-01 + -4.540966585092119e-01 + -4.535997618400510e-01 + -4.531030641427475e-01 + -4.526065682230286e-01 + -4.521102761991050e-01 + -4.516141909960451e-01 + -4.511183155232459e-01 + -4.506226519976986e-01 + -4.501272029129367e-01 + -4.496319709853875e-01 + -4.491369589904647e-01 + -4.486421693813144e-01 + -4.481476045149588e-01 + -4.476532669406306e-01 + -4.471591591812127e-01 + -4.466652837454304e-01 + -4.461716431800687e-01 + -4.456782399407630e-01 + -4.451850764283579e-01 + -4.446921550770006e-01 + -4.441994783152980e-01 + -4.437070485743053e-01 + -4.432148683148817e-01 + -4.427229399164072e-01 + -4.422312656936392e-01 + -4.417398479943503e-01 + -4.412486891611525e-01 + -4.407577915365379e-01 + -4.402671575060104e-01 + -4.397767893532367e-01 + -4.392866892910770e-01 + -4.387968597340107e-01 + -4.383073029681190e-01 + -4.378180210879689e-01 + -4.373290163087382e-01 + -4.368402909924953e-01 + -4.363518475687816e-01 + -4.358636880897754e-01 + -4.353758146064535e-01 + -4.348882293452391e-01 + -4.344009344375495e-01 + -4.339139321041776e-01 + -4.334272247184532e-01 + -4.329408142509990e-01 + -4.324547026649819e-01 + -4.319688922210647e-01 + -4.314833850702666e-01 + -4.309981832317592e-01 + -4.305132886299618e-01 + -4.300287034407881e-01 + -4.295444298858639e-01 + -4.290604699557538e-01 + -4.285768254794737e-01 + -4.280934983815344e-01 + -4.276104909880938e-01 + -4.271278053251724e-01 + -4.266454431999359e-01 + -4.261634065006114e-01 + -4.256816972235624e-01 + -4.252003173934318e-01 + -4.247192689049444e-01 + -4.242385536499166e-01 + -4.237581735472888e-01 + -4.232781305399061e-01 + -4.227984264773803e-01 + -4.223190631457318e-01 + -4.218400424415704e-01 + -4.213613663598799e-01 + -4.208830368540804e-01 + -4.204050553934426e-01 + -4.199274237890383e-01 + -4.194501442515862e-01 + -4.189732183034799e-01 + -4.184966476302857e-01 + -4.180204343198091e-01 + -4.175445800025869e-01 + -4.170690862458790e-01 + -4.165939547960728e-01 + -4.161191875920802e-01 + -4.156447863485650e-01 + -4.151707524172177e-01 + -4.146970878143395e-01 + -4.142237944432959e-01 + -4.137508735370036e-01 + -4.132783267358912e-01 + -4.128061559031381e-01 + -4.123343628419361e-01 + -4.118629489726368e-01 + -4.113919157251955e-01 + -4.109212649651476e-01 + -4.104509983224866e-01 + -4.099811172669397e-01 + -4.095116233202941e-01 + -4.090425181490535e-01 + -4.085738034100841e-01 + -4.081054804950061e-01 + -4.076375508831032e-01 + -4.071700161671795e-01 + -4.067028779692610e-01 + -4.062361376606066e-01 + -4.057697965353959e-01 + -4.053038563335837e-01 + -4.048383186645677e-01 + -4.043731849141807e-01 + -4.039084564103662e-01 + -4.034441345521896e-01 + -4.029802208230617e-01 + -4.025167167043049e-01 + -4.020536236099443e-01 + -4.015909428779320e-01 + -4.011286758375757e-01 + -4.006668238996902e-01 + -4.002053885607212e-01 + -3.997443712200008e-01 + -3.992837731698375e-01 + -3.988235956310492e-01 + -3.983638399923307e-01 + -3.979045076012857e-01 + -3.974455996644362e-01 + -3.969871175556028e-01 + -3.965290626455606e-01 + -3.960714361699136e-01 + -3.956142393772600e-01 + -3.951574735515524e-01 + -3.947011399985758e-01 + -3.942452398354794e-01 + -3.937897742524872e-01 + -3.933347447660869e-01 + -3.928801524442024e-01 + -3.924259983101159e-01 + -3.919722839283850e-01 + -3.915190103656777e-01 + -3.910661784756895e-01 + -3.906137895655892e-01 + -3.901618450499643e-01 + -3.897103461604648e-01 + -3.892592935529919e-01 + -3.888086884121152e-01 + -3.883585322771145e-01 + -3.879088261485191e-01 + -3.874595710701922e-01 + -3.870107681556456e-01 + -3.865624182556531e-01 + -3.861145225228043e-01 + -3.856670823544472e-01 + -3.852200986473037e-01 + -3.847735723987576e-01 + -3.843275047747315e-01 + -3.838818965026468e-01 + -3.834367486164244e-01 + -3.829920625791138e-01 + -3.825478392535004e-01 + -3.821040794322394e-01 + -3.816607841213338e-01 + -3.812179544462010e-01 + -3.807755914230820e-01 + -3.803336958584798e-01 + -3.798922686597118e-01 + -3.794513108818827e-01 + -3.790108236907996e-01 + -3.785708078449803e-01 + -3.781312640889848e-01 + -3.776921934895628e-01 + -3.772535969260590e-01 + -3.768154752452403e-01 + -3.763778294637203e-01 + -3.759406605249527e-01 + -3.755039692439986e-01 + -3.750677562951532e-01 + -3.746320226741715e-01 + -3.741967693787305e-01 + -3.737619968727901e-01 + -3.733277061470673e-01 + -3.728938983949728e-01 + -3.724605741247380e-01 + -3.720277341192338e-01 + -3.715953793429491e-01 + -3.711635103186269e-01 + -3.707321278978636e-01 + -3.703012331728173e-01 + -3.698708267439747e-01 + -3.694409092657609e-01 + -3.690114815572269e-01 + -3.685825444178367e-01 + -3.681540985405766e-01 + -3.677261445488677e-01 + -3.672986832474148e-01 + -3.668717154245260e-01 + -3.664452417818624e-01 + -3.660192630178876e-01 + -3.655937798167319e-01 + -3.651687928442244e-01 + -3.647443027770453e-01 + -3.643203103057334e-01 + -3.638968161236534e-01 + -3.634738208501973e-01 + -3.630513251204181e-01 + -3.626293296340570e-01 + -3.622078349959966e-01 + -3.617868418102546e-01 + -3.613663507559756e-01 + -3.609463624258993e-01 + -3.605268773947004e-01 + -3.601078962972172e-01 + -3.596894197319259e-01 + -3.592714482724653e-01 + -3.588539824938156e-01 + -3.584370229705016e-01 + -3.580205702765974e-01 + -3.576046249835235e-01 + -3.571891876270012e-01 + -3.567742587321382e-01 + -3.563598388637925e-01 + -3.559459285557617e-01 + -3.555325283179373e-01 + -3.551196386794110e-01 + -3.547072601423791e-01 + -3.542953931967586e-01 + -3.538840383835105e-01 + -3.534731961849680e-01 + -3.530628670336623e-01 + -3.526530514374770e-01 + -3.522437498876279e-01 + -3.518349628366106e-01 + -3.514266907403692e-01 + -3.510189340474827e-01 + -3.506116931966672e-01 + -3.502049686305037e-01 + -3.497987607886671e-01 + -3.493930701015490e-01 + -3.489878969787067e-01 + -3.485832418243141e-01 + -3.481791050461803e-01 + -3.477754870557839e-01 + -3.473723882547367e-01 + -3.469698090253131e-01 + -3.465677497415546e-01 + -3.461662107762984e-01 + -3.457651925045610e-01 + -3.453646952920019e-01 + -3.449647194961470e-01 + -3.445652654674211e-01 + -3.441663335489293e-01 + -3.437679240760669e-01 + -3.433700373751777e-01 + -3.429726737709668e-01 + -3.425758335909973e-01 + -3.421795171685511e-01 + -3.417837248082679e-01 + -3.413884567949557e-01 + -3.409937134173290e-01 + -3.405994949798623e-01 + -3.402058017865278e-01 + -3.398126341027504e-01 + -3.394199921921239e-01 + -3.390278763247643e-01 + -3.386362867748747e-01 + -3.382452237948665e-01 + -3.378546876227794e-01 + -3.374646785249709e-01 + -3.370751967392704e-01 + -3.366862424749215e-01 + -3.362978159833941e-01 + -3.359099174981644e-01 + -3.355225472182573e-01 + -3.351357053498810e-01 + -3.347493921057998e-01 + -3.343636076973308e-01 + -3.339783523006465e-01 + -3.335936261089775e-01 + -3.332094293460974e-01 + -3.328257621547543e-01 + -3.324426246881435e-01 + -3.320600171619409e-01 + -3.316779397280902e-01 + -3.312963925265998e-01 + -3.309153757279716e-01 + -3.305348894653861e-01 + -3.301549338737799e-01 + -3.297755091265357e-01 + -3.293966153413448e-01 + -3.290182526215280e-01 + -3.286404211114115e-01 + -3.282631209285871e-01 + -3.278863521751846e-01 + -3.275101149642331e-01 + -3.271344093953196e-01 + -3.267592355617253e-01 + -3.263845935667566e-01 + -3.260104834866909e-01 + -3.256369053882101e-01 + -3.252638593756906e-01 + -3.248913455268053e-01 + -3.245193638907094e-01 + -3.241479145199382e-01 + -3.237769974872735e-01 + -3.234066128661293e-01 + -3.230367606622389e-01 + -3.226674409186036e-01 + -3.222986537192250e-01 + -3.219303990783447e-01 + -3.215626770146125e-01 + -3.211954875680534e-01 + -3.208288307426270e-01 + -3.204627065455645e-01 + -3.200971150001409e-01 + -3.197320561103191e-01 + -3.193675298795836e-01 + -3.190035363166080e-01 + -3.186400754009213e-01 + -3.182771471114187e-01 + -3.179147514432106e-01 + -3.175528883799994e-01 + -3.171915578921511e-01 + -3.168307599394173e-01 + -3.164704945026028e-01 + -3.161107615528752e-01 + -3.157515610223091e-01 + -3.153928928534279e-01 + -3.150347570035595e-01 + -3.146771534412893e-01 + -3.143200821026652e-01 + -3.139635429048980e-01 + -3.136075357713601e-01 + -3.132520606345795e-01 + -3.128971174301287e-01 + -3.125427060850263e-01 + -3.121888265003789e-01 + -3.118354785699559e-01 + -3.114826622209773e-01 + -3.111303773636806e-01 + -3.107786238863177e-01 + -3.104274016699458e-01 + -3.100767106006651e-01 + -3.097265505731036e-01 + -3.093769214901858e-01 + -3.090278232197667e-01 + -3.086792556057311e-01 + -3.083312185528201e-01 + -3.079837119302323e-01 + -3.076367355606988e-01 + -3.072902893338402e-01 + -3.069443731087555e-01 + -3.065989866918071e-01 + -3.062541299629242e-01 + -3.059098027895394e-01 + -3.055660049841727e-01 + -3.052227363624680e-01 + -3.048799967605036e-01 + -3.045377860349815e-01 + -3.041961039937703e-01 + -3.038549504461246e-01 + -3.035143252370935e-01 + -3.031742281687436e-01 + -3.028346590287212e-01 + -3.024956176228422e-01 + -3.021571037727226e-01 + -3.018191172891879e-01 + -3.014816579447884e-01 + -3.011447255196590e-01 + -3.008083198102758e-01 + -3.004724406299011e-01 + -3.001370877400161e-01 + -2.998022608844934e-01 + -2.994679598576119e-01 + -2.991341844426006e-01 + -2.988009344054493e-01 + -2.984682095047237e-01 + -2.981360094999961e-01 + -2.978043341494513e-01 + -2.974731831997393e-01 + -2.971425563888731e-01 + -2.968124534589884e-01 + -2.964828741880918e-01 + -2.961538183196636e-01 + -2.958252855581221e-01 + -2.954972756291568e-01 + -2.951697882690829e-01 + -2.948428232168150e-01 + -2.945163801997789e-01 + -2.941904589349781e-01 + -2.938650591314966e-01 + -2.935401805013078e-01 + -2.932158227514122e-01 + -2.928919855851135e-01 + -2.925686687341852e-01 + -2.922458719001491e-01 + -2.919235947316832e-01 + -2.916018369508684e-01 + -2.912805982685433e-01 + -2.909598783275820e-01 + -2.906396768390501e-01 + -2.903199935181080e-01 + -2.900008280188033e-01 + -2.896821799916231e-01 + -2.893640491090957e-01 + -2.890464350819901e-01 + -2.887293375718574e-01 + -2.884127562142981e-01 + -2.880966906629407e-01 + -2.877811406014185e-01 + -2.874661057081589e-01 + -2.871515856017081e-01 + -2.868375799224188e-01 + -2.865240883310016e-01 + -2.862111104788733e-01 + -2.858986460098822e-01 + -2.855866945593247e-01 + -2.852752557499685e-01 + -2.849643292069009e-01 + -2.846539145659956e-01 + -2.843440114851778e-01 + -2.840346195818731e-01 + -2.837257384382189e-01 + -2.834173676812886e-01 + -2.831095069534547e-01 + -2.828021558855677e-01 + -2.824953140398479e-01 + -2.821889810170294e-01 + -2.818831564761255e-01 + -2.815778399984394e-01 + -2.812730311718642e-01 + -2.809687296216254e-01 + -2.806649349155874e-01 + -2.803616466281044e-01 + -2.800588643758425e-01 + -2.797565877594176e-01 + -2.794548163592918e-01 + -2.791535497376408e-01 + -2.788527874670974e-01 + -2.785525291296920e-01 + -2.782527743138438e-01 + -2.779535226034689e-01 + -2.776547735631448e-01 + -2.773565267276963e-01 + -2.770587816777879e-01 + -2.767615379983304e-01 + -2.764647952191276e-01 + -2.761685529026718e-01 + -2.758728106207560e-01 + -2.755775679024411e-01 + -2.752828243050798e-01 + -2.749885793975552e-01 + -2.746948327088052e-01 + -2.744015837838452e-01 + -2.741088321749212e-01 + -2.738165773969221e-01 + -2.735248189792139e-01 + -2.732335564721672e-01 + -2.729427894286903e-01 + -2.726525173657923e-01 + -2.723627397775934e-01 + -2.720734562117226e-01 + -2.717846661993187e-01 + -2.714963692431568e-01 + -2.712085648712142e-01 + -2.709212526021956e-01 + -2.706344319386959e-01 + -2.703481024077038e-01 + -2.700622635195931e-01 + -2.697769147573882e-01 + -2.694920556414950e-01 + -2.692076856844960e-01 + -2.689238043678598e-01 + -2.686404111894522e-01 + -2.683575056539274e-01 + -2.680750872618156e-01 + -2.677931554956217e-01 + -2.675117098420171e-01 + -2.672307498062658e-01 + -2.669502748541435e-01 + -2.666702844564393e-01 + -2.663907781339158e-01 + -2.661117553567487e-01 + -2.658332155842312e-01 + -2.655551583220328e-01 + -2.652775830228958e-01 + -2.650004891310402e-01 + -2.647238761635896e-01 + -2.644477435849875e-01 + -2.641720908275026e-01 + -2.638969173638899e-01 + -2.636222226679477e-01 + -2.633480062028563e-01 + -2.630742674151484e-01 + -2.628010057675256e-01 + -2.625282207270022e-01 + -2.622559117157733e-01 + -2.619840781930577e-01 + -2.617127196438671e-01 + -2.614418354712815e-01 + -2.611714251171109e-01 + -2.609014880694051e-01 + -2.606320237300626e-01 + -2.603630315312080e-01 + -2.600945109576975e-01 + -2.598264614192900e-01 + -2.595588823272051e-01 + -2.592917731268140e-01 + -2.590251332515323e-01 + -2.587589621335352e-01 + -2.584932592080277e-01 + -2.582280238899204e-01 + -2.579632555996659e-01 + -2.576989537753064e-01 + -2.574351178135512e-01 + -2.571717471205571e-01 + -2.569088411498421e-01 + -2.566463993032694e-01 + -2.563844209736399e-01 + -2.561229055922761e-01 + -2.558618525664796e-01 + -2.556012613001373e-01 + -2.553411312228936e-01 + -2.550814617199114e-01 + -2.548222521633088e-01 + -2.545635019702024e-01 + -2.543052105636615e-01 + -2.540473773489130e-01 + -2.537900016856341e-01 + -2.535330829668063e-01 + -2.532766206092137e-01 + -2.530206140025920e-01 + -2.527650625373169e-01 + -2.525099656058339e-01 + -2.522553225882911e-01 + -2.520011328519368e-01 + -2.517473957689335e-01 + -2.514941107682424e-01 + -2.512412772287477e-01 + -2.509888944763962e-01 + -2.507369619085763e-01 + -2.504854789235316e-01 + -2.502344448942996e-01 + -2.499838591881081e-01 + -2.497337211675947e-01 + -2.494840301941274e-01 + -2.492347856449565e-01 + -2.489859868896926e-01 + -2.487376332816026e-01 + -2.484897241934407e-01 + -2.482422589923619e-01 + -2.479952370263670e-01 + -2.477486576676466e-01 + -2.475025202765806e-01 + -2.472568241776227e-01 + -2.470115687463755e-01 + -2.467667533575219e-01 + -2.465223773319763e-01 + -2.462784400223504e-01 + -2.460349407926405e-01 + -2.457918789841980e-01 + -2.455492539346369e-01 + -2.453070649885519e-01 + -2.450653115068263e-01 + -2.448239928212929e-01 + -2.445831082525867e-01 + -2.443426571517872e-01 + -2.441026388729581e-01 + -2.438630527595843e-01 + -2.436238981283992e-01 + -2.433851743000419e-01 + -2.431468806082007e-01 + -2.429090164035606e-01 + -2.426715810065056e-01 + -2.424345737214870e-01 + -2.421979939156839e-01 + -2.419618409122474e-01 + -2.417261139893861e-01 + -2.414908125047008e-01 + -2.412559357926221e-01 + -2.410214831428485e-01 + -2.407874538901436e-01 + -2.405538473594189e-01 + -2.403206628448788e-01 + -2.400878996494245e-01 + -2.398555571012901e-01 + -2.396236345489386e-01 + -2.393921312730178e-01 + -2.391610465589564e-01 + -2.389303797400675e-01 + -2.387001301206918e-01 + -2.384702970040557e-01 + -2.382408797142012e-01 + -2.380118775445049e-01 + -2.377832897784107e-01 + -2.375551157141706e-01 + -2.373273546676602e-01 + -2.371000059435911e-01 + -2.368730688080822e-01 + -2.366465425799626e-01 + -2.364204265795865e-01 + -2.361947200488833e-01 + -2.359694222928865e-01 + -2.357445326362645e-01 + -2.355200503199650e-01 + -2.352959746331166e-01 + -2.350723049000333e-01 + -2.348490404054174e-01 + -2.346261804119323e-01 + -2.344037241869703e-01 + -2.341816710532240e-01 + -2.339600202844186e-01 + -2.337387711161095e-01 + -2.335179228570654e-01 + -2.332974748051559e-01 + -2.330774262250223e-01 + -2.328577763695832e-01 + -2.326385245095027e-01 + -2.324196699339893e-01 + -2.322012119071908e-01 + -2.319831496985562e-01 + -2.317654825913850e-01 + -2.315482098501277e-01 + -2.313313307350805e-01 + -2.311148445095759e-01 + -2.308987504304352e-01 + -2.306830477605995e-01 + -2.304677357746768e-01 + -2.302528137313455e-01 + -2.300382808821649e-01 + -2.298241364802842e-01 + -2.296103797849810e-01 + -2.293970100645507e-01 + -2.291840265929282e-01 + -2.289714285853380e-01 + -2.287592152687787e-01 + -2.285473859575702e-01 + -2.283359398767679e-01 + -2.281248762354934e-01 + -2.279141943457151e-01 + -2.277038934347241e-01 + -2.274939727028900e-01 + -2.272844314557521e-01 + -2.270752689260884e-01 + -2.268664843063303e-01 + -2.266580768713468e-01 + -2.264500458690802e-01 + -2.262423905203003e-01 + -2.260351100742427e-01 + -2.258282037663114e-01 + -2.256216708189812e-01 + -2.254155104759721e-01 + -2.252097219699820e-01 + -2.250043045201664e-01 + -2.247992573593203e-01 + -2.245945797283117e-01 + -2.243902708632074e-01 + -2.241863299547213e-01 + -2.239827562297785e-01 + -2.237795489631809e-01 + -2.235767073479459e-01 + -2.233742305913306e-01 + -2.231721179508258e-01 + -2.229703686190215e-01 + -2.227689817967141e-01 + -2.225679567331226e-01 + -2.223672926389324e-01 + -2.221669887137792e-01 + -2.219670441735496e-01 + -2.217674582593449e-01 + -2.215682301863681e-01 + -2.213693591068690e-01 + -2.211708442449674e-01 + -2.209726848493868e-01 + -2.207748801280919e-01 + -2.205774292618204e-01 + -2.203803314334422e-01 + -2.201835858632719e-01 + -2.199871917828554e-01 + -2.197911484006530e-01 + -2.195954548584606e-01 + -2.194001103587555e-01 + -2.192051141421317e-01 + -2.190104654040938e-01 + -2.188161633272981e-01 + -2.186222070944807e-01 + -2.184285959003400e-01 + -2.182353289399314e-01 + -2.180424054040691e-01 + -2.178498244760091e-01 + -2.176575853577229e-01 + -2.174656872563868e-01 + -2.172741293153170e-01 + -2.170829107184905e-01 + -2.168920306960583e-01 + -2.167014883907840e-01 + -2.165112829716549e-01 + -2.163214136638050e-01 + -2.161318796380734e-01 + -2.159426800456078e-01 + -2.157538140457635e-01 + -2.155652808465100e-01 + -2.153770796378261e-01 + -2.151892095617934e-01 + -2.150016698096599e-01 + -2.148144595574814e-01 + -2.146275779264766e-01 + -2.144410241083473e-01 + -2.142547972973645e-01 + -2.140688966286696e-01 + -2.138833212713278e-01 + -2.136980704068839e-01 + -2.135131431962469e-01 + -2.133285387811797e-01 + -2.131442563097479e-01 + -2.129602949644751e-01 + -2.127766538888946e-01 + -2.125933322205115e-01 + -2.124103291538361e-01 + -2.122276438425608e-01 + -2.120452754131807e-01 + -2.118632230196053e-01 + -2.116814858091140e-01 + -2.115000629262062e-01 + -2.113189535366684e-01 + -2.111381567835468e-01 + -2.109576717897184e-01 + -2.107774976942802e-01 + -2.105976336477213e-01 + -2.104180788038142e-01 + -2.102388322993181e-01 + -2.100598932684242e-01 + -2.098812608470688e-01 + -2.097029341689293e-01 + -2.095249123662445e-01 + -2.093471945702010e-01 + -2.091697799107427e-01 + -2.089926675162990e-01 + -2.088158565139439e-01 + -2.086393460309292e-01 + -2.084631351904399e-01 + -2.082872231124364e-01 + -2.081116089361562e-01 + -2.079362917906518e-01 + -2.077612707758083e-01 + -2.075865450013143e-01 + -2.074121135936494e-01 + -2.072379756933602e-01 + -2.070641303864229e-01 + -2.068905767613353e-01 + -2.067173139701657e-01 + -2.065443411403108e-01 + -2.063716573678470e-01 + -2.061992617242399e-01 + -2.060271533191409e-01 + -2.058553312792487e-01 + -2.056837947057731e-01 + -2.055125426973447e-01 + -2.053415743575506e-01 + -2.051708887985432e-01 + -2.050004850901530e-01 + -2.048303622987561e-01 + -2.046605195888223e-01 + -2.044909560501713e-01 + -2.043216707137636e-01 + -2.041526627003059e-01 + -2.039839311053935e-01 + -2.038154749924674e-01 + -2.036472934779214e-01 + -2.034793856435669e-01 + -2.033117505334730e-01 + -2.031443872603400e-01 + -2.029772949053144e-01 + -2.028104725040109e-01 + -2.026439191805672e-01 + -2.024776340229360e-01 + -2.023116160442759e-01 + -2.021458643362796e-01 + -2.019803779931268e-01 + -2.018151560674621e-01 + -2.016501976152565e-01 + -2.014855017030598e-01 + -2.013210674074579e-01 + -2.011568937830132e-01 + -2.009929798869589e-01 + -2.008293247967324e-01 + -2.006659275530022e-01 + -2.005027871933542e-01 + -2.003399027889975e-01 + -2.001772733993469e-01 + -2.000148980648089e-01 + -1.998527758076494e-01 + -1.996909056937942e-01 + -1.995292867991256e-01 + -1.993679181495860e-01 + -1.992067987609143e-01 + -1.990459276660739e-01 + -1.988853039482369e-01 + -1.987249266385540e-01 + -1.985647947339026e-01 + -1.984049072747179e-01 + -1.982452632992336e-01 + -1.980858618372365e-01 + -1.979267019229292e-01 + -1.977677825816225e-01 + -1.976091028269504e-01 + -1.974506616652059e-01 + -1.972924581269546e-01 + -1.971344912595772e-01 + -1.969767600601777e-01 + -1.968192635197257e-01 + -1.966620006473419e-01 + -1.965049704914111e-01 + -1.963481720619957e-01 + -1.961916043159508e-01 + -1.960352662866058e-01 + -1.958791570023236e-01 + -1.957232754408347e-01 + -1.955676205736371e-01 + -1.954121914050787e-01 + -1.952569869811404e-01 + -1.951020062511957e-01 + -1.949472481743376e-01 + -1.947927118000400e-01 + -1.946383960967118e-01 + -1.944843000224015e-01 + -1.943304226010136e-01 + -1.941767627996486e-01 + -1.940233195719773e-01 + -1.938700919198527e-01 + -1.937170788011597e-01 + -1.935642791691659e-01 + -1.934116920404960e-01 + -1.932593163702813e-01 + -1.931071510855982e-01 + -1.929551951739442e-01 + -1.928034476111422e-01 + -1.926519073523142e-01 + -1.925005733427448e-01 + -1.923494445333268e-01 + -1.921985198850023e-01 + -1.920477983694047e-01 + -1.918972789099489e-01 + -1.917469604078139e-01 + -1.915968418647559e-01 + -1.914469222323946e-01 + -1.912972003977130e-01 + -1.911476753153419e-01 + -1.909983459354311e-01 + -1.908492111828614e-01 + -1.907002699990133e-01 + -1.905515213021572e-01 + -1.904029639829450e-01 + -1.902545969863778e-01 + -1.901064192518998e-01 + -1.899584296832387e-01 + -1.898106271909556e-01 + -1.896630106872466e-01 + -1.895155790805884e-01 + -1.893683312769490e-01 + -1.892212661839438e-01 + -1.890743827110565e-01 + -1.889276797429612e-01 + -1.887811561730439e-01 + -1.886348109311961e-01 + -1.884886428944042e-01 + -1.883426509295119e-01 + -1.881968339481026e-01 + -1.880511908244114e-01 + -1.879057204210790e-01 + -1.877604216379617e-01 + -1.876152933660684e-01 + -1.874703344759240e-01 + -1.873255438107255e-01 + -1.871809202288037e-01 + -1.870364626111082e-01 + -1.868921698587295e-01 + -1.867480408184902e-01 + -1.866040743068453e-01 + -1.864602692045282e-01 + -1.863166243684211e-01 + -1.861731386269271e-01 + -1.860298108368848e-01 + -1.858866398430710e-01 + -1.857436244746902e-01 + -1.856007635808946e-01 + -1.854580560069532e-01 + -1.853155005825300e-01 + -1.851730961153894e-01 + -1.850308414342497e-01 + -1.848887353926573e-01 + -1.847467767859671e-01 + -1.846049644137440e-01 + -1.844632971091011e-01 + -1.843217736893654e-01 + -1.841803929497542e-01 + -1.840391536710132e-01 + -1.838980546953601e-01 + -1.837570948428414e-01 + -1.836162728527662e-01 + -1.834755875142904e-01 + -1.833350376404768e-01 + -1.831946220322505e-01 + -1.830543394615971e-01 + -1.829141886951903e-01 + -1.827741685199023e-01 + -1.826342777082210e-01 + -1.824945150204152e-01 + -1.823548792146146e-01 + -1.822153690716325e-01 + -1.820759833652730e-01 + -1.819367208114371e-01 + -1.817975801678411e-01 + -1.816585602158104e-01 + -1.815196596908165e-01 + -1.813808773333636e-01 + -1.812422118861797e-01 + -1.811036620617022e-01 + -1.809652266051679e-01 + -1.808269042761388e-01 + -1.806886937510042e-01 + -1.805505937426575e-01 + -1.804126030135119e-01 + -1.802747202867512e-01 + -1.801369442562599e-01 + -1.799992736029428e-01 + -1.798617070428147e-01 + -1.797242432859927e-01 + -1.795868810229852e-01 + -1.794496189531687e-01 + -1.793124557648680e-01 + -1.791753901302353e-01 + -1.790384207459855e-01 + -1.789015462957270e-01 + -1.787647654308534e-01 + -1.786280768266134e-01 + -1.784914791546937e-01 + -1.783549710639760e-01 + -1.782185512276328e-01 + -1.780822183108111e-01 + -1.779459709418989e-01 + -1.778098077766785e-01 + -1.776737274641330e-01 + -1.775377286080068e-01 + -1.774018098685905e-01 + -1.772659699092733e-01 + -1.771302073149079e-01 + -1.769945207098710e-01 + -1.768589087303928e-01 + -1.767233699571559e-01 + -1.765879030229077e-01 + -1.764525065751527e-01 + -1.763171791703831e-01 + -1.761819193770276e-01 + -1.760467257991546e-01 + -1.759115970764966e-01 + -1.757765317826701e-01 + -1.756415284467932e-01 + -1.755065856823099e-01 + -1.753717020824757e-01 + -1.752368761936218e-01 + -1.751021065398607e-01 + -1.749673916979465e-01 + -1.748327302843573e-01 + -1.746981207950505e-01 + -1.745635617515470e-01 + -1.744290517410085e-01 + -1.742945892830799e-01 + -1.741601728915786e-01 + -1.740258011048612e-01 + -1.738914724532109e-01 + -1.737571854631234e-01 + -1.736229386541751e-01 + -1.734887304969607e-01 + -1.733545594900176e-01 + -1.732204242005106e-01 + -1.730863230847530e-01 + -1.729522545880760e-01 + -1.728182172369972e-01 + -1.726842095169024e-01 + -1.725502298924380e-01 + -1.724162768405715e-01 + -1.722823488297978e-01 + -1.721484443102564e-01 + -1.720145617082366e-01 + -1.718806995062888e-01 + -1.717468561876117e-01 + -1.716130301380271e-01 + -1.714792197950507e-01 + -1.713454236195253e-01 + -1.712116399980747e-01 + -1.710778673588311e-01 + -1.709441041583024e-01 + -1.708103487974718e-01 + -1.706765996579921e-01 + -1.705428551238503e-01 + -1.704091136122197e-01 + -1.702753735243250e-01 + -1.701416332337174e-01 + -1.700078910979654e-01 + -1.698741454939726e-01 + -1.697403948188084e-01 + -1.696066374424440e-01 + -1.694728717024124e-01 + -1.693390959182572e-01 + -1.692053084687103e-01 + -1.690715077127793e-01 + -1.689376919605925e-01 + -1.688038595533757e-01 + -1.686700088249354e-01 + -1.685361380772680e-01 + -1.684022456043037e-01 + -1.682683297240474e-01 + -1.681343887869522e-01 + -1.680004210524648e-01 + -1.678664247764668e-01 + -1.677323982879569e-01 + -1.675983398888628e-01 + -1.674642478421112e-01 + -1.673301203727337e-01 + -1.671959557470875e-01 + -1.670617522514179e-01 + -1.669275081523977e-01 + -1.667932216748827e-01 + -1.666588910302535e-01 + -1.665245144618999e-01 + -1.663900902261115e-01 + -1.662556165573334e-01 + -1.661210916150787e-01 + -1.659865136192500e-01 + -1.658518808205523e-01 + -1.657171913793817e-01 + -1.655824434869752e-01 + -1.654476353589082e-01 + -1.653127651402884e-01 + -1.651778310039089e-01 + -1.650428311475997e-01 + -1.649077636947325e-01 + -1.647726268054878e-01 + -1.646374186795111e-01 + -1.645021374143022e-01 + -1.643667811219276e-01 + -1.642313479645375e-01 + -1.640958360797542e-01 + -1.639602435821208e-01 + -1.638245685678920e-01 + -1.636888091342957e-01 + -1.635529633807896e-01 + -1.634170294056715e-01 + -1.632810052843662e-01 + -1.631448890987297e-01 + -1.630086789505001e-01 + -1.628723728914803e-01 + -1.627359689604526e-01 + -1.625994652175857e-01 + -1.624628597296842e-01 + -1.623261505411777e-01 + -1.621893356498993e-01 + -1.620524130844338e-01 + -1.619153808874262e-01 + -1.617782370829140e-01 + -1.616409796594323e-01 + -1.615036066042981e-01 + -1.613661159511956e-01 + -1.612285056749289e-01 + -1.610907737177844e-01 + -1.609529180644884e-01 + -1.608149367058714e-01 + -1.606768276166446e-01 + -1.605385887249777e-01 + -1.604002179666082e-01 + -1.602617132920370e-01 + -1.601230726507729e-01 + -1.599842939686116e-01 + -1.598453751511857e-01 + -1.597063141136520e-01 + -1.595671087574746e-01 + -1.594277569704320e-01 + -1.592882566576958e-01 + -1.591486057239582e-01 + -1.590088020577491e-01 + -1.588688434990599e-01 + -1.587287278927537e-01 + -1.585884531125515e-01 + -1.584480170419727e-01 + -1.583074175205078e-01 + -1.581666523320080e-01 + -1.580257193426469e-01 + -1.578846164090064e-01 + -1.577433413163997e-01 + -1.576018918624315e-01 + -1.574602658488709e-01 + -1.573184610699215e-01 + -1.571764753211041e-01 + -1.570343063929105e-01 + -1.568919520623428e-01 + -1.567494100904384e-01 + -1.566066782348808e-01 + -1.564637542612438e-01 + -1.563206358983480e-01 + -1.561773208679784e-01 + -1.560338069324230e-01 + -1.558900918260982e-01 + -1.557461732592160e-01 + -1.556020489404925e-01 + -1.554577165574562e-01 + -1.553131737930803e-01 + -1.551684183667380e-01 + -1.550234479797016e-01 + -1.548782603038531e-01 + -1.547328529851514e-01 + -1.545872236759721e-01 + -1.544413700402710e-01 + -1.542952897416497e-01 + -1.541489804232354e-01 + -1.540024397056934e-01 + -1.538556652055174e-01 + -1.537086545589509e-01 + -1.535614054107616e-01 + -1.534139153265976e-01 + -1.532661818894184e-01 + -1.531182027303390e-01 + -1.529699754364569e-01 + -1.528214975805129e-01 + -1.526727667337492e-01 + -1.525237804315604e-01 + -1.523745362382714e-01 + -1.522250317710374e-01 + -1.520752645246755e-01 + -1.519252320010430e-01 + -1.517749318043420e-01 + -1.516243614355856e-01 + -1.514735183783821e-01 + -1.513224001894709e-01 + -1.511710043669709e-01 + -1.510193283881065e-01 + -1.508673697644896e-01 + -1.507151259726486e-01 + -1.505625944726656e-01 + -1.504097727456887e-01 + -1.502566582737337e-01 + -1.501032485283334e-01 + -1.499495409558064e-01 + -1.497955329817901e-01 + -1.496412220312461e-01 + -1.494866055649385e-01 + -1.493316810218445e-01 + -1.491764458131949e-01 + -1.490208973395277e-01 + -1.488650330073115e-01 + -1.487088502321930e-01 + -1.485523464347014e-01 + -1.483955190026080e-01 + -1.482383652905619e-01 + -1.480808826638546e-01 + -1.479230685198502e-01 + -1.477649202713839e-01 + -1.476064352431592e-01 + -1.474476107543291e-01 + -1.472884441602761e-01 + -1.471289328556787e-01 + -1.469690741904771e-01 + -1.468088654425319e-01 + -1.466483039513407e-01 + -1.464873870643751e-01 + -1.463261120992439e-01 + -1.461644763685366e-01 + -1.460024771711030e-01 + -1.458401117873900e-01 + -1.456773775256328e-01 + -1.455142717039091e-01 + -1.453507916253095e-01 + -1.451869345373153e-01 + -1.450226976893540e-01 + -1.448580783981553e-01 + -1.446930739597145e-01 + -1.445276816274984e-01 + -1.443618986028185e-01 + -1.441957221635531e-01 + -1.440291496154044e-01 + -1.438621781832398e-01 + -1.436948050802729e-01 + -1.435270275400491e-01 + -1.433588428498931e-01 + -1.431902482406432e-01 + -1.430212409022274e-01 + -1.428518180756519e-01 + -1.426819769869911e-01 + -1.425117148424586e-01 + -1.423410288662751e-01 + -1.421699162875700e-01 + -1.419983743276740e-01 + -1.418264001727286e-01 + -1.416539910114460e-01 + -1.414811440470098e-01 + -1.413078564852965e-01 + -1.411341255423312e-01 + -1.409599484327010e-01 + -1.407853222965617e-01 + -1.406102443173973e-01 + -1.404347117561954e-01 + -1.402587217519092e-01 + -1.400822714659246e-01 + -1.399053581557436e-01 + -1.397279789611117e-01 + -1.395501310267636e-01 + -1.393718115937786e-01 + -1.391930178402762e-01 + -1.390137469173205e-01 + -1.388339959972602e-01 + -1.386537622953096e-01 + -1.384730430031857e-01 + -1.382918352262208e-01 + -1.381101361784074e-01 + -1.379279430940128e-01 + -1.377452530885734e-01 + -1.375620633453186e-01 + -1.373783710804598e-01 + -1.371941734475252e-01 + -1.370094676273606e-01 + -1.368242508215203e-01 + -1.366385202106050e-01 + -1.364522729969404e-01 + -1.362655063841710e-01 + -1.360782175127518e-01 + -1.358904035964053e-01 + -1.357020618930480e-01 + -1.355131895302153e-01 + -1.353237837054462e-01 + -1.351338416922692e-01 + -1.349433606493001e-01 + -1.347523377766847e-01 + -1.345607703467432e-01 + -1.343686555800670e-01 + -1.341759906744941e-01 + -1.339827728348106e-01 + -1.337889993407819e-01 + -1.335946674413715e-01 + -1.333997743261954e-01 + -1.332043172962129e-01 + -1.330082936252972e-01 + -1.328117004983506e-01 + -1.326145352418741e-01 + -1.324167951779929e-01 + -1.322184775207451e-01 + -1.320195795498869e-01 + -1.318200985870884e-01 + -1.316200319613715e-01 + -1.314193769770063e-01 + -1.312181309395709e-01 + -1.310162911860090e-01 + -1.308138550466099e-01 + -1.306108198680959e-01 + -1.304071830473609e-01 + -1.302029419358534e-01 + -1.299980938748928e-01 + -1.297926362720222e-01 + -1.295865665431496e-01 + -1.293798820977739e-01 + -1.291725803345192e-01 + -1.289646586840201e-01 + -1.287561146028010e-01 + -1.285469455426532e-01 + -1.283371489741077e-01 + -1.281267223855238e-01 + -1.279156632656401e-01 + -1.277039691180485e-01 + -1.274916374652222e-01 + -1.272786658468367e-01 + -1.270650518164191e-01 + -1.268507929437779e-01 + -1.266358868257778e-01 + -1.264203310530910e-01 + -1.262041232080785e-01 + -1.259872609212462e-01 + -1.257697418614985e-01 + -1.255515637216261e-01 + -1.253327241582721e-01 + -1.251132208452764e-01 + -1.248930515084944e-01 + -1.246722139029198e-01 + -1.244507057917270e-01 + -1.242285249387353e-01 + -1.240056691417677e-01 + -1.237821362222557e-01 + -1.235579240163024e-01 + -1.233330303641722e-01 + -1.231074531412701e-01 + -1.228811902909497e-01 + -1.226542397328350e-01 + -1.224265993926709e-01 + -1.221982672554952e-01 + -1.219692413178169e-01 + -1.217395195880366e-01 + -1.215091001037845e-01 + -1.212779809466277e-01 + -1.210461602256936e-01 + -1.208136360512357e-01 + -1.205804065871132e-01 + -1.203464700244819e-01 + -1.201118245085727e-01 + -1.198764682774020e-01 + -1.196403996473917e-01 + -1.194036168896808e-01 + -1.191661183059585e-01 + -1.189279022478277e-01 + -1.186889671039581e-01 + -1.184493112863093e-01 + -1.182089332317922e-01 + -1.179678314226111e-01 + -1.177260043675167e-01 + -1.174834505981953e-01 + -1.172401686820595e-01 + -1.169961572351099e-01 + -1.167514149230550e-01 + -1.165059404117411e-01 + -1.162597324012717e-01 + -1.160127896494247e-01 + -1.157651109471359e-01 + -1.155166951180050e-01 + -1.152675410245315e-01 + -1.150176475647009e-01 + -1.147670136740031e-01 + -1.145156383362403e-01 + -1.142635205942373e-01 + -1.140106595167702e-01 + -1.137570541746248e-01 + -1.135027037005393e-01 + -1.132476072885310e-01 + -1.129917641930528e-01 + -1.127351736954119e-01 + -1.124778350964029e-01 + -1.122197477241451e-01 + -1.119609110122396e-01 + -1.117013244465126e-01 + -1.114409874638995e-01 + -1.111798996306716e-01 + -1.109180606010562e-01 + -1.106554699641756e-01 + -1.103921273860412e-01 + -1.101280326261970e-01 + -1.098631854961093e-01 + -1.095975858385727e-01 + -1.093312335328026e-01 + -1.090641285282797e-01 + -1.087962708201439e-01 + -1.085276604456853e-01 + -1.082582975002112e-01 + -1.079881821487157e-01 + -1.077173146235393e-01 + -1.074456951780328e-01 + -1.071733241222459e-01 + -1.069002018428373e-01 + -1.066263287812705e-01 + -1.063517054311524e-01 + -1.060763323447030e-01 + -1.058002101349244e-01 + -1.055233394662503e-01 + -1.052457210591484e-01 + -1.049673557111256e-01 + -1.046882442747131e-01 + -1.044083876525769e-01 + -1.041277868396362e-01 + -1.038464428925467e-01 + -1.035643569070283e-01 + -1.032815300276830e-01 + -1.029979634724506e-01 + -1.027136585692167e-01 + -1.024286166734134e-01 + -1.021428391952729e-01 + -1.018563276648054e-01 + -1.015690836461885e-01 + -1.012811087487812e-01 + -1.009924046817547e-01 + -1.007029732152899e-01 + -1.004128161832720e-01 + -1.001219355157012e-01 + -9.983033318947014e-02 + -9.953801123789381e-02 + -9.924497181065381e-02 + -9.895121710675164e-02 + -9.865674937610269e-02 + -9.836157098451072e-02 + -9.806568435550449e-02 + -9.776909196067303e-02 + -9.747179635418697e-02 + -9.717380018935569e-02 + -9.687510621844479e-02 + -9.657571724434684e-02 + -9.627563612429342e-02 + -9.597486578738405e-02 + -9.567340928055243e-02 + -9.537126972794066e-02 + -9.506845031433311e-02 + -9.476495428226560e-02 + -9.446078497156121e-02 + -9.415594584893169e-02 + -9.385044042616277e-02 + -9.354427227181475e-02 + -9.323744504129654e-02 + -9.292996250412121e-02 + -9.262182850819620e-02 + -9.231304696021701e-02 + -9.200362185853247e-02 + -9.169355729461479e-02 + -9.138285746240042e-02 + -9.107152658866502e-02 + -9.075956898041553e-02 + -9.044698911371001e-02 + -9.013379148226228e-02 + -8.981998063663986e-02 + -8.950556130228651e-02 + -8.919053824746566e-02 + -8.887491629233073e-02 + -8.855870039130662e-02 + -8.824189556852820e-02 + -8.792450692262105e-02 + -8.760653968435946e-02 + -8.728799913331863e-02 + -8.696889060279582e-02 + -8.664921956673162e-02 + -8.632899159487925e-02 + -8.600821233560418e-02 + -8.568688750540578e-02 + -8.536502290181033e-02 + -8.504262441718942e-02 + -8.471969805751253e-02 + -8.439624991228166e-02 + -8.407228614692197e-02 + -8.374781300392073e-02 + -8.342283682562317e-02 + -8.309736406563741e-02 + -8.277140122021633e-02 + -8.244495489588025e-02 + -8.211803185128563e-02 + -8.179063884910803e-02 + -8.146278271646745e-02 + -8.113447041766966e-02 + -8.080570906396743e-02 + -8.047650581443898e-02 + -8.014686781276095e-02 + -7.981680240792462e-02 + -7.948631704884632e-02 + -7.915541916537581e-02 + -7.882411634928695e-02 + -7.849241630024972e-02 + -7.816032674199919e-02 + -7.782785552351333e-02 + -7.749501058768483e-02 + -7.716179991262426e-02 + -7.682823159405887e-02 + -7.649431382724470e-02 + -7.616005485767063e-02 + -7.582546305035900e-02 + -7.549054686796573e-02 + -7.515531479419155e-02 + -7.481977542018518e-02 + -7.448393744841550e-02 + -7.414780964186224e-02 + -7.381140084542565e-02 + -7.347471999131863e-02 + -7.313777608894645e-02 + -7.280057821330421e-02 + -7.246313551410177e-02 + -7.212545725749871e-02 + -7.178755277686714e-02 + -7.144943146088469e-02 + -7.111110278446059e-02 + -7.077257629326426e-02 + -7.043386160590276e-02 + -7.009496843792977e-02 + -6.975590657859251e-02 + -6.941668588038064e-02 + -6.907731623736572e-02 + -6.873780762436528e-02 + -6.839817013263073e-02 + -6.805841390025939e-02 + -6.771854911509186e-02 + -6.737858603607710e-02 + -6.703853501790490e-02 + -6.669840646610939e-02 + -6.635821080020089e-02 + -6.601795854049251e-02 + -6.567766029345612e-02 + -6.533732672933422e-02 + -6.499696856244534e-02 + -6.465659655384894e-02 + -6.431622152906083e-02 + -6.397585437117004e-02 + -6.363550601749234e-02 + -6.329518746487584e-02 + -6.295490977261127e-02 + -6.261468405232740e-02 + -6.227452144919842e-02 + -6.193443314782195e-02 + -6.159443038613483e-02 + -6.125452449175782e-02 + -6.091472682299209e-02 + -6.057504875634485e-02 + -6.023550170757411e-02 + -5.989609715178513e-02 + -5.955684662483948e-02 + -5.921776169220818e-02 + -5.887885392543541e-02 + -5.854013491232313e-02 + -5.820161634720029e-02 + -5.786330995026137e-02 + -5.752522743033200e-02 + -5.718738052319666e-02 + -5.684978102238282e-02 + -5.651244078415603e-02 + -5.617537160914179e-02 + -5.583858533460867e-02 + -5.550209389764803e-02 + -5.516590918758542e-02 + -5.483004310132478e-02 + -5.449450760284144e-02 + -5.415931467224975e-02 + -5.382447627285256e-02 + -5.349000433169809e-02 + -5.315591089010736e-02 + -5.282220800586091e-02 + -5.248890761304951e-02 + -5.215602173570349e-02 + -5.182356244333215e-02 + -5.149154173522678e-02 + -5.115997163165661e-02 + -5.082886416999516e-02 + -5.049823136495267e-02 + -5.016808523392447e-02 + -4.983843779429676e-02 + -4.950930104255546e-02 + -4.918068697018742e-02 + -4.885260756307724e-02 + -4.852507478656667e-02 + -4.819810059193260e-02 + -4.787169691654435e-02 + -4.754587567726797e-02 + -4.722064876717863e-02 + -4.689602805511645e-02 + -4.657202539266345e-02 + -4.624865260469157e-02 + -4.592592148337583e-02 + -4.560384379218360e-02 + -4.528243126408893e-02 + -4.496169559628109e-02 + -4.464164845065374e-02 + -4.432230145577399e-02 + -4.400366620090135e-02 + -4.368575422918912e-02 + -4.336857704568598e-02 + -4.305214611378011e-02 + -4.273647284209545e-02 + -4.242156859535529e-02 + -4.210744469338823e-02 + -4.179411239819886e-02 + -4.148158291999365e-02 + -4.116986741259326e-02 + -4.085897697584740e-02 + -4.054892265423421e-02 + -4.023971542131075e-02 + -3.993136619382251e-02 + -3.962388583123071e-02 + -3.931728511726146e-02 + -3.901157477126550e-02 + -3.870676544906659e-02 + -3.840286772630143e-02 + -3.809989211312446e-02 + -3.779784905219891e-02 + -3.749674889469364e-02 + -3.719660192057603e-02 + -3.689741833904486e-02 + -3.659920826728452e-02 + -3.630198174507707e-02 + -3.600574873447752e-02 + -3.571051910326670e-02 + -3.541630263528623e-02 + -3.512310902944081e-02 + -3.483094789226868e-02 + -3.453982873906083e-02 + -3.424976098893085e-02 + -3.396075396966352e-02 + -3.367281691959212e-02 + -3.338595897696143e-02 + -3.310018917253339e-02 + -3.281551644593574e-02 + -3.253194963945627e-02 + -3.224949747985180e-02 + -3.196816859917648e-02 + -3.168797153104005e-02 + -3.140891468854379e-02 + -3.113100638362195e-02 + -3.085425482400445e-02 + -3.057866810270934e-02 + -3.030425420554928e-02 + -3.003102099769914e-02 + -2.975897623664901e-02 + -2.948812757423692e-02 + -2.921848253324668e-02 + -2.895004852283206e-02 + -2.868283284398848e-02 + -2.841684266935389e-02 + -2.815208505554401e-02 + -2.788856694668657e-02 + -2.762629515277662e-02 + -2.736527636581735e-02 + -2.710551716378724e-02 + -2.684702399109470e-02 + -2.658980317107453e-02 + -2.633386090723251e-02 + -2.607920326420626e-02 + -2.582583618939347e-02 + -2.557376551170278e-02 + -2.532299691452088e-02 + -2.507353595904179e-02 + -2.482538808499682e-02 + -2.457855859507147e-02 + -2.433305266394129e-02 + -2.408887533435539e-02 + -2.384603151841623e-02 + -2.360452600145211e-02 + -2.336436343201402e-02 + -2.312554832609387e-02 + -2.288808507226996e-02 + -2.265197791904223e-02 + -2.241723098404469e-02 + -2.218384825955722e-02 + -2.195183359578144e-02 + -2.172119070564114e-02 + -2.149192317903017e-02 + -2.126403446983029e-02 + -2.103752788800763e-02 + -2.081240661670658e-02 + -2.058867370373213e-02 + -2.036633206151460e-02 + -2.014538447470562e-02 + -1.992583358220780e-02 + -1.970768189350454e-02 + -1.949093179354288e-02 + -1.927558551890179e-02 + -1.906164517346932e-02 + -1.884911273803516e-02 + -1.863799005561765e-02 + -1.842827883084014e-02 + -1.821998063919436e-02 + -1.801309692347624e-02 + -1.780762899421212e-02 + -1.760357803216378e-02 + -1.740094507853177e-02 + -1.719973104881908e-02 + -1.699993673354589e-02 + -1.680156278321953e-02 + -1.660460972117107e-02 + -1.640907794332531e-02 + -1.621496771146485e-02 + -1.602227916433384e-02 + -1.583101231371427e-02 + -1.564116703862817e-02 + -1.545274309883748e-02 + -1.526574012772238e-02 + -1.508015761914186e-02 + -1.489599495303444e-02 + -1.471325139301425e-02 + -1.453192606709004e-02 + -1.435201798658663e-02 + -1.417352604127694e-02 + -1.399644899327932e-02 + -1.382078549302728e-02 + -1.364653407164367e-02 + -1.347369313289915e-02 + -1.330226097046918e-02 + -1.313223576283522e-02 + -1.296361556501821e-02 + -1.279639832327015e-02 + -1.263058187022890e-02 + -1.246616392163480e-02 + -1.230314208589229e-02 + -1.214151385550256e-02 + -1.198127661271531e-02 + -1.182242763705999e-02 + -1.166496409538715e-02 + -1.150888304589847e-02 + -1.135418144553720e-02 + -1.120085614521435e-02 + -1.104890388968280e-02 + -1.089832132231912e-02 + -1.074910498405016e-02 + -1.060125131793449e-02 + -1.045475666967276e-02 + -1.030961728045836e-02 + -1.016582929759894e-02 + -1.002338877483435e-02 + -9.882291666582974e-03 + -9.742533838318660e-03 + -9.604111064390042e-03 + -9.467019020603698e-03 + -9.331253297647066e-03 + -9.196809399717920e-03 + -9.063682741005771e-03 + -8.931868652392608e-03 + -8.801362376326071e-03 + -8.672159070075210e-03 + -8.544253813557785e-03 + -8.417641603575709e-03 + -8.292317355716863e-03 + -8.168275911736944e-03 + -8.045512032694060e-03 + -7.924020397696789e-03 + -7.803795618017998e-03 + -7.684832236645861e-03 + -7.567124714929550e-03 + -7.450667444285828e-03 + -7.335454748033539e-03 + -7.221480883382279e-03 + -7.108740043647531e-03 + -6.997226348639058e-03 + -6.886933855315346e-03 + -6.777856561420043e-03 + -6.669988397445708e-03 + -6.563323236817328e-03 + -6.457854897474280e-03 + -6.353577131920331e-03 + -6.250483639666226e-03 + -6.148568068731946e-03 + -6.047824005897748e-03 + -5.948244987599076e-03 + -5.849824501347084e-03 + -5.752555978513681e-03 + -5.656432807264865e-03 + -5.561448332218353e-03 + -5.467595840626229e-03 + -5.374868578311233e-03 + -5.283259752176146e-03 + -5.192762521793490e-03 + -5.103370006626977e-03 + -5.015075286629148e-03 + -4.927871401782054e-03 + -4.841751355904269e-03 + -4.756708115524279e-03 + -4.672734611342882e-03 + -4.589823741835811e-03 + -4.507968371897898e-03 + -4.427161334111035e-03 + -4.347395432360992e-03 + -4.268663440821394e-03 + -4.190958105123732e-03 + -4.114272145237943e-03 + -4.038598254543495e-03 + -3.963929102325757e-03 + -3.890257335680627e-03 + -3.817575578228647e-03 + -3.745876433060482e-03 + -3.675152484340320e-03 + -3.605396296371936e-03 + -3.536600416225055e-03 + -3.468757375181376e-03 + -3.401859688514519e-03 + -3.335899857279008e-03 + -3.270870369624132e-03 + -3.206763701552144e-03 + -3.143572317880366e-03 + -3.081288673370167e-03 + -3.019905214426235e-03 + -2.959414379161780e-03 + -2.899808598243805e-03 + -2.841080297295089e-03 + -2.783221897187762e-03 + -2.726225814481816e-03 + -2.670084462721576e-03 + -2.614790253659214e-03 + -2.560335598217736e-03 + -2.506712907773368e-03 + -2.453914594305456e-03 + -2.401933071402304e-03 + -2.350760756079694e-03 + -2.300390068609935e-03 + -2.250813433830373e-03 + -2.202023282926124e-03 + -2.154012052729510e-03 + -2.106772187286276e-03 + -2.060296139752280e-03 + -2.014576371403607e-03 + -1.969605353508076e-03 + -1.925375568654703e-03 + -1.881879509874747e-03 + -1.839109683164610e-03 + -1.797058608044933e-03 + -1.755718816706420e-03 + -1.715082857166331e-03 + -1.675143293280333e-03 + -1.635892703443056e-03 + -1.597323683697497e-03 + -1.559428848712492e-03 + -1.522200830321536e-03 + -1.485632280183934e-03 + -1.449715870092584e-03 + -1.414444291041649e-03 + -1.379810256474448e-03 + -1.345806501791486e-03 + -1.312425783716085e-03 + -1.279660883530597e-03 + -1.247504606026809e-03 + -1.215949779407020e-03 + -1.184989258561609e-03 + -1.154615923413642e-03 + -1.124822679454977e-03 + -1.095602461024569e-03 + -1.066948228352973e-03 + -1.038852968864840e-03 + -1.011309701783447e-03 + -9.843114734261671e-04 + -9.578513585924095e-04 + -9.319224654553773e-04 + -9.065179311555053e-04 + -8.816309237062864e-04 + -8.572546450286300e-04 + -8.333823272254078e-04 + -8.100072355517002e-04 + -7.871226705798197e-04 + -7.647219641139264e-04 + -7.427984826268421e-04 + -7.213456295816899e-04 + -7.003568409622573e-04 + -6.798255887472549e-04 + -6.597453829223955e-04 + -6.401097674232178e-04 + -6.209123240167792e-04 + -6.021466729184949e-04 + -5.838064694784330e-04 + -5.658854088227745e-04 + -5.483772251938055e-04 + -5.312756891491387e-04 + -5.145746125261390e-04 + -4.982678466736966e-04 + -4.823492804447342e-04 + -4.668128456938979e-04 + -4.516525141842837e-04 + -4.368622962674093e-04 + -4.224362466667431e-04 + -4.083684605074392e-04 + -3.946530726767535e-04 + -3.812842634726816e-04 + -3.682562540846601e-04 + -3.555633067280622e-04 + -3.431997298953713e-04 + -3.311598733734528e-04 + -3.194381292167991e-04 + -3.080289365424697e-04 + -2.969267761950205e-04 + -2.861261725181284e-04 + -2.756216974519896e-04 + -2.654079650780054e-04 + -2.554796342192304e-04 + -2.458314117831762e-04 + -2.364580472261255e-04 + -2.273543358783364e-04 + -2.185151214599497e-04 + -2.099352907310207e-04 + -2.016097775283348e-04 + -1.935335642058985e-04 + -1.857016766078902e-04 + -1.781091887621722e-04 + -1.707512232636694e-04 + -1.636229467286454e-04 + -1.567195750846747e-04 + -1.500363727286114e-04 + -1.435686485603325e-04 + -1.373117617901810e-04 + -1.312611200233150e-04 + -1.254121759173047e-04 + -1.197604333131590e-04 + -1.143014443478538e-04 + -1.090308067994204e-04 + -1.039441703359566e-04 + -9.903723276260782e-05 + -9.430573818110019e-05 + -8.974548326970851e-05 + -8.535231263786576e-05 + -8.112211782233505e-05 + -7.705084341917009e-05 + -7.313448187442813e-05 + -6.936907333027644e-05 + -6.575071119249467e-05 + -6.227553658275769e-05 + -5.893973914900695e-05 + -5.573956203853484e-05 + -5.267129595161032e-05 + -4.973128084575291e-05 + -4.691591029739419e-05 + -4.422162539456759e-05 + -4.164491727590835e-05 + -3.918233061143914e-05 + -3.683045757249665e-05 + -3.458594118927672e-05 + -3.244547785270521e-05 + -3.040581149216269e-05 + -2.846373767254703e-05 + -2.661610506551807e-05 + -2.485980997915884e-05 + -2.319180110803831e-05 + -2.160907991664170e-05 + -2.010869565536285e-05 + -1.868775067119526e-05 + -1.734339969669114e-05 + -1.607284546059277e-05 + -1.487334444060605e-05 + -1.374220506933501e-05 + -1.267678404087566e-05 + -1.167449238525668e-05 + -1.073279264625808e-05 + -9.849195952118955e-06 + -9.021268250139948e-06 + -8.246626567968640e-06 + -7.522936912951058e-06 + -6.847920492700095e-06 + -6.219349168072015e-06 + -5.635044236568559e-06 + -5.092882470932048e-06 + -4.590790895044201e-06 + -4.126746487291455e-06 + -3.698781859286729e-06 + -3.304979520120749e-06 + -2.943472506918129e-06 + -2.612449510984778e-06 + -2.310148834845521e-06 + -2.034859929271865e-06 + -1.784927798002038e-06 + -1.558746837246077e-06 + -1.354763241038800e-06 + -1.171478551126456e-06 + -1.007443551831214e-06 + -8.612614901318561e-07 + -7.315906671893518e-07 + -6.171385461507109e-07 + -5.166657190198488e-07 + -4.289874671965282e-07 + -3.529682224298676e-07 + -2.875261969469119e-07 + -2.316338735426513e-07 + -1.843129441850586e-07 + -1.446395043479685e-07 + -1.117434618507405e-07 + -8.480406305106530e-08 + -6.305553831873507e-08 + -4.578544870958251e-08 + -3.233089380928512e-08 + -2.208448022054805e-08 + -1.449165750478591e-08 + -9.047683233008124e-09 + -5.303766739728543e-09 + -2.863504097386866e-09 + -1.380623394721897e-09 + -5.650641971230870e-10 + -1.790372475752245e-10 + -3.511990521548007e-11 + -8.192061809269277e-13 + 0.000000000000000e+00 diff --git a/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice new file mode 100644 index 0000000000..1d63f01d43 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_lattice @@ -0,0 +1,51 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin 2.2 0.0 0.0 1.0 +velocity all create 400 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 4.0 spin/neel 4.0 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.1 0.2171 1.841 +pair_coeff * * spin/neel neel 4.0 0.02 0.0 1.841 0.0 0.0 1.0 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin 200.0 200.0 10.0 48279 +fix 3 all langevin/spin 0.0 0.00001 321 +fix 4 all nve/spin lattice moving +timestep 0.001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 200000 diff --git a/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin new file mode 100644 index 0000000000..435e877bdf --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/in.spin.nvt_spin @@ -0,0 +1,49 @@ +# bcc iron in a 3d periodic box + +clear +units metal +atom_style spin + +dimension 3 +boundary p p p + +# necessary for the serial algorithm (sametag) +atom_modify map array + +lattice bcc 2.8665 +region box block 0.0 3.0 0.0 3.0 0.0 3.0 +create_box 1 box +create_atoms 1 box + +# setting mass, mag. moments, and interactions for bcc iron + +mass 1 55.845 +set group all spin 2.2 0.0 0.0 1.0 +velocity all create 0 4928459 rot yes dist gaussian + +pair_style hybrid/overlay eam/alloy spin/exchange 3.5 +pair_coeff * * eam/alloy Fe_Mishin2006.eam.alloy Fe +pair_coeff * * spin/exchange exchange 3.4 0.1 0.2171 1.841 + +neighbor 0.1 bin +neigh_modify every 10 check yes delay 20 + +fix 1 all precession/spin zeeman 0.0 0.0 0.0 1.0 +fix 2 all langevin/spin 200.0 0.1 321 +fix 3 all nve/spin lattice moving +timestep 0.001 + +# compute and output options + +compute out_mag all spin +compute out_pe all pe +compute out_ke all ke +compute out_temp all temp + +variable emag equal c_out_mag[5] +variable tmag equal c_out_mag[6] + +thermo_style custom step time v_tmag temp v_emag ke pe etotal +thermo 200 + +run 200000 diff --git a/examples/SPIN/test_problems/validation_nvt/plot_nvt.py b/examples/SPIN/test_problems/validation_nvt/plot_nvt.py new file mode 100755 index 0000000000..06c48b4c28 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/plot_nvt.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import numpy as np, pylab, tkinter +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +from decimal import * +import sys, string, os + + +argv = sys.argv +if len(argv) != 3: + print("Syntax: ./plot_precession.py res_nvt_spin.dat res_nvt_lattice.dat") + sys.exit() + +dirname = os.path.join(os.getcwd(), "Feb_07") +nvtspin_file = sys.argv[1] +nvtlatt_file = sys.argv[2] + +ts,tmags,temps = np.loadtxt(nvtspin_file,skiprows=0, usecols=(1,2,3),unpack=True) +tl,tmagl,templ = np.loadtxt(nvtlatt_file,skiprows=0, usecols=(1,2,3),unpack=True) + +fig = plt.figure(figsize=(8,8)) +ax1 = plt.subplot(2,1,1) +ax2 = plt.subplot(2,1,2) + +ax1.plot(ts, tmags, 'r-', label='Spin temp. (thermostat)') +ax1.plot(ts, temps, 'g-', label='Lattice temp.') +ax1.set_yscale("log") +ax1.set_ylabel("T (K)") +ax1.legend(loc=3) + +ax2.plot(tl, tmagl, 'r-', label='Spin temp.') +ax2.plot(tl, templ, 'g-', label='Lattice temp. (thermostat)') +ax2.set_yscale("log") +ax2.set_ylabel("T (K)") +ax2.legend(loc=3) + +plt.xlabel('Time (in ps)') +plt.legend() +plt.show() + +fig.savefig(os.path.join(os.getcwd(), "nve_spin_lattice.pdf"), bbox_inches="tight") +plt.close(fig) diff --git a/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh b/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh new file mode 100755 index 0000000000..d3abbbe1a5 --- /dev/null +++ b/examples/SPIN/test_problems/validation_nvt/run-test-nvt.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# clean old res +rm res_*.dat + +# compute NVT Spin -> Lattice +./../../../../src/lmp_serial -in in.spin.nvt_spin +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_nvt_spin.dat + +# compute NVT Lattice -> Spin +./../../../../src/lmp_serial -in in.spin.nvt_lattice +in="$(grep -n Step log.lammps | awk -F ':' '{print $1}')" +en="$(grep -n Loop log.lammps | awk -F ':' '{print $1}')" +in="$(echo "$in+1" | bc -l)" +en="$(echo "$en-$in" | bc -l)" +tail -n +$in log.lammps | head -n $en > res_nvt_lattice.dat + +# plot results +python3 plot_nvt.py res_nvt_spin.dat res_nvt_lattice.dat diff --git a/examples/USER/dpd/dpd-shardlow/in.dpd-shardlow b/examples/USER/dpd/dpd-shardlow/in.dpd-shardlow index e6e46bc38c..6251623374 100644 --- a/examples/USER/dpd/dpd-shardlow/in.dpd-shardlow +++ b/examples/USER/dpd/dpd-shardlow/in.dpd-shardlow @@ -1,5 +1,5 @@ # Input File for DPD fluid under isothermal conditions using the VV-SSA integration scheme -log log.dpd-shardlow + boundary p p p units metal diff --git a/examples/USER/dpd/dpd-vv/in.dpd-vv b/examples/USER/dpd/dpd-vv/in.dpd-vv index 5360ca2431..6890c881d4 100644 --- a/examples/USER/dpd/dpd-vv/in.dpd-vv +++ b/examples/USER/dpd/dpd-vv/in.dpd-vv @@ -1,6 +1,5 @@ # INPUT FILE FOR DPD_Fluid -log log.dpd-vv boundary p p p units metal # ev, ps diff --git a/examples/USER/dpd/dpde-shardlow/in.dpde-shardlow b/examples/USER/dpd/dpde-shardlow/in.dpde-shardlow index 68361adfa5..c639bef9f2 100644 --- a/examples/USER/dpd/dpde-shardlow/in.dpde-shardlow +++ b/examples/USER/dpd/dpde-shardlow/in.dpde-shardlow @@ -1,5 +1,5 @@ # Input File for DPD fluid under isoenergetic conditions using the VV-SSA integration scheme -log log.dpde-shardlow + boundary p p p units metal # ev, ps diff --git a/examples/USER/dpd/dpde-vv/in.dpde-vv b/examples/USER/dpd/dpde-vv/in.dpde-vv index 3ffcf0ec07..52309682c3 100644 --- a/examples/USER/dpd/dpde-vv/in.dpde-vv +++ b/examples/USER/dpd/dpde-vv/in.dpde-vv @@ -1,6 +1,5 @@ # INPUT FILE FOR DPD_Fluid -log log.dpde-vv boundary p p p units metal # ev, ps diff --git a/examples/USER/dpd/dpdh-shardlow/in.dpdh-shardlow b/examples/USER/dpd/dpdh-shardlow/in.dpdh-shardlow index e403175e7a..5353085359 100644 --- a/examples/USER/dpd/dpdh-shardlow/in.dpdh-shardlow +++ b/examples/USER/dpd/dpdh-shardlow/in.dpdh-shardlow @@ -1,5 +1,5 @@ # Input File for DPD fluid under isoenthalpic conditions using the VV-SSA integration scheme -log log.dpdh-shardlow + boundary p p p units metal # ev, ps diff --git a/examples/USER/dpd/dpdp-shardlow/in.dpdp-shardlow b/examples/USER/dpd/dpdp-shardlow/in.dpdp-shardlow index 1d5eb5dd7d..93f26bd946 100644 --- a/examples/USER/dpd/dpdp-shardlow/in.dpdp-shardlow +++ b/examples/USER/dpd/dpdp-shardlow/in.dpdp-shardlow @@ -1,5 +1,5 @@ # Input File for DPD fluid under isobaric conditions using the VV-SSA integration scheme -log log.dpdp-shardlow + boundary p p p units metal diff --git a/examples/USER/dpd/dpdrx-shardlow/in.dpdrx-shardlow b/examples/USER/dpd/dpdrx-shardlow/in.dpdrx-shardlow index 815c974741..f93b316e14 100755 --- a/examples/USER/dpd/dpdrx-shardlow/in.dpdrx-shardlow +++ b/examples/USER/dpd/dpdrx-shardlow/in.dpdrx-shardlow @@ -1,6 +1,5 @@ # Example for running DPD-RX -log log.dpdrx-shardlow boundary p p p units metal # ev, ps atom_style dpd diff --git a/examples/USER/dpd/multi-lucy/in.multi-lucy b/examples/USER/dpd/multi-lucy/in.multi-lucy index 26d21f66ea..bb223e0a25 100644 --- a/examples/USER/dpd/multi-lucy/in.multi-lucy +++ b/examples/USER/dpd/multi-lucy/in.multi-lucy @@ -1,5 +1,5 @@ # RDX coarse-grain model -log log.multi-lucy + units metal # ev, ps atom_style dpd atom_modify map array diff --git a/examples/USER/eff/Auger-Adamantane/in.adamantane_ionized.nve b/examples/USER/eff/Auger-Adamantane/in.adamantane_ionized.nve index 77c7b0d465..edbe76beef 100644 --- a/examples/USER/eff/Auger-Adamantane/in.adamantane_ionized.nve +++ b/examples/USER/eff/Auger-Adamantane/in.adamantane_ionized.nve @@ -1,5 +1,4 @@ variable sname index adamantane_ionized -log ${sname}.nve.log units electron newton on diff --git a/examples/USER/eff/Be-solid/in.Be-solid.spe b/examples/USER/eff/Be-solid/in.Be-solid.spe index e7245cabbc..ce61a21f8c 100644 --- a/examples/USER/eff/Be-solid/in.Be-solid.spe +++ b/examples/USER/eff/Be-solid/in.Be-solid.spe @@ -1,5 +1,4 @@ variable sname index Be-solid -log ${sname}.spe.log units electron newton on diff --git a/examples/USER/eff/CH4/in.ch4.dynamics b/examples/USER/eff/CH4/in.ch4.dynamics index 2536fa8f48..cb06a1c8cf 100644 --- a/examples/USER/eff/CH4/in.ch4.dynamics +++ b/examples/USER/eff/CH4/in.ch4.dynamics @@ -1,5 +1,4 @@ variable sname index ch4 -log ${sname}.nve.log units electron newton on diff --git a/examples/USER/eff/CH4/in.ch4.min b/examples/USER/eff/CH4/in.ch4.min index 69b7c15bd4..df052c728e 100644 --- a/examples/USER/eff/CH4/in.ch4.min +++ b/examples/USER/eff/CH4/in.ch4.min @@ -1,5 +1,4 @@ variable sname index ch4 -log ${sname}.nve.log units electron newton on diff --git a/examples/USER/eff/CH4/in.ch4_ionized.dynamics b/examples/USER/eff/CH4/in.ch4_ionized.dynamics index 14f214296f..51ca071fc6 100644 --- a/examples/USER/eff/CH4/in.ch4_ionized.dynamics +++ b/examples/USER/eff/CH4/in.ch4_ionized.dynamics @@ -1,5 +1,4 @@ variable sname index ch4_ionized -log ${sname}.nvt.log units electron newton on @@ -15,6 +14,7 @@ pair_coeff * * comm_modify vel yes # minimize + min_style cg min_modify line quadratic minimize 0 1.0e-6 10000 100000 diff --git a/examples/USER/eff/H/in.h_atom.spe.ang b/examples/USER/eff/H/in.h_atom.spe.ang index 2493e273c4..85bedf5b59 100644 --- a/examples/USER/eff/H/in.h_atom.spe.ang +++ b/examples/USER/eff/H/in.h_atom.spe.ang @@ -1,5 +1,4 @@ variable sname index h_atom.ang -log ${sname}.spe.log units real newton on diff --git a/examples/USER/eff/H/in.h_atom.spe.bohr b/examples/USER/eff/H/in.h_atom.spe.bohr index 2f18ee4e43..91b39a70db 100644 --- a/examples/USER/eff/H/in.h_atom.spe.bohr +++ b/examples/USER/eff/H/in.h_atom.spe.bohr @@ -1,5 +1,4 @@ variable sname index h_atom.bohr -log ${sname}.spe.log units electron newton on diff --git a/examples/USER/eff/H2/in.h2 b/examples/USER/eff/H2/in.h2 index 8f4a63cc84..0c9dd620a3 100644 --- a/examples/USER/eff/H2/in.h2 +++ b/examples/USER/eff/H2/in.h2 @@ -1,5 +1,4 @@ variable sname index h2 -log ${sname}.spe.log units electron newton on @@ -47,4 +46,4 @@ unfix 3 #fix 1 all nve/eff -run 100000 +run 10000 diff --git a/examples/USER/eff/H_plasma/in.h2bulk.npt b/examples/USER/eff/H_plasma/in.h2bulk.npt index 6d9fc5d55f..55c3d9b16f 100644 --- a/examples/USER/eff/H_plasma/in.h2bulk.npt +++ b/examples/USER/eff/H_plasma/in.h2bulk.npt @@ -1,5 +1,4 @@ variable sname index h2bulk -log ${sname}.npt.log units electron newton on @@ -41,7 +40,8 @@ dump 2 all xyz 10000 ${sname}.npt.xyz compute 1 all property/atom spin eradius dump 3 all custom 10000 ${sname}.npt.lammpstrj id type x y z c_1[1] c_1[2] -run 10000000 +thermo 1 +run 10 unfix 1 undump 2 diff --git a/examples/USER/eff/H_plasma/in.h2bulk.nve b/examples/USER/eff/H_plasma/in.h2bulk.nve index 9786ff476f..32b724c539 100644 --- a/examples/USER/eff/H_plasma/in.h2bulk.nve +++ b/examples/USER/eff/H_plasma/in.h2bulk.nve @@ -1,5 +1,4 @@ variable sname index h2bulk -log ${sname}.nve.log units electron newton on @@ -18,13 +17,13 @@ comm_modify vel yes compute effTemp all temp/eff -thermo 1000 +thermo 10 thermo_style custom step pe temp press thermo_modify temp effTemp # structure minimization min_style cg -minimize 0 1.0e-4 1000 10000 +minimize 0 1.0e-4 10 10 timestep 0.001 @@ -41,7 +40,7 @@ dump 2 all xyz 1000 ${sname}.nve.xyz compute 1 all property/atom spin eradius dump 3 all custom 1000 ${sname}.nve.lammpstrj id type x y z c_1[1] c_1[2] c_peatom c_keatom -run 100000 +run 10 unfix 1 #unfix 2 diff --git a/examples/USER/eff/H_plasma/in.h2bulk.nve.ang b/examples/USER/eff/H_plasma/in.h2bulk.nve.ang index 4935e39be3..3377d63580 100644 --- a/examples/USER/eff/H_plasma/in.h2bulk.nve.ang +++ b/examples/USER/eff/H_plasma/in.h2bulk.nve.ang @@ -1,5 +1,4 @@ variable sname index h2bulk.ang -log ${sname}.nve.log units real newton on @@ -29,13 +28,13 @@ variable epauli equal c_energies[2] variable ecoul equal c_energies[3] variable erres equal c_energies[4] -thermo 100 +thermo 10 thermo_style custom step etotal pe ke v_eke v_epauli v_ecoul v_erres press v_press temp thermo_modify temp effTemp press effPress flush yes # structure minimization min_style cg -minimize 0 1.0e-4 1000 10000 +minimize 0 1.0e-4 10 10 timestep 0.001 @@ -52,7 +51,7 @@ dump 2 all xyz 1000 ${sname}.nve.xyz compute 1 all property/atom spin eradius dump 3 all custom 1000 ${sname}.nve.lammpstrj id type x y z c_1[1] c_1[2] c_peatom c_keatom -run 100000 +run 10 unfix 1 #unfix 2 diff --git a/examples/USER/eff/Li-dendritic/in.Li-dendritic.min b/examples/USER/eff/Li-dendritic/in.Li-dendritic.min index f791544e44..5dfa85e52c 100644 --- a/examples/USER/eff/Li-dendritic/in.Li-dendritic.min +++ b/examples/USER/eff/Li-dendritic/in.Li-dendritic.min @@ -1,5 +1,4 @@ variable sname index Li-dendritic -log ${sname}.min.log units electron newton on @@ -7,8 +6,8 @@ boundary p p p atom_style electron -#read_data data.${sname} -read_restart ${sname}.min.restart2 +read_data data.${sname} +#read_restart ${sname}.min.restart2 pair_style eff/cut 50.112 pair_coeff * * diff --git a/examples/USER/eff/Li-dendritic/in.Li-dendritic.nvt b/examples/USER/eff/Li-dendritic/in.Li-dendritic.nvt index 143e23cbc1..4203b50dde 100644 --- a/examples/USER/eff/Li-dendritic/in.Li-dendritic.nvt +++ b/examples/USER/eff/Li-dendritic/in.Li-dendritic.nvt @@ -1,5 +1,4 @@ variable sname index Li-dendritic -log ${sname}.min.log units electron newton on @@ -29,7 +28,7 @@ compute 1 all property/atom spin eradius #dump 1 all custom 100 ${sname}.min.lammpstrj id type x y z q c_1[1] c_1[2] #dump 2 all xyz 100 ${sname}.min.xyz min_modify line quadratic dmax 0.05 -minimize 0 1.0e-7 1000 2000 +minimize 0 1.0e-7 100 100 write_restart ${sname}.min.restart @@ -46,7 +45,7 @@ dump 1 all custom 100 ${sname}.nvt.lammpstrj id type x y z c_1[1] c_1 dump 2 all xyz 100 ${sname}.nvt.xyz restart 100 ${sname}.nvt.restart1 ${sname}.nvt.restart2 -run 10000 +run 100 undump 1 undump 2 diff --git a/examples/USER/eff/Li-solid/in.Li.ang b/examples/USER/eff/Li-solid/in.Li.ang index c9a726b88c..e4f5614c56 100644 --- a/examples/USER/eff/Li-solid/in.Li.ang +++ b/examples/USER/eff/Li-solid/in.Li.ang @@ -1,5 +1,4 @@ variable sname index Li.ang -log ${sname}.spe.log units real newton on @@ -46,13 +45,13 @@ fix 0 all temp/rescale/eff 1 10.0 3000.0 0.05 1.0 #fix 0 all langevin/eff 3000.0 3000.0 10.0 699483 fix 1 all nve/eff -run 1000 +run 200 unfix 0 unfix 1 fix 1 all nvt/eff temp 3000.0 3000.0 100.0 compute 1 all property/atom spin eradius ervel -dump 1 all custom 1000 ${sname}.nvt.lammpstrj id type q c_1[1] c_1[2] x y z vx vy vz c_1[3] +dump 1 all custom 500 ${sname}.nvt.lammpstrj id type q c_1[1] c_1[2] x y z vx vy vz c_1[3] -run 100000 +run 500 diff --git a/examples/USER/eff/Li-solid/in.Li.bohr b/examples/USER/eff/Li-solid/in.Li.bohr index 2843d3ca6a..2ff8b661d5 100644 --- a/examples/USER/eff/Li-solid/in.Li.bohr +++ b/examples/USER/eff/Li-solid/in.Li.bohr @@ -1,5 +1,4 @@ variable sname index Li.bohr -log ${sname}.spe.log units electron newton off @@ -35,15 +34,15 @@ thermo_style custom step etotal pe ke v_eke v_epauli v_estatics v_errestrain thermo_modify temp effTemp press effPress min_style cg -minimize 0 1e-6 100 2000 +minimize 0 1e-6 100 200 fix 0 all temp/rescale/eff 1 0.0 3000.0 0.02 0.5 fix 1 all npt/eff temp 3000.0 3000.0 0.1 iso 1e7 1e7 1.0 compute 1 all property/atom spin eradius -dump 1 all custom 1 ${sname}.spe.lammpstrj & +dump 1 all custom 100 ${sname}.spe.lammpstrj & id type q c_1[1] c_1[2] x y z -run 1000 +run 100 diff --git a/examples/USER/reaction/nylon,6-6_melt/large_nylon_melt.data.gz b/examples/USER/reaction/nylon,6-6_melt/large_nylon_melt.data.gz new file mode 100644 index 0000000000..c620b879a8 Binary files /dev/null and b/examples/USER/reaction/nylon,6-6_melt/large_nylon_melt.data.gz differ diff --git a/examples/USER/reaction/tiny_nylon/in.tiny_nylon.stabilized_variable_probability b/examples/USER/reaction/tiny_nylon/in.tiny_nylon.stabilized_variable_probability new file mode 100644 index 0000000000..2c101ac77c --- /dev/null +++ b/examples/USER/reaction/tiny_nylon/in.tiny_nylon.stabilized_variable_probability @@ -0,0 +1,56 @@ +# two monomer nylon example +# reaction produces a condensed water molecule + +units real + +boundary p p p + +atom_style full + +kspace_style pppm 1.0e-4 + +pair_style lj/class2/coul/long 8.5 + +angle_style class2 + +bond_style class2 + +dihedral_style class2 + +improper_style class2 + +read_data tiny_nylon.data + +variable runsteps equal 1000 +variable prob1 equal step/v_runsteps*2 +variable prob2 equal (step/v_runsteps)>0.5 + +velocity all create 300.0 4928459 dist gaussian + +molecule mol1 rxn1_stp1_unreacted.data_template +molecule mol2 rxn1_stp1_reacted.data_template +molecule mol3 rxn1_stp2_unreacted.data_template +molecule mol4 rxn1_stp2_reacted.data_template + +thermo 50 + +# dump 1 all xyz 1 test_vis.xyz + +fix myrxns all bond/react stabilization yes statted_grp .03 & + react rxn1 all 1 0.0 5.0 mol1 mol2 rxn1_stp1_map prob v_prob1 1234 & + react rxn2 all 1 0.0 5.0 mol3 mol4 rxn1_stp2_map prob v_prob2 1234 + +fix 1 statted_grp_REACT nvt temp 300 300 100 + +# optionally, you can customize behavior of reacting atoms, +# by using the internally-created 'bond_react_MASTER_group', like so: +fix 4 bond_react_MASTER_group temp/rescale 1 300 300 10 1 + +thermo_style custom step temp press density v_prob1 v_prob2 f_myrxns[1] f_myrxns[2] + +# restart 100 restart1 restart2 + +run ${runsteps} + +# write_restart restart_longrun +# write_data restart_longrun.data diff --git a/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.1 b/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.1 new file mode 100644 index 0000000000..16e4deef51 --- /dev/null +++ b/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.1 @@ -0,0 +1,201 @@ +LAMMPS (15 Apr 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# two monomer nylon example +# reaction produces a condensed water molecule + +units real + +boundary p p p + +atom_style full + +kspace_style pppm 1.0e-4 + +pair_style lj/class2/coul/long 8.5 + +angle_style class2 + +bond_style class2 + +dihedral_style class2 + +improper_style class2 + +read_data tiny_nylon.data + orthogonal box = (-25 -25 -25) to (25 25 25) + 1 by 1 by 1 MPI processor grid + reading atoms ... + 44 atoms + reading velocities ... + 44 velocities + scanning bonds ... + 9 = max bonds/atom + scanning angles ... + 21 = max angles/atom + scanning dihedrals ... + 29 = max dihedrals/atom + scanning impropers ... + 29 = max impropers/atom + reading bonds ... + 42 bonds + reading angles ... + 74 angles + reading dihedrals ... + 100 dihedrals + reading impropers ... + 44 impropers + 4 = max # of 1-2 neighbors + 6 = max # of 1-3 neighbors + 12 = max # of 1-4 neighbors + 41 = max # of special neighbors + special bonds CPU = 0.000385045 secs + read_data CPU = 0.013443 secs + +variable runsteps equal 1000 +variable prob1 equal step/v_runsteps*2 +variable prob2 equal (step/v_runsteps)>0.5 + +velocity all create 300.0 4928459 dist gaussian + +molecule mol1 rxn1_stp1_unreacted.data_template +Read molecule template mol1: + 1 molecules + 18 atoms with max type 8 + 16 bonds with max type 14 + 25 angles with max type 28 + 23 dihedrals with max type 36 + 14 impropers with max type 11 +molecule mol2 rxn1_stp1_reacted.data_template +Read molecule template mol2: + 1 molecules + 18 atoms with max type 9 + 17 bonds with max type 13 + 31 angles with max type 27 + 39 dihedrals with max type 33 + 20 impropers with max type 1 +molecule mol3 rxn1_stp2_unreacted.data_template +Read molecule template mol3: + 1 molecules + 15 atoms with max type 9 + 14 bonds with max type 13 + 25 angles with max type 27 + 30 dihedrals with max type 33 + 16 impropers with max type 1 +molecule mol4 rxn1_stp2_reacted.data_template +Read molecule template mol4: + 1 molecules + 15 atoms with max type 11 + 13 bonds with max type 15 + 19 angles with max type 29 + 16 dihedrals with max type 32 + 10 impropers with max type 13 + +thermo 50 + +# dump 1 all xyz 1 test_vis.xyz + +fix myrxns all bond/react stabilization yes statted_grp .03 react rxn1 all 1 0.0 5.0 mol1 mol2 rxn1_stp1_map prob v_prob1 1234 react rxn2 all 1 0.0 5.0 mol3 mol4 rxn1_stp2_map prob v_prob2 1234 +WARNING: Bond/react: Atom affected by reaction rxn1 too close to template edge (src/USER-REACTION/fix_bond_react.cpp:2051) +WARNING: Bond/react: Atom affected by reaction rxn2 too close to template edge (src/USER-REACTION/fix_bond_react.cpp:2051) +dynamic group bond_react_MASTER_group defined +dynamic group statted_grp_REACT defined + +fix 1 statted_grp_REACT nvt temp 300 300 100 + +# optionally, you can customize behavior of reacting atoms, +# by using the internally-created 'bond_react_MASTER_group', like so: +fix 4 bond_react_MASTER_group temp/rescale 1 300 300 10 1 + +thermo_style custom step temp press density v_prob1 v_prob2 f_myrxns[1] f_myrxns[2] + +# restart 100 restart1 restart2 + +run ${runsteps} +run 1000 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/kspace.cpp:332) + G vector (1/distance) = 0.0534597 + grid = 2 2 2 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0402256 + estimated relative force accuracy = 0.000121138 + using double precision FFTW3 + 3d grid and FFT values/proc = 343 8 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 10.5 + ghost atom cutoff = 10.5 + binsize = 5.25, bins = 10 10 10 + 2 neighbor lists, perpetual/occasional/extra = 1 1 0 + (1) pair lj/class2/coul/long, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard + (2) fix bond/react, occasional, copy from (1) + attributes: half, newton on + pair build: copy + stencil: none + bin: none +WARNING: Inconsistent image flags (src/domain.cpp:812) +Per MPI rank memory allocation (min/avg/max) = 33.78 | 33.78 | 33.78 Mbytes +Step Temp Press Density v_prob1 v_prob2 f_myrxns[1] f_myrxns[2] + 0 300 346.78165 0.0034851739 0 0 0 0 + 50 262.63913 -492.10749 0.0034851739 0.1 0 1 0 + 100 766.52962 -29.714349 0.0034851739 0.2 0 1 0 + 150 503.86837 50.220304 0.0034851739 0.3 0 1 0 + 200 456.51295 12.312892 0.0034851739 0.4 0 1 0 + 250 391.54928 9.2335844 0.0034851739 0.5 0 1 0 + 300 336.6988 -47.193937 0.0034851739 0.6 0 1 0 + 350 254.06985 -9.2867898 0.0034851739 0.7 0 1 0 + 400 259.41098 -25.657321 0.0034851739 0.8 0 1 0 + 450 258.10364 22.5086 0.0034851739 0.9 0 1 0 + 500 272.13412 -6.5391448 0.0034851739 1 0 1 0 + 550 202.75504 54.658731 0.0034851739 1.1 1 1 1 + 600 344.79887 23.798478 0.0034851739 1.2 1 1 1 + 650 328.44488 -29.908484 0.0034851739 1.3 1 1 1 + 700 280.13593 -8.3223255 0.0034851739 1.4 1 1 1 + 750 300.67624 1.0632669 0.0034851739 1.5 1 1 1 + 800 376.64234 12.488392 0.0034851739 1.6 1 1 1 + 850 321.07642 19.814074 0.0034851739 1.7 1 1 1 + 900 332.23751 30.814079 0.0034851739 1.8 1 1 1 + 950 311.14029 5.7853136 0.0034851739 1.9 1 1 1 + 1000 253.14634 -37.560642 0.0034851739 2 1 1 1 +Loop time of 0.379454 on 1 procs for 1000 steps with 44 atoms + +Performance: 227.696 ns/day, 0.105 hours/ns, 2635.368 timesteps/s +99.6% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.069723 | 0.069723 | 0.069723 | 0.0 | 18.37 +Bond | 0.14802 | 0.14802 | 0.14802 | 0.0 | 39.01 +Kspace | 0.044252 | 0.044252 | 0.044252 | 0.0 | 11.66 +Neigh | 0.072359 | 0.072359 | 0.072359 | 0.0 | 19.07 +Comm | 0.0044748 | 0.0044748 | 0.0044748 | 0.0 | 1.18 +Output | 0.0022775 | 0.0022775 | 0.0022775 | 0.0 | 0.60 +Modify | 0.036509 | 0.036509 | 0.036509 | 0.0 | 9.62 +Other | | 0.00184 | | | 0.48 + +Nlocal: 44 ave 44 max 44 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 3 ave 3 max 3 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 722 ave 722 max 722 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 722 +Ave neighs/atom = 16.4091 +Ave special neighs/atom = 9.77273 +Neighbor list builds = 1000 +Dangerous builds = 0 + +# write_restart restart_longrun +# write_data restart_longrun.data + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:00 diff --git a/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.4 b/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.4 new file mode 100644 index 0000000000..527d71ce87 --- /dev/null +++ b/examples/USER/reaction/tiny_nylon/log.22Apr20.tiny_nylon.stabilized_variable_probability.g++.4 @@ -0,0 +1,201 @@ +LAMMPS (15 Apr 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# two monomer nylon example +# reaction produces a condensed water molecule + +units real + +boundary p p p + +atom_style full + +kspace_style pppm 1.0e-4 + +pair_style lj/class2/coul/long 8.5 + +angle_style class2 + +bond_style class2 + +dihedral_style class2 + +improper_style class2 + +read_data tiny_nylon.data + orthogonal box = (-25 -25 -25) to (25 25 25) + 1 by 2 by 2 MPI processor grid + reading atoms ... + 44 atoms + reading velocities ... + 44 velocities + scanning bonds ... + 9 = max bonds/atom + scanning angles ... + 21 = max angles/atom + scanning dihedrals ... + 29 = max dihedrals/atom + scanning impropers ... + 29 = max impropers/atom + reading bonds ... + 42 bonds + reading angles ... + 74 angles + reading dihedrals ... + 100 dihedrals + reading impropers ... + 44 impropers + 4 = max # of 1-2 neighbors + 6 = max # of 1-3 neighbors + 12 = max # of 1-4 neighbors + 41 = max # of special neighbors + special bonds CPU = 0.000431282 secs + read_data CPU = 0.0129571 secs + +variable runsteps equal 1000 +variable prob1 equal step/v_runsteps*2 +variable prob2 equal (step/v_runsteps)>0.5 + +velocity all create 300.0 4928459 dist gaussian + +molecule mol1 rxn1_stp1_unreacted.data_template +Read molecule template mol1: + 1 molecules + 18 atoms with max type 8 + 16 bonds with max type 14 + 25 angles with max type 28 + 23 dihedrals with max type 36 + 14 impropers with max type 11 +molecule mol2 rxn1_stp1_reacted.data_template +Read molecule template mol2: + 1 molecules + 18 atoms with max type 9 + 17 bonds with max type 13 + 31 angles with max type 27 + 39 dihedrals with max type 33 + 20 impropers with max type 1 +molecule mol3 rxn1_stp2_unreacted.data_template +Read molecule template mol3: + 1 molecules + 15 atoms with max type 9 + 14 bonds with max type 13 + 25 angles with max type 27 + 30 dihedrals with max type 33 + 16 impropers with max type 1 +molecule mol4 rxn1_stp2_reacted.data_template +Read molecule template mol4: + 1 molecules + 15 atoms with max type 11 + 13 bonds with max type 15 + 19 angles with max type 29 + 16 dihedrals with max type 32 + 10 impropers with max type 13 + +thermo 50 + +# dump 1 all xyz 1 test_vis.xyz + +fix myrxns all bond/react stabilization yes statted_grp .03 react rxn1 all 1 0.0 5.0 mol1 mol2 rxn1_stp1_map prob v_prob1 1234 react rxn2 all 1 0.0 5.0 mol3 mol4 rxn1_stp2_map prob v_prob2 1234 +WARNING: Bond/react: Atom affected by reaction rxn1 too close to template edge (src/USER-REACTION/fix_bond_react.cpp:2051) +WARNING: Bond/react: Atom affected by reaction rxn2 too close to template edge (src/USER-REACTION/fix_bond_react.cpp:2051) +dynamic group bond_react_MASTER_group defined +dynamic group statted_grp_REACT defined + +fix 1 statted_grp_REACT nvt temp 300 300 100 + +# optionally, you can customize behavior of reacting atoms, +# by using the internally-created 'bond_react_MASTER_group', like so: +fix 4 bond_react_MASTER_group temp/rescale 1 300 300 10 1 + +thermo_style custom step temp press density v_prob1 v_prob2 f_myrxns[1] f_myrxns[2] + +# restart 100 restart1 restart2 + +run ${runsteps} +run 1000 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/kspace.cpp:332) + G vector (1/distance) = 0.0534597 + grid = 2 2 2 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0402256 + estimated relative force accuracy = 0.000121138 + using double precision FFTW3 + 3d grid and FFT values/proc = 252 2 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 10.5 + ghost atom cutoff = 10.5 + binsize = 5.25, bins = 10 10 10 + 2 neighbor lists, perpetual/occasional/extra = 1 1 0 + (1) pair lj/class2/coul/long, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard + (2) fix bond/react, occasional, copy from (1) + attributes: half, newton on + pair build: copy + stencil: none + bin: none +WARNING: Inconsistent image flags (src/domain.cpp:812) +Per MPI rank memory allocation (min/avg/max) = 33.66 | 33.88 | 34.43 Mbytes +Step Temp Press Density v_prob1 v_prob2 f_myrxns[1] f_myrxns[2] + 0 300 346.78165 0.0034851739 0 0 0 0 + 50 266.5092 -90.813802 0.0034851739 0.1 0 1 0 + 100 559.41271 -53.23688 0.0034851739 0.2 0 1 0 + 150 489.90516 31.555817 0.0034851739 0.3 0 1 0 + 200 326.18391 7.7889992 0.0034851739 0.4 0 1 0 + 250 339.78203 2.3919541 0.0034851739 0.5 0 1 0 + 300 370.90263 -32.01673 0.0034851739 0.6 0 1 0 + 350 294.07547 -5.4019813 0.0034851739 0.7 0 1 0 + 400 287.76477 12.254133 0.0034851739 0.8 0 1 0 + 450 293.36482 66.372956 0.0034851739 0.9 0 1 0 + 500 246.84496 26.132317 0.0034851739 1 0 1 0 + 550 253.08778 -15.350262 0.0034851739 1.1 1 1 1 + 600 358.83641 25.007371 0.0034851739 1.2 1 1 1 + 650 320.51492 -32.34823 0.0034851739 1.3 1 1 1 + 700 310.87976 -8.2306669 0.0034851739 1.4 1 1 1 + 750 307.54142 12.025818 0.0034851739 1.5 1 1 1 + 800 272.51724 -22.92823 0.0034851739 1.6 1 1 1 + 850 268.66181 10.069534 0.0034851739 1.7 1 1 1 + 900 265.5531 -10.471377 0.0034851739 1.8 1 1 1 + 950 259.43086 9.4546712 0.0034851739 1.9 1 1 1 + 1000 247.14622 20.250308 0.0034851739 2 1 1 1 +Loop time of 0.357762 on 4 procs for 1000 steps with 44 atoms + +Performance: 241.502 ns/day, 0.099 hours/ns, 2795.157 timesteps/s +99.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.0003917 | 0.015545 | 0.033317 | 11.9 | 4.35 +Bond | 0.0010131 | 0.030153 | 0.076975 | 18.2 | 8.43 +Kspace | 0.092857 | 0.1462 | 0.18688 | 10.7 | 40.87 +Neigh | 0.043786 | 0.044014 | 0.044189 | 0.1 | 12.30 +Comm | 0.03636 | 0.038345 | 0.040538 | 0.8 | 10.72 +Output | 0.00091578 | 0.0012541 | 0.0020923 | 1.4 | 0.35 +Modify | 0.075379 | 0.080791 | 0.086052 | 1.8 | 22.58 +Other | | 0.00146 | | | 0.41 + +Nlocal: 11 ave 32 max 0 min +Histogram: 2 0 1 0 0 0 0 0 0 1 +Nghost: 40 ave 51 max 19 min +Histogram: 1 0 0 0 0 0 0 1 0 2 +Neighs: 191 ave 529 max 0 min +Histogram: 2 0 0 0 1 0 0 0 0 1 + +Total # of neighbors = 764 +Ave neighs/atom = 17.3636 +Ave special neighs/atom = 9.77273 +Neighbor list builds = 1000 +Dangerous builds = 0 + +# write_restart restart_longrun +# write_data restart_longrun.data + +Please see the log.cite file for references relevant to this simulation + +Total wall time: 0:00:00 diff --git a/examples/USER/sph/cavity_flow/cavity_flow.lmp b/examples/USER/sph/cavity_flow/cavity_flow.lmp index 5c1517687f..19d3a3c14f 100644 --- a/examples/USER/sph/cavity_flow/cavity_flow.lmp +++ b/examples/USER/sph/cavity_flow/cavity_flow.lmp @@ -1,6 +1,6 @@ dimension 2 units si -atom_style meso +atom_style sph # create simulation box region box block -0.050e-3 1.044e-3 -0.05e-3 1.044e-3 -1.0e-6 1.0e-6 units box @@ -28,7 +28,7 @@ group integrate_full union fluid driver mass 3 2.0e-7 mass 2 2.0e-7 mass 1 4.0e-7 -set group all meso/rho 1000.0 +set group all sph/rho 1000.0 # use Tait's EOS in combination with Morris' laminar viscosity. # We set rho_0 = 1000 kg/m^3, c = 0.1 m/s, h = 6.5e-5 m. @@ -37,8 +37,8 @@ pair_style hybrid sph/taitwater/morris pair_coeff * * sph/taitwater/morris 1000 0.1 1.0e-3 6.5e-5 pair_coeff 2 3 none # exclude interaction between walls and shear driver -compute rho_peratom all meso/rho/atom -compute e_peratom all meso/e/atom +compute rho_peratom all sph/rho/atom +compute e_peratom all sph/e/atom compute ke_peratom all ke/atom compute esph all reduce sum c_e_peratom compute ke all ke @@ -49,8 +49,8 @@ velocity driver set 0.001 0.0 0.0 units box fix freeze_fix driver setforce 0.0 0.0 0.0 # do full time integration for shear driver and fluid, but keep walls stationary -fix integrate_fix_full integrate_full meso -fix integrate_fix_stationary walls meso/stationary +fix integrate_fix_full integrate_full sph +fix integrate_fix_stationary walls sph/stationary dump dump_id all custom 100 dump.lammpstrj id type xs ys zs vx vy c_rho_peratom c_e_peratom dump_modify dump_id first yes diff --git a/examples/USER/sph/heatconduction/sph_heat_conduction_2d.lmp b/examples/USER/sph/heatconduction/sph_heat_conduction_2d.lmp index 808646059f..4d0db9c75a 100644 --- a/examples/USER/sph/heatconduction/sph_heat_conduction_2d.lmp +++ b/examples/USER/sph/heatconduction/sph_heat_conduction_2d.lmp @@ -1,4 +1,4 @@ -# mesoscopic heat conduction +# SPH heat conduction # heat flow from hot right region to cold left region # compare the temperature profile at the end opf the simulation, # contained in file dump.last, to analytic solution. @@ -6,7 +6,7 @@ # dimension 2 units si -atom_style meso +atom_style sph boundary f p p lattice sq 0.01 @@ -17,23 +17,23 @@ mass 1 1.0e-5 region left block EDGE 49.9 EDGE EDGE EDGE EDGE region right block 50 EDGE EDGE EDGE EDGE EDGE -set region left meso/e 1.0 # internal energies -set region right meso/e 2.0 -set group all meso/rho 0.1 # mesoscopic density is also needed for this pair style +set region left sph/e 1.0 # internal energies +set region right sph/e 2.0 +set group all sph/rho 0.1 # SPH density is also needed for this pair style # For correct temperature profiles, mescoscopic density and mass * number density must coincide! pair_style sph/heatconduction # i j diffusion coeff. cutoff pair_coeff 1 1 1.0e-4 2.0e-2 -compute ie_atom all meso/e/atom +compute ie_atom all sph/e/atom compute ie all reduce sum c_ie_atom thermo 10 thermo_style custom step temp c_ie timestep 0.25e-1 neighbor 0.2e-2 bin -fix integrate_fix all meso/stationary +fix integrate_fix all sph/stationary dump dump_fix all custom 10 dump.heat id type x y z c_ie_atom dump_modify dump_fix first yes diff --git a/examples/USER/sph/heatconduction/sph_heat_conduction_3d.lmp b/examples/USER/sph/heatconduction/sph_heat_conduction_3d.lmp index fc6fa07457..ee9a5a8661 100644 --- a/examples/USER/sph/heatconduction/sph_heat_conduction_3d.lmp +++ b/examples/USER/sph/heatconduction/sph_heat_conduction_3d.lmp @@ -1,11 +1,11 @@ -# mesoscopic heat conduction +# SPH heat conduction # heat flow from hot right region to cold left region # compare the temperature profile at the end opf the simulation, # contained in file dump.last, to analytic solution. # # units si -atom_style meso +atom_style sph newton on boundary f p p @@ -17,9 +17,9 @@ mass 1 1.0e-5 region left block EDGE 49.9 EDGE EDGE EDGE EDGE region right block 50 EDGE EDGE EDGE EDGE EDGE -set region left meso/e 1.0 # internal energies -set region right meso/e 2.0 -set group all meso/rho 10.0 # mesoscopic density is also needed for this pair style +set region left sph/e 1.0 # internal energies +set region right sph/e 2.0 +set group all sph/rho 10.0 # SPH density is also needed for this pair style # For correct temperature profiles, mescoscopic density and mass * number density must coincide! pair_style sph/heatconduction @@ -28,13 +28,13 @@ pair_coeff 1 1 1.0e-4 2.0e-2 neighbor 0.2e-2 bin neigh_modify every 20 delay 0 check no -compute ie_atom all meso/e/atom +compute ie_atom all sph/e/atom compute ie all reduce sum c_ie_atom thermo_style custom step temp c_ie thermo_modify norm no -fix integrate_fix all meso/stationary +fix integrate_fix all sph/stationary thermo 10 timestep 0.25e-1 diff --git a/examples/USER/sph/shock_tube/shock2d.lmp b/examples/USER/sph/shock_tube/shock2d.lmp index bd67362ca9..32cfd8067b 100755 --- a/examples/USER/sph/shock_tube/shock2d.lmp +++ b/examples/USER/sph/shock_tube/shock2d.lmp @@ -1,4 +1,4 @@ -atom_style meso +atom_style sph dimension 2 boundary s p p @@ -13,20 +13,20 @@ set region right type 2 mass 1 1 mass 2 0.25 -set type 1 meso/e 2.5 # internal energy corresponding to p=1, rho=1 -set type 2 meso/e 0.625 # internal energy corresponding to p=0.25, rho=0.25 -set type 1 meso/rho 1.0 -set type 2 meso/rho 0.25 +set type 1 sph/e 2.5 # internal energy corresponding to p=1, rho=1 +set type 2 sph/e 0.625 # internal energy corresponding to p=0.25, rho=0.25 +set type 1 sph/rho 1.0 +set type 2 sph/rho 0.25 pair_style hybrid/overlay sph/rhosum 1 sph/idealgas pair_coeff * * sph/rhosum 4.0 pair_coeff * * sph/idealgas 0.75 4.0 -compute rhoatom all meso/rho/atom -compute ieatom all meso/e/atom -compute emeso all reduce sum c_ieatom # total internal energy +compute rhoatom all shp/rho/atom +compute ieatom all sph/e/atom +compute esph all reduce sum c_ieatom # total internal energy compute ke all ke -variable etot equal c_ke+c_emeso # total energy +variable etot equal c_ke+c_esph # total energy # dump positions and local density dump dump_id all custom 100 dump.2d id type x z y c_rhoatom @@ -35,10 +35,10 @@ dump_modify dump_id first yes neighbor 0.5 bin neigh_modify every 5 delay 0 check yes thermo 10 -thermo_style custom step c_ke c_emeso v_etot +thermo_style custom step c_ke c_esph v_etot thermo_modify norm no -fix integration_fix all meso +fix integration_fix all sph fix 1 all setforce NULL 0.0 0.0 # treat as a quasi 1d problem timestep 0.05 log log.2d diff --git a/examples/USER/sph/shock_tube/shock3d.lmp b/examples/USER/sph/shock_tube/shock3d.lmp index 2da892d836..d82f081e85 100644 --- a/examples/USER/sph/shock_tube/shock3d.lmp +++ b/examples/USER/sph/shock_tube/shock3d.lmp @@ -1,4 +1,4 @@ -atom_style meso +atom_style sph boundary s p p region box block -100 150 -4 4 -4 4 units box @@ -12,20 +12,20 @@ set region right type 2 mass 1 1 mass 2 0.25 -set type 1 meso/e 2.5 # internal energy corresponding to p=1, rho=1 -set type 2 meso/e 0.625 # internal energy corresponding to p=0.25, rho=0.25 -set type 1 meso/rho 1.0 -set type 2 meso/rho 0.25 +set type 1 sph/e 2.5 # internal energy corresponding to p=1, rho=1 +set type 2 sph/e 0.625 # internal energy corresponding to p=0.25, rho=0.25 +set type 1 sph/rho 1.0 +set type 2 sph/rho 0.25 pair_style hybrid/overlay sph/rhosum 1 sph/idealgas pair_coeff * * sph/rhosum 4.0 pair_coeff * * sph/idealgas 0.75 4.0 -compute rhoatom all meso/rho/atom -compute ieatom all meso/e/atom -compute emeso all reduce sum c_ieatom # total internal energy +compute rhoatom all sph/rho/atom +compute ieatom all sph/e/atom +compute esph all reduce sum c_ieatom # total internal energy compute ke all ke -variable etot equal c_ke+c_emeso # total energy +variable etot equal c_ke+c_esph # total energy # dump positions and local density dump dump_id all custom 100 dump.3d id type x z y c_rhoatom @@ -34,10 +34,10 @@ dump_modify dump_id first yes neighbor 0.5 bin neigh_modify every 5 delay 0 check yes thermo 10 -thermo_style custom step c_ke c_emeso v_etot +thermo_style custom step c_ke c_esph v_etot thermo_modify norm no -fix integration_fix all meso +fix integration_fix all sph fix 1 all setforce NULL 0.0 0.0 # treat as a quasi 1d problem timestep 0.05 log log.3d diff --git a/examples/USER/sph/water_collapse/water_collapse.lmp b/examples/USER/sph/water_collapse/water_collapse.lmp index 5e1359dae5..c1f3815947 100644 --- a/examples/USER/sph/water_collapse/water_collapse.lmp +++ b/examples/USER/sph/water_collapse/water_collapse.lmp @@ -1,5 +1,5 @@ processors * 1 1 # manually assign processors to spatial regions -atom_style meso +atom_style sph dimension 2 newton on boundary f f p @@ -22,8 +22,8 @@ pair_coeff 1 1 sph/rhosum ${h} fix gfix water gravity -9.81 vector 0 1 0 # add gravity. This fix also computes potential energy of mass in gravity field. fix 2d_fix all enforce2d -compute rho_peratom all meso/rho/atom -compute e_peratom all meso/e/atom +compute rho_peratom all sph/rho/atom +compute e_peratom all sph/e/atom compute esph all reduce sum c_e_peratom compute ke all ke variable etot equal c_esph+c_ke+f_gfix @@ -32,10 +32,10 @@ variable etot equal c_esph+c_ke+f_gfix fix dtfix all dt/reset 1 NULL ${dt} 0.0005 units box # use a variable timestep # time-integrate position, velocities, internal energy and density of water particles -fix integrate_water_fix water meso +fix integrate_water_fix water sph # time-integrate only internal energy and density of boundary particles -fix integrate_bc_fix bc meso/stationary +fix integrate_bc_fix bc sph/stationary dump dump_id all custom 100 dump.lammpstrj id type xs ys zs c_rho_peratom c_e_peratom fx fy dump_modify dump_id first yes thermo 10 diff --git a/examples/VISCOSITY/README b/examples/VISCOSITY/README index 38bee0c58d..8235fa1f77 100644 --- a/examples/VISCOSITY/README +++ b/examples/VISCOSITY/README @@ -1,5 +1,6 @@ -This directory has 5 scripts that compute the viscosity (eta) of a -Lennard-Jones fluid using 5 different methods. See the discussion in +This directory has 6 scripts that compute the viscosity (eta) of fluid +using 6 different methods. 5 of them are for a Lennard-Jones fluid +and the last one is for SPC/E water model. See the discussion in Section 6.21 of the manual for an overview of the methods and pointers to doc pages for the commands which implement them. Citations for the various methods can also be found in the manual. @@ -10,7 +11,7 @@ enough to generate good statistics and highly accurate results. ------------- -These are the 5 methods for computing viscosity. The first 3 are +These are the 5 methods for computing viscosity of a LJ fluid. The first 3 are non-equilibrium methods; the last 2 are equilibrium methods. in.wall = move a wall to shear the fluid between two walls @@ -89,3 +90,18 @@ heat/flux doc page - the resulting value prints at the end of the run and is in the log file eta = 1.07 + +------------- + +in.cos.1000SPCE is an example script of using cosine periodic perturbation method +to calculate the viscosity of SPC/E water model. + +The reciprocal of eta is computed within the script, and printed out as v_invVis +in thermo_style command. The result will converge after hundreds of picoseconds. +Then eta is obtained from the reciprocal of time average of v_invVis. + +eta = 0.75 mPa*s + +Note that the calculated viscosity by this method decreases with increased acceleration. +It is therefore generally necessary to perform calculation at different accelerations +and extrapolate the viscosity to zero shear. diff --git a/examples/VISCOSITY/data.cos.1000SPCE b/examples/VISCOSITY/data.cos.1000SPCE new file mode 100644 index 0000000000..e04d3cbf51 --- /dev/null +++ b/examples/VISCOSITY/data.cos.1000SPCE @@ -0,0 +1,6043 @@ +# DFF generated LAMMPS data file +3000 atoms +2000 bonds +1000 angles +0 dihedrals +0 impropers + +2 atom types +1 bond types +1 angle types +0 dihedral types +0 improper types + + 0.0000 31.0430 xlo xhi + 0.0000 31.0430 ylo yhi + 0.0000 31.0430 zlo zhi + +Masses + + 1 15.9994 # o_2w + 2 1.0079 # h_1o + +Atoms + + 1 1 1 -0.84760 10.02000 17.96200 20.41400 # O1 + 2 1 2 0.42380 10.01000 18.85000 20.89600 # H2 + 3 1 2 0.42380 10.87100 17.45700 20.64700 # H3 + 4 2 1 -0.84760 5.99700 12.62400 25.60500 # O4 + 5 2 2 0.42380 5.78800 12.30400 26.55300 # H5 + 6 2 2 0.42380 6.69100 11.94300 25.34500 # H6 + 7 3 1 -0.84760 8.41400 5.80900 17.01200 # O7 + 8 3 2 0.42380 8.48900 5.69300 16.00800 # H8 + 9 3 2 0.42380 7.90800 4.99800 17.40500 # H9 + 10 4 1 -0.84760 17.58100 16.62000 6.43700 # O10 + 11 4 2 0.42380 17.25100 17.30600 7.17000 # H11 + 12 4 2 0.42380 18.39900 17.02700 5.90600 # H12 + 13 5 1 -0.84760 15.82700 21.92200 15.08600 # O13 + 14 5 2 0.42380 16.43600 22.69100 15.11000 # H14 + 15 5 2 0.42380 16.56400 21.17400 14.98900 # H15 + 16 6 1 -0.84760 24.18200 -0.18100 27.97700 # O16 + 17 6 2 0.42380 23.95500 -0.27600 26.96300 # H17 + 18 6 2 0.42380 23.44400 0.37300 28.30000 # H18 + 19 7 1 -0.84760 7.19500 7.91600 -0.56500 # O19 + 20 7 2 0.42380 7.55900 6.96600 -0.38300 # H20 + 21 7 2 0.42380 7.20700 8.39600 0.34200 # H21 + 22 8 1 -0.84760 5.63500 13.85300 11.25200 # O22 + 23 8 2 0.42380 6.08200 13.16700 10.62800 # H23 + 24 8 2 0.42380 4.84500 13.43100 11.72500 # H24 + 25 9 1 -0.84760 23.85900 -0.47100 19.57500 # O25 + 26 9 2 0.42380 24.23000 -0.80400 18.71100 # H26 + 27 9 2 0.42380 23.88300 -1.23400 20.25500 # H27 + 28 10 1 -0.84760 7.19100 20.92100 28.09900 # O28 + 29 10 2 0.42380 6.61400 21.56500 27.55300 # H29 + 30 10 2 0.42380 6.68200 20.08700 28.02500 # H30 + 31 11 1 -0.84760 6.10000 27.87700 18.99800 # O31 + 32 11 2 0.42380 5.49800 27.32200 18.45300 # H32 + 33 11 2 0.42380 7.03600 27.63800 18.73900 # H33 + 34 12 1 -0.84760 22.42000 9.51900 9.70000 # O34 + 35 12 2 0.42380 22.08300 9.67000 10.67900 # H35 + 36 12 2 0.42380 23.36100 9.97900 9.69000 # H36 + 37 13 1 -0.84760 29.33600 21.61500 18.38900 # O37 + 38 13 2 0.42380 29.20300 22.58600 18.29200 # H38 + 39 13 2 0.42380 28.37100 21.33000 18.42500 # H39 + 40 14 1 -0.84760 20.82900 25.68600 29.05700 # O40 + 41 14 2 0.42380 20.68200 24.91600 29.71000 # H41 + 42 14 2 0.42380 19.95500 25.65300 28.50100 # H42 + 43 15 1 -0.84760 20.16600 20.34600 17.44500 # O43 + 44 15 2 0.42380 20.37000 19.35800 17.15100 # H44 + 45 15 2 0.42380 20.89300 20.56700 18.05800 # H45 + 46 16 1 -0.84760 24.91300 27.21500 15.65400 # O46 + 47 16 2 0.42380 24.95000 27.94200 16.40800 # H47 + 48 16 2 0.42380 25.30100 27.67200 14.82300 # H48 + 49 17 1 -0.84760 10.86100 25.07200 22.61800 # O49 + 50 17 2 0.42380 10.51500 25.85100 21.97900 # H50 + 51 17 2 0.42380 10.26100 25.02600 23.45100 # H51 + 52 18 1 -0.84760 20.41700 4.62600 14.12400 # O52 + 53 18 2 0.42380 21.28400 3.99600 14.15300 # H53 + 54 18 2 0.42380 20.66000 5.60900 13.89000 # H54 + 55 19 1 -0.84760 18.65500 9.73300 16.03500 # O55 + 56 19 2 0.42380 18.81100 9.24400 15.15200 # H56 + 57 19 2 0.42380 17.82800 10.19500 15.75000 # H57 + 58 20 1 -0.84760 15.15000 10.75800 24.17300 # O58 + 59 20 2 0.42380 14.29300 11.08600 24.54700 # H59 + 60 20 2 0.42380 15.86200 11.41200 24.55500 # H60 + 61 21 1 -0.84760 0.43700 1.33900 16.19600 # O61 + 62 21 2 0.42380 0.45100 1.11700 15.20700 # H62 + 63 21 2 0.42380 0.46400 0.48900 16.74600 # H63 + 64 22 1 -0.84760 6.40400 22.65100 26.12000 # O64 + 65 22 2 0.42380 5.42300 22.87000 26.37400 # H65 + 66 22 2 0.42380 6.11500 22.28300 25.17700 # H66 + 67 23 1 -0.84760 24.88400 20.16700 25.26600 # O67 + 68 23 2 0.42380 23.95400 20.57300 24.98900 # H68 + 69 23 2 0.42380 25.39500 21.06100 25.25300 # H69 + 70 24 1 -0.84760 13.97500 16.88900 10.88100 # O70 + 71 24 2 0.42380 13.82600 16.33300 10.04100 # H71 + 72 24 2 0.42380 13.34100 16.77700 11.59500 # H72 + 73 25 1 -0.84760 0.73200 20.62100 5.08000 # O73 + 74 25 2 0.42380 0.48100 19.75800 5.45100 # H74 + 75 25 2 0.42380 0.24900 20.62700 4.22900 # H75 + 76 26 1 -0.84760 4.21500 23.27600 20.59300 # O76 + 77 26 2 0.42380 4.32700 24.17100 20.06800 # H77 + 78 26 2 0.42380 3.75400 23.53200 21.42600 # H78 + 79 27 1 -0.84760 5.08400 20.34600 6.43100 # O79 + 80 27 2 0.42380 4.33700 20.78900 5.90200 # H80 + 81 27 2 0.42380 4.78300 19.42800 6.59200 # H81 + 82 28 1 -0.84760 9.94400 22.43600 27.21700 # O82 + 83 28 2 0.42380 9.49200 21.88100 26.52400 # H83 + 84 28 2 0.42380 10.75500 21.84300 27.39500 # H84 + 85 29 1 -0.84760 22.87700 5.01000 28.11900 # O85 + 86 29 2 0.42380 22.40000 5.25100 27.22800 # H86 + 87 29 2 0.42380 23.73300 5.50100 28.05400 # H87 + 88 30 1 -0.84760 3.15000 23.45700 0.05000 # O88 + 89 30 2 0.42380 3.63500 24.06500 0.66700 # H89 + 90 30 2 0.42380 3.49800 23.55000 -0.87900 # H90 + 91 31 1 -0.84760 21.21700 14.97600 28.85900 # O91 + 92 31 2 0.42380 22.02000 15.18800 28.28200 # H92 + 93 31 2 0.42380 21.49700 15.06100 29.85600 # H93 + 94 32 1 -0.84760 18.14800 25.30800 3.57400 # O94 + 95 32 2 0.42380 17.65500 24.94900 2.75600 # H95 + 96 32 2 0.42380 17.86600 24.72600 4.39200 # H96 + 97 33 1 -0.84760 14.78100 10.19800 19.09700 # O97 + 98 33 2 0.42380 13.87800 10.49300 19.47300 # H98 + 99 33 2 0.42380 14.43200 9.89800 18.13600 # H99 + 100 34 1 -0.84760 22.84400 2.27900 23.49900 # O100 + 101 34 2 0.42380 21.82600 2.13500 23.68400 # H101 + 102 34 2 0.42380 23.22400 1.40000 23.16000 # H102 + 103 35 1 -0.84760 21.93300 14.23900 9.87300 # O103 + 104 35 2 0.42380 21.23200 13.49500 9.81800 # H104 + 105 35 2 0.42380 21.51100 15.06800 9.33000 # H105 + 106 36 1 -0.84760 19.42400 30.06700 7.14100 # O106 + 107 36 2 0.42380 18.65200 29.34600 7.16500 # H107 + 108 36 2 0.42380 20.01100 29.93000 7.93500 # H108 + 109 37 1 -0.84760 3.00900 7.29000 19.12900 # O109 + 110 37 2 0.42380 2.34300 6.56000 19.04200 # H110 + 111 37 2 0.42380 2.52200 7.60100 20.03000 # H111 + 112 38 1 -0.84760 14.46300 13.71300 10.47200 # O112 + 113 38 2 0.42380 13.77400 13.63700 9.69500 # H113 + 114 38 2 0.42380 14.93100 14.60600 10.49300 # H114 + 115 39 1 -0.84760 26.02000 3.52800 19.78800 # O115 + 116 39 2 0.42380 25.04700 3.83300 19.83300 # H116 + 117 39 2 0.42380 26.03000 2.63800 20.28600 # H117 + 118 40 1 -0.84760 15.63500 27.14400 26.49400 # O118 + 119 40 2 0.42380 14.59800 27.43000 26.44700 # H119 + 120 40 2 0.42380 15.84700 26.35000 27.11500 # H120 + 121 41 1 -0.84760 18.20200 -0.34100 13.08200 # O121 + 122 41 2 0.42380 17.63600 0.47800 12.83100 # H122 + 123 41 2 0.42380 17.53000 -1.12700 13.03800 # H123 + 124 42 1 -0.84760 5.36400 1.04500 3.37800 # O124 + 125 42 2 0.42380 6.38800 0.72200 3.40900 # H125 + 126 42 2 0.42380 4.90900 0.71200 4.24200 # H126 + 127 43 1 -0.84760 28.56000 17.87800 12.38900 # O127 + 128 43 2 0.42380 27.75900 17.74800 12.93600 # H128 + 129 43 2 0.42380 29.36300 18.37200 12.69000 # H129 + 130 44 1 -0.84760 11.17600 9.31000 19.41400 # O130 + 131 44 2 0.42380 10.42900 8.65200 19.27900 # H131 + 132 44 2 0.42380 11.96800 8.92900 18.93400 # H132 + 133 45 1 -0.84760 23.15200 11.07200 16.86400 # O133 + 134 45 2 0.42380 23.22800 11.88000 16.23700 # H134 + 135 45 2 0.42380 22.39000 10.45900 16.53300 # H135 + 136 46 1 -0.84760 17.75000 13.43200 4.45000 # O136 + 137 46 2 0.42380 18.72000 13.75500 4.33700 # H137 + 138 46 2 0.42380 17.56000 12.78800 5.29500 # H138 + 139 47 1 -0.84760 11.70100 14.33600 17.46700 # O139 + 140 47 2 0.42380 10.83000 14.15500 17.93500 # H140 + 141 47 2 0.42380 11.64900 15.19800 16.91500 # H141 + 142 48 1 -0.84760 23.22600 -0.66600 25.69700 # O142 + 143 48 2 0.42380 23.62500 0.00100 25.06700 # H143 + 144 48 2 0.42380 22.51900 -1.05400 25.16600 # H144 + 145 49 1 -0.84760 16.94400 11.52900 17.25900 # O145 + 146 49 2 0.42380 15.97000 11.88500 17.38400 # H146 + 147 49 2 0.42380 17.65000 12.05900 17.77100 # H147 + 148 50 1 -0.84760 21.12700 26.88300 23.25400 # O148 + 149 50 2 0.42380 21.07100 25.96900 22.88900 # H149 + 150 50 2 0.42380 20.32100 27.34000 22.85300 # H150 + 151 51 1 -0.84760 13.08200 28.59900 9.25700 # O151 + 152 51 2 0.42380 12.99700 27.59400 9.03200 # H152 + 153 51 2 0.42380 13.95400 28.78900 9.61000 # H153 + 154 52 1 -0.84760 18.07800 1.87000 15.56900 # O154 + 155 52 2 0.42380 17.67300 1.18000 14.94100 # H155 + 156 52 2 0.42380 18.32700 2.58700 14.90100 # H156 + 157 53 1 -0.84760 24.21600 23.08900 20.28300 # O157 + 158 53 2 0.42380 23.81800 24.02700 20.26300 # H158 + 159 53 2 0.42380 23.51800 22.38600 20.07300 # H159 + 160 54 1 -0.84760 13.49600 3.82000 4.73600 # O160 + 161 54 2 0.42380 13.46000 4.83000 4.74500 # H161 + 162 54 2 0.42380 13.12400 3.62600 5.72000 # H162 + 163 55 1 -0.84760 9.97500 10.77100 6.90400 # O163 + 164 55 2 0.42380 10.85200 10.71000 6.43600 # H164 + 165 55 2 0.42380 9.43300 10.27900 6.27000 # H165 + 166 56 1 -0.84760 2.65800 15.47300 21.75500 # O166 + 167 56 2 0.42380 2.85400 16.12500 22.44000 # H167 + 168 56 2 0.42380 2.66900 14.51500 22.11500 # H168 + 169 57 1 -0.84760 15.23100 25.00200 15.10200 # O169 + 170 57 2 0.42380 14.21300 25.02400 15.21400 # H170 + 171 57 2 0.42380 15.55100 25.61300 15.85500 # H171 + 172 58 1 -0.84760 24.76200 12.98300 29.24400 # O172 + 173 58 2 0.42380 24.23400 13.72400 29.65700 # H173 + 174 58 2 0.42380 24.20700 12.40600 28.57500 # H174 + 175 59 1 -0.84760 16.70100 23.48200 12.88000 # O175 + 176 59 2 0.42380 16.06200 24.22400 13.24000 # H176 + 177 59 2 0.42380 17.48400 23.50400 13.57700 # H177 + 178 60 1 -0.84760 5.03400 14.97500 8.22300 # O178 + 179 60 2 0.42380 4.66600 15.74600 7.70100 # H179 + 180 60 2 0.42380 5.36300 15.29600 9.11700 # H180 + 181 61 1 -0.84760 1.41800 0.11800 13.87300 # O181 + 182 61 2 0.42380 1.81800 -0.01400 14.78800 # H182 + 183 61 2 0.42380 0.38900 -0.01500 13.84100 # H183 + 184 62 1 -0.84760 29.01700 8.90400 24.43500 # O184 + 185 62 2 0.42380 28.39400 9.21500 23.74100 # H185 + 186 62 2 0.42380 29.68600 8.45500 23.89000 # H186 + 187 63 1 -0.84760 2.46500 19.03100 23.65900 # O187 + 188 63 2 0.42380 2.12900 18.43200 24.46500 # H188 + 189 63 2 0.42380 2.69700 18.32200 22.96300 # H189 + 190 64 1 -0.84760 19.67500 19.94000 27.72700 # O190 + 191 64 2 0.42380 20.13100 20.57000 27.09800 # H191 + 192 64 2 0.42380 19.18000 20.42400 28.45600 # H192 + 193 65 1 -0.84760 10.07000 23.67200 11.11200 # O193 + 194 65 2 0.42380 10.51900 23.08700 10.45100 # H194 + 195 65 2 0.42380 9.92900 24.48200 10.59100 # H195 + 196 66 1 -0.84760 13.25600 28.34400 25.62000 # O196 + 197 66 2 0.42380 12.63100 28.05800 24.83700 # H197 + 198 66 2 0.42380 13.03800 28.09500 26.62000 # H198 + 199 67 1 -0.84760 12.37800 10.79300 10.02000 # O199 + 200 67 2 0.42380 13.11900 11.13600 9.53100 # H200 + 201 67 2 0.42380 12.39300 9.81500 9.94000 # H201 + 202 68 1 -0.84760 22.75300 14.39900 7.19200 # O202 + 203 68 2 0.42380 21.78500 14.25000 6.98600 # H203 + 204 68 2 0.42380 22.98300 13.64100 7.76400 # H204 + 205 69 1 -0.84760 21.34100 2.49900 2.14600 # O205 + 206 69 2 0.42380 20.61300 2.89500 2.74300 # H206 + 207 69 2 0.42380 22.06500 3.22600 1.83100 # H207 + 208 70 1 -0.84760 11.64800 20.29400 27.70500 # O208 + 209 70 2 0.42380 11.88800 20.72800 28.56500 # H209 + 210 70 2 0.42380 11.93400 19.29500 27.96600 # H210 + 211 71 1 -0.84760 8.01800 25.70500 -0.48800 # O211 + 212 71 2 0.42380 7.69400 25.20200 -1.34500 # H212 + 213 71 2 0.42380 9.00600 25.68300 -0.45200 # H213 + 214 72 1 -0.84760 7.55000 2.04700 1.09200 # O214 + 215 72 2 0.42380 6.78400 2.29700 1.61100 # H215 + 216 72 2 0.42380 7.30700 2.15000 0.14800 # H216 + 217 73 1 -0.84760 27.52800 17.17200 8.33700 # O217 + 218 73 2 0.42380 27.39900 16.58100 7.46000 # H218 + 219 73 2 0.42380 28.56000 16.85700 8.49500 # H219 + 220 74 1 -0.84760 25.28900 27.23700 6.77400 # O220 + 221 74 2 0.42380 24.50500 27.84000 6.69400 # H221 + 222 74 2 0.42380 25.38500 26.87200 7.72400 # H222 + 223 75 1 -0.84760 26.37200 0.72700 -0.31700 # O223 + 224 75 2 0.42380 27.34600 0.85700 -0.12100 # H224 + 225 75 2 0.42380 26.01600 1.55700 -0.75100 # H225 + 226 76 1 -0.84760 26.16200 3.51400 24.32700 # O226 + 227 76 2 0.42380 27.13500 3.66200 24.08000 # H227 + 228 76 2 0.42380 25.54600 3.30400 23.57500 # H228 + 229 77 1 -0.84760 22.84800 27.59600 29.22800 # O229 + 230 77 2 0.42380 22.08000 26.90900 29.06500 # H230 + 231 77 2 0.42380 23.41000 26.96700 29.79900 # H231 + 232 78 1 -0.84760 26.87700 3.82500 29.76200 # O232 + 233 78 2 0.42380 27.54400 3.19700 30.19100 # H233 + 234 78 2 0.42380 26.75100 3.68200 28.79700 # H234 + 235 79 1 -0.84760 13.12900 12.86500 29.25300 # O235 + 236 79 2 0.42380 12.43400 13.14800 28.59400 # H236 + 237 79 2 0.42380 13.71300 13.67800 29.38000 # H237 + 238 80 1 -0.84760 17.81000 10.12300 7.12400 # O238 + 239 80 2 0.42380 17.14200 9.62900 6.54700 # H239 + 240 80 2 0.42380 18.64100 9.88200 6.63300 # H240 + 241 81 1 -0.84760 9.11100 14.30700 26.16000 # O241 + 242 81 2 0.42380 9.94000 14.12900 26.74400 # H242 + 243 81 2 0.42380 9.26700 13.63600 25.36000 # H243 + 244 82 1 -0.84760 13.93000 18.44400 22.37300 # O244 + 245 82 2 0.42380 14.20900 18.54400 23.29400 # H245 + 246 82 2 0.42380 13.59300 19.26100 21.97000 # H246 + 247 83 1 -0.84760 0.98800 6.29300 27.55700 # O247 + 248 83 2 0.42380 0.66100 5.39100 27.24600 # H248 + 249 83 2 0.42380 1.66000 6.39900 26.78000 # H249 + 250 84 1 -0.84760 16.84600 29.24900 24.51000 # O250 + 251 84 2 0.42380 16.15500 28.61600 24.90700 # H251 + 252 84 2 0.42380 16.38300 29.46200 23.67000 # H252 + 253 85 1 -0.84760 6.35100 17.25400 0.83800 # O253 + 254 85 2 0.42380 6.87700 16.45400 0.54800 # H254 + 255 85 2 0.42380 5.40800 16.85300 0.93700 # H255 + 256 86 1 -0.84760 14.99500 7.90900 26.86300 # O256 + 257 86 2 0.42380 14.70900 8.05000 27.84900 # H257 + 258 86 2 0.42380 15.49800 7.06200 26.68800 # H258 + 259 87 1 -0.84760 6.14100 28.63000 15.75200 # O259 + 260 87 2 0.42380 6.30700 28.03400 16.59200 # H260 + 261 87 2 0.42380 6.77000 29.40400 15.69900 # H261 + 262 88 1 -0.84760 18.65800 14.29600 15.00200 # O262 + 263 88 2 0.42380 19.31800 13.86500 14.37000 # H263 + 264 88 2 0.42380 19.01100 15.24500 15.11500 # H264 + 265 89 1 -0.84760 2.33700 9.03400 14.49900 # O265 + 266 89 2 0.42380 2.08400 8.54100 13.66900 # H266 + 267 89 2 0.42380 2.36800 8.41700 15.26300 # H267 + 268 90 1 -0.84760 25.94900 19.31100 7.58500 # O268 + 269 90 2 0.42380 25.95900 19.30900 8.56800 # H269 + 270 90 2 0.42380 26.66100 18.63500 7.29300 # H270 + 271 91 1 -0.84760 26.62900 23.49600 12.15600 # O271 + 272 91 2 0.42380 26.96300 22.56800 12.24600 # H272 + 273 91 2 0.42380 27.03600 24.04000 12.88200 # H273 + 274 92 1 -0.84760 11.48400 2.32400 12.59700 # O274 + 275 92 2 0.42380 11.49400 2.62300 11.62500 # H275 + 276 92 2 0.42380 10.74900 1.62200 12.64000 # H276 + 277 93 1 -0.84760 -0.51800 22.20500 8.88100 # O277 + 278 93 2 0.42380 0.24900 22.55800 8.29500 # H278 + 279 93 2 0.42380 -0.27400 22.11400 9.87800 # H279 + 280 94 1 -0.84760 9.41700 -0.60000 -0.33000 # O280 + 281 94 2 0.42380 10.01100 0.03500 0.20200 # H281 + 282 94 2 0.42380 9.97700 -0.71600 -1.18400 # H282 + 283 95 1 -0.84760 8.06700 23.08800 12.87300 # O283 + 284 95 2 0.42380 8.84700 23.20800 12.17200 # H284 + 285 95 2 0.42380 7.89200 23.83600 13.54200 # H285 + 286 96 1 -0.84760 19.90600 3.96900 6.75100 # O286 + 287 96 2 0.42380 18.90100 3.94900 6.74700 # H287 + 288 96 2 0.42380 20.12800 4.43200 5.89700 # H288 + 289 97 1 -0.84760 3.70900 13.47300 12.79500 # O289 + 290 97 2 0.42380 3.70700 14.47600 12.92800 # H290 + 291 97 2 0.42380 3.13100 13.06300 13.52000 # H291 + 292 98 1 -0.84760 8.17700 5.56700 22.63100 # O292 + 293 98 2 0.42380 8.71100 5.65200 21.78000 # H293 + 294 98 2 0.42380 8.01600 4.52300 22.82400 # H294 + 295 99 1 -0.84760 7.65500 25.50400 20.30100 # O295 + 296 99 2 0.42380 7.86200 24.54100 20.00200 # H296 + 297 99 2 0.42380 6.97300 25.75100 19.67200 # H297 + 298 100 1 -0.84760 6.31400 24.43900 23.75900 # O298 + 299 100 2 0.42380 7.09900 23.87900 24.14400 # H299 + 300 100 2 0.42380 5.98400 23.90400 22.85200 # H300 + 301 101 1 -0.84760 8.63500 28.19600 26.74700 # O301 + 302 101 2 0.42380 7.96500 28.81900 27.07100 # H302 + 303 101 2 0.42380 8.75300 27.62000 27.55300 # H303 + 304 102 1 -0.84760 19.14700 8.19000 5.28700 # O304 + 305 102 2 0.42380 18.85200 7.35700 4.87600 # H305 + 306 102 2 0.42380 19.18000 8.94500 4.66300 # H306 + 307 103 1 -0.84760 0.72400 17.61700 5.56500 # O307 + 308 103 2 0.42380 -0.26000 17.53400 5.32500 # H308 + 309 103 2 0.42380 1.10200 16.67300 5.43600 # H309 + 310 104 1 -0.84760 26.48800 8.52500 17.62200 # O310 + 311 104 2 0.42380 27.25900 8.55600 16.94700 # H311 + 312 104 2 0.42380 25.98500 9.40700 17.53100 # H312 + 313 105 1 -0.84760 26.37000 26.56500 22.42700 # O313 + 314 105 2 0.42380 26.68000 26.97900 23.27200 # H314 + 315 105 2 0.42380 26.30300 25.54500 22.60600 # H315 + 316 106 1 -0.84760 6.20600 2.71900 27.71200 # O316 + 317 106 2 0.42380 6.54100 1.75300 27.92300 # H317 + 318 106 2 0.42380 6.84600 3.50400 27.85600 # H318 + 319 107 1 -0.84760 21.33300 8.71000 15.84400 # O319 + 320 107 2 0.42380 20.96500 8.84900 16.82000 # H320 + 321 107 2 0.42380 20.61300 8.51500 15.16100 # H321 + 322 108 1 -0.84760 23.04900 8.30800 25.42000 # O322 + 323 108 2 0.42380 23.30100 8.29700 24.44100 # H323 + 324 108 2 0.42380 22.55500 7.46700 25.63500 # H324 + 325 109 1 -0.84760 28.13700 21.97400 25.15200 # O325 + 326 109 2 0.42380 28.78100 22.74100 25.08500 # H326 + 327 109 2 0.42380 28.61900 21.10300 25.05900 # H327 + 328 110 1 -0.84760 20.07800 11.96900 9.22700 # O328 + 329 110 2 0.42380 20.64100 11.21200 9.59300 # H329 + 330 110 2 0.42380 19.66900 11.65800 8.45100 # H330 + 331 111 1 -0.84760 3.18100 5.04200 11.07600 # O331 + 332 111 2 0.42380 3.60900 4.28400 11.50800 # H332 + 333 111 2 0.42380 2.80100 4.47300 10.35300 # H333 + 334 112 1 -0.84760 16.12500 13.48300 14.11400 # O334 + 335 112 2 0.42380 16.64900 14.27700 13.69200 # H335 + 336 112 2 0.42380 16.88300 13.06900 14.61500 # H336 + 337 113 1 -0.84760 7.02000 26.55300 3.72800 # O337 + 338 113 2 0.42380 6.55400 27.16200 4.38600 # H338 + 339 113 2 0.42380 6.75100 25.66300 4.02100 # H339 + 340 114 1 -0.84760 29.22600 18.67700 20.15300 # O340 + 341 114 2 0.42380 29.49900 19.55900 19.67000 # H341 + 342 114 2 0.42380 28.21200 18.73200 20.32800 # H342 + 343 115 1 -0.84760 12.55200 21.55000 5.16200 # O343 + 344 115 2 0.42380 12.98500 21.34800 6.11700 # H344 + 345 115 2 0.42380 11.61400 21.12600 5.16400 # H345 + 346 116 1 -0.84760 8.63800 -0.13900 14.55100 # O346 + 347 116 2 0.42380 9.31700 -0.85400 14.91100 # H347 + 348 116 2 0.42380 8.44100 0.61400 15.22200 # H348 + 349 117 1 -0.84760 9.80400 16.72700 17.47600 # O349 + 350 117 2 0.42380 9.76900 15.83000 17.87100 # H350 + 351 117 2 0.42380 8.86600 17.01200 17.34500 # H351 + 352 118 1 -0.84760 24.87300 17.76500 1.87500 # O352 + 353 118 2 0.42380 25.13400 17.11300 2.61000 # H353 + 354 118 2 0.42380 25.10600 18.69800 2.25800 # H354 + 355 119 1 -0.84760 4.18000 0.08600 5.94900 # O355 + 356 119 2 0.42380 3.29100 0.24300 5.70200 # H356 + 357 119 2 0.42380 3.96600 -0.00400 6.91500 # H357 + 358 120 1 -0.84760 1.99900 2.18900 3.47900 # O358 + 359 120 2 0.42380 1.53600 1.62500 4.19900 # H359 + 360 120 2 0.42380 2.34400 1.44200 2.87900 # H360 + 361 121 1 -0.84760 -0.82900 21.94500 2.26100 # O361 + 362 121 2 0.42380 -1.18100 22.83300 2.44400 # H362 + 363 121 2 0.42380 0.13700 22.20000 2.12600 # H363 + 364 122 1 -0.84760 4.26200 17.28500 15.28600 # O364 + 365 122 2 0.42380 3.38100 17.59300 14.78700 # H365 + 366 122 2 0.42380 4.52500 18.08500 15.79900 # H366 + 367 123 1 -0.84760 0.45100 7.25700 2.06100 # O367 + 368 123 2 0.42380 -0.59500 7.26500 2.17100 # H368 + 369 123 2 0.42380 0.84600 6.33100 2.34900 # H369 + 370 124 1 -0.84760 8.49200 9.41900 4.76400 # O370 + 371 124 2 0.42380 7.94000 10.31500 4.61500 # H371 + 372 124 2 0.42380 8.45800 9.16200 3.75100 # H372 + 373 125 1 -0.84760 11.76800 10.07000 21.96000 # O373 + 374 125 2 0.42380 11.79800 10.94000 22.43500 # H374 + 375 125 2 0.42380 11.34400 10.44800 21.11100 # H375 + 376 126 1 -0.84760 17.93100 4.97400 5.34200 # O376 + 377 126 2 0.42380 18.04500 5.26500 4.42800 # H377 + 378 126 2 0.42380 17.12300 4.42000 5.46300 # H378 + 379 127 1 -0.84760 13.94600 21.75000 29.55100 # O379 + 380 127 2 0.42380 14.62600 21.14800 29.12000 # H380 + 381 127 2 0.42380 14.49600 22.33000 30.24200 # H381 + 382 128 1 -0.84760 29.95900 11.24400 15.21000 # O382 + 383 128 2 0.42380 30.53100 10.86500 15.96200 # H383 + 384 128 2 0.42380 29.23500 11.90500 15.49100 # H384 + 385 129 1 -0.84760 28.73000 25.04000 8.86300 # O385 + 386 129 2 0.42380 28.78500 26.00000 8.63100 # H386 + 387 129 2 0.42380 27.89900 24.81400 9.29500 # H387 + 388 130 1 -0.84760 3.12200 2.48500 12.69900 # O388 + 389 130 2 0.42380 2.57300 2.16500 11.87300 # H389 + 390 130 2 0.42380 2.55100 2.74700 13.53700 # H390 + 391 131 1 -0.84760 12.19900 20.81200 19.63000 # O391 + 392 131 2 0.42380 12.29500 21.40500 18.85100 # H392 + 393 131 2 0.42380 12.18600 19.85400 19.25400 # H393 + 394 132 1 -0.84760 4.27600 17.63700 29.06200 # O394 + 395 132 2 0.42380 4.43200 18.55600 28.70900 # H395 + 396 132 2 0.42380 3.92800 17.79500 30.01900 # H396 + 397 133 1 -0.84760 26.46500 15.91900 13.35100 # O397 + 398 133 2 0.42380 27.22300 15.38700 12.91400 # H398 + 399 133 2 0.42380 26.22500 15.35000 14.23300 # H399 + 400 134 1 -0.84760 1.06200 24.79400 6.30800 # O400 + 401 134 2 0.42380 0.82600 24.56300 5.37600 # H401 + 402 134 2 0.42380 0.19000 24.65200 6.67800 # H402 + 403 135 1 -0.84760 11.84500 4.80100 11.12900 # O403 + 404 135 2 0.42380 10.98000 4.34500 10.76500 # H404 + 405 135 2 0.42380 11.92900 5.63200 10.54100 # H405 + 406 136 1 -0.84760 6.61800 22.08500 7.40900 # O406 + 407 136 2 0.42380 6.09300 22.97200 7.17300 # H407 + 408 136 2 0.42380 6.00700 21.57900 6.77300 # H408 + 409 137 1 -0.84760 16.37500 17.74800 14.02800 # O409 + 410 137 2 0.42380 16.44400 18.21800 13.07300 # H410 + 411 137 2 0.42380 15.38000 17.35200 14.09800 # H411 + 412 138 1 -0.84760 14.02800 0.14900 8.99200 # O412 + 413 138 2 0.42380 14.01000 0.15700 10.00200 # H413 + 414 138 2 0.42380 13.33900 0.85800 8.75100 # H414 + 415 139 1 -0.84760 19.23800 27.39900 14.06700 # O415 + 416 139 2 0.42380 20.23700 27.14300 14.17200 # H416 + 417 139 2 0.42380 19.00400 28.32900 13.75900 # H417 + 418 140 1 -0.84760 12.14500 10.56400 1.61100 # O418 + 419 140 2 0.42380 11.37500 10.03100 1.35600 # H419 + 420 140 2 0.42380 12.73500 10.54900 0.79300 # H420 + 421 141 1 -0.84760 6.73400 2.81800 6.80400 # O421 + 422 141 2 0.42380 6.04500 2.72400 6.06900 # H422 + 423 141 2 0.42380 7.65700 3.04600 6.47000 # H423 + 424 142 1 -0.84760 22.08100 23.34800 11.46100 # O424 + 425 142 2 0.42380 21.44900 22.98900 10.74200 # H425 + 426 142 2 0.42380 22.07100 24.29500 11.75100 # H426 + 427 143 1 -0.84760 -0.99100 19.78800 11.83600 # O427 + 428 143 2 0.42380 -0.66600 18.93700 11.38700 # H428 + 429 143 2 0.42380 -0.64100 20.63300 11.27700 # H429 + 430 144 1 -0.84760 27.38400 19.96600 29.05800 # O430 + 431 144 2 0.42380 26.75100 19.42900 29.70600 # H431 + 432 144 2 0.42380 26.96000 19.77900 28.17800 # H432 + 433 145 1 -0.84760 25.28400 23.63700 29.75800 # O433 + 434 145 2 0.42380 24.96000 22.87800 30.33000 # H434 + 435 145 2 0.42380 24.64700 24.39500 29.67000 # H435 + 436 146 1 -0.84760 5.18400 13.82300 3.14900 # O436 + 437 146 2 0.42380 5.79400 14.03200 3.89700 # H437 + 438 146 2 0.42380 4.24500 13.85600 3.56600 # H438 + 439 147 1 -0.84760 12.96500 23.45100 0.42800 # O439 + 440 147 2 0.42380 13.25300 23.68300 1.36400 # H440 + 441 147 2 0.42380 12.98600 22.44300 0.18800 # H441 + 442 148 1 -0.84760 3.53300 9.15500 9.68400 # O442 + 443 148 2 0.42380 3.01800 9.94300 9.63500 # H443 + 444 148 2 0.42380 4.29800 9.31000 10.26700 # H444 + 445 149 1 -0.84760 28.25100 24.91700 24.44700 # O445 + 446 149 2 0.42380 27.37900 24.50200 24.43100 # H446 + 447 149 2 0.42380 28.25600 25.80300 23.96900 # H447 + 448 150 1 -0.84760 10.72700 2.36700 0.31000 # O448 + 449 150 2 0.42380 11.29000 2.93300 0.86400 # H449 + 450 150 2 0.42380 10.60300 1.60100 0.94600 # H450 + 451 151 1 -0.84760 28.15000 24.26400 1.63500 # O451 + 452 151 2 0.42380 28.58300 23.65800 0.88000 # H452 + 453 151 2 0.42380 28.49800 24.27500 2.57800 # H453 + 454 152 1 -0.84760 0.25300 15.93600 20.26800 # O454 + 455 152 2 0.42380 0.28800 16.26900 19.32800 # H455 + 456 152 2 0.42380 0.25400 14.92700 20.17800 # H456 + 457 153 1 -0.84760 24.94900 10.19400 19.81100 # O457 + 458 153 2 0.42380 25.08300 11.14900 19.29000 # H458 + 459 153 2 0.42380 25.92900 9.72800 19.87000 # H459 + 460 154 1 -0.84760 13.75400 28.21500 16.12700 # O460 + 461 154 2 0.42380 14.06700 28.71900 16.93600 # H461 + 462 154 2 0.42380 14.51200 28.22500 15.41900 # H462 + 463 155 1 -0.84760 7.07800 24.11600 18.01800 # O463 + 464 155 2 0.42380 7.09000 23.80600 17.08400 # H464 + 465 155 2 0.42380 6.13600 23.82500 18.26600 # H465 + 466 156 1 -0.84760 27.85600 19.17100 5.01000 # O466 + 467 156 2 0.42380 27.32200 19.89900 4.55900 # H467 + 468 156 2 0.42380 28.38500 19.53800 5.78900 # H468 + 469 157 1 -0.84760 20.62000 22.45300 29.96600 # O469 + 470 157 2 0.42380 20.91100 22.66600 30.86200 # H470 + 471 157 2 0.42380 20.42400 21.47400 30.08200 # H471 + 472 158 1 -0.84760 21.08700 4.56800 21.66900 # O472 + 473 158 2 0.42380 20.58700 4.15300 20.84400 # H473 + 474 158 2 0.42380 21.88400 4.06300 22.08300 # H474 + 475 159 1 -0.84760 15.06800 27.00000 8.00000 # O475 + 476 159 2 0.42380 16.03500 26.91000 8.40000 # H476 + 477 159 2 0.42380 15.16800 27.69600 7.21300 # H477 + 478 160 1 -0.84760 14.23200 10.03000 -0.52200 # O478 + 479 160 2 0.42380 13.77600 10.13200 -1.47600 # H479 + 480 160 2 0.42380 14.84100 9.20900 -0.57500 # H480 + 481 161 1 -0.84760 12.95200 7.21000 1.31400 # O481 + 482 161 2 0.42380 12.27200 7.81500 0.88100 # H482 + 483 161 2 0.42380 13.40600 7.71600 2.08100 # H483 + 484 162 1 -0.84760 22.41200 13.99100 4.68900 # O484 + 485 162 2 0.42380 21.52000 14.48700 4.62100 # H485 + 486 162 2 0.42380 22.76900 14.50200 5.49300 # H486 + 487 163 1 -0.84760 20.07900 8.68600 -0.60200 # O487 + 488 163 2 0.42380 19.81200 9.66100 -0.32600 # H488 + 489 163 2 0.42380 21.07400 8.50100 -0.62000 # H489 + 490 164 1 -0.84760 19.40500 3.95400 19.50300 # O490 + 491 164 2 0.42380 19.26400 4.92100 19.20300 # H491 + 492 164 2 0.42380 20.12100 3.71500 18.83700 # H492 + 493 165 1 -0.84760 22.82000 22.39200 5.52600 # O493 + 494 165 2 0.42380 22.93100 22.21200 4.54800 # H494 + 495 165 2 0.42380 21.83200 22.52500 5.60500 # H495 + 496 166 1 -0.84760 18.20800 2.98900 13.31600 # O496 + 497 166 2 0.42380 18.46600 3.96300 13.54700 # H497 + 498 166 2 0.42380 19.10000 2.48200 12.98400 # H498 + 499 167 1 -0.84760 12.98300 16.84500 17.12000 # O499 + 500 167 2 0.42380 13.53200 16.05800 17.01200 # H500 + 501 167 2 0.42380 12.94100 16.80900 18.15700 # H501 + 502 168 1 -0.84760 28.53200 11.25600 19.74200 # O502 + 503 168 2 0.42380 27.95500 11.67300 20.42700 # H503 + 504 168 2 0.42380 29.21600 11.99200 19.69500 # H504 + 505 169 1 -0.84760 8.23900 7.27400 24.66700 # O505 + 506 169 2 0.42380 8.18800 6.37900 24.20300 # H506 + 507 169 2 0.42380 7.39500 7.37600 25.12300 # H507 + 508 170 1 -0.84760 12.41600 1.34000 16.33800 # O508 + 509 170 2 0.42380 12.98100 1.63300 15.55000 # H509 + 510 170 2 0.42380 11.69600 0.62700 16.12300 # H510 + 511 171 1 -0.84760 14.44100 23.55200 5.37700 # O511 + 512 171 2 0.42380 13.91700 22.64700 5.08900 # H512 + 513 171 2 0.42380 13.90100 24.38200 4.97100 # H513 + 514 172 1 -0.84760 12.34700 8.67200 4.62100 # O514 + 515 172 2 0.42380 11.63100 8.10600 4.17100 # H515 + 516 172 2 0.42380 12.19400 9.60700 4.21400 # H516 + 517 173 1 -0.84760 21.73500 15.69400 14.61100 # O517 + 518 173 2 0.42380 21.52000 14.78400 14.25600 # H518 + 519 173 2 0.42380 20.84300 16.25900 14.41100 # H519 + 520 174 1 -0.84760 26.28500 15.55500 5.80200 # O520 + 521 174 2 0.42380 26.92100 14.91600 6.29300 # H521 + 522 174 2 0.42380 25.34200 15.42500 6.13300 # H522 + 523 175 1 -0.84760 0.67500 13.59000 5.58300 # O523 + 524 175 2 0.42380 0.91700 14.27500 6.28100 # H524 + 525 175 2 0.42380 1.48100 12.98000 5.49700 # H525 + 526 176 1 -0.84760 20.33500 13.51000 22.74200 # O526 + 527 176 2 0.42380 19.46500 14.01700 22.72500 # H527 + 528 176 2 0.42380 20.47400 12.96200 23.58900 # H528 + 529 177 1 -0.84760 2.31300 28.51700 19.81500 # O529 + 530 177 2 0.42380 2.01300 28.77000 20.71500 # H530 + 531 177 2 0.42380 1.41400 28.34100 19.44700 # H531 + 532 178 1 -0.84760 17.72900 10.27700 29.56200 # O532 + 533 178 2 0.42380 17.02900 10.58200 28.90200 # H533 + 534 178 2 0.42380 18.13500 11.06100 30.09800 # H534 + 535 179 1 -0.84760 20.90800 0.61300 24.55000 # O535 + 536 179 2 0.42380 20.06900 0.85600 24.97000 # H536 + 537 179 2 0.42380 21.51200 0.43400 25.38100 # H537 + 538 180 1 -0.84760 14.32000 12.90500 22.84800 # O538 + 539 180 2 0.42380 15.00600 12.09300 22.88700 # H539 + 540 180 2 0.42380 14.64800 13.61500 22.18900 # H540 + 541 181 1 -0.84760 17.51700 13.05600 20.10500 # O541 + 542 181 2 0.42380 17.37800 13.44500 21.06400 # H542 + 543 181 2 0.42380 17.77500 12.04900 20.18600 # H543 + 544 182 1 -0.84760 3.57500 24.92100 2.24600 # O544 + 545 182 2 0.42380 3.89100 23.94600 2.43100 # H545 + 546 182 2 0.42380 3.28600 25.02900 3.18900 # H546 + 547 183 1 -0.84760 0.04900 17.15900 25.41000 # O547 + 548 183 2 0.42380 -0.72300 17.23100 24.64300 # H548 + 549 183 2 0.42380 -0.10900 17.88300 26.14300 # H549 + 550 184 1 -0.84760 8.81400 28.47200 19.58400 # O550 + 551 184 2 0.42380 8.22100 28.47700 20.40300 # H551 + 552 184 2 0.42380 8.64600 29.21600 18.92700 # H552 + 553 185 1 -0.84760 26.32400 22.12200 20.94200 # O553 + 554 185 2 0.42380 25.96700 21.18200 20.85800 # H554 + 555 185 2 0.42380 25.46800 22.66900 20.99100 # H555 + 556 186 1 -0.84760 29.84500 6.84900 20.25200 # O556 + 557 186 2 0.42380 29.85200 7.58200 20.95000 # H557 + 558 186 2 0.42380 29.05600 6.26000 20.42900 # H558 + 559 187 1 -0.84760 15.72100 10.32200 9.03100 # O559 + 560 187 2 0.42380 15.43600 9.57500 9.60800 # H560 + 561 187 2 0.42380 16.48100 9.96400 8.45500 # H561 + 562 188 1 -0.84760 16.02400 27.53700 5.47600 # O562 + 563 188 2 0.42380 15.89400 28.57400 5.36000 # H563 + 564 188 2 0.42380 17.09100 27.44100 5.69100 # H564 + 565 189 1 -0.84760 27.45800 12.03000 15.51400 # O565 + 566 189 2 0.42380 27.26400 12.99000 15.88700 # H566 + 567 189 2 0.42380 26.53200 11.59100 15.36700 # H567 + 568 190 1 -0.84760 1.37300 -0.01600 10.70200 # O568 + 569 190 2 0.42380 0.57500 0.20500 11.24100 # H569 + 570 190 2 0.42380 2.08300 -0.23200 11.40500 # H570 + 571 191 1 -0.84760 5.14400 4.21600 2.65800 # O571 + 572 191 2 0.42380 5.90100 4.44800 3.27700 # H572 + 573 191 2 0.42380 4.76000 3.40200 3.10100 # H573 + 574 192 1 -0.84760 13.62400 15.89200 13.41800 # O574 + 575 192 2 0.42380 13.21500 14.97700 13.20500 # H575 + 576 192 2 0.42380 14.14000 15.69800 14.27200 # H576 + 577 193 1 -0.84760 18.02700 18.45900 22.76700 # O577 + 578 193 2 0.42380 17.89700 18.24600 21.77000 # H578 + 579 193 2 0.42380 17.55900 17.74400 23.29200 # H579 + 580 194 1 -0.84760 14.03500 6.72100 3.82200 # O580 + 581 194 2 0.42380 13.07200 6.52400 4.10400 # H581 + 582 194 2 0.42380 14.22500 7.76000 3.75200 # H582 + 583 195 1 -0.84760 2.70000 26.70500 7.58000 # O583 + 584 195 2 0.42380 1.95400 26.02000 7.52700 # H584 + 585 195 2 0.42380 2.31900 27.44700 6.99500 # H585 + 586 196 1 -0.84760 17.29000 2.40200 1.41600 # O586 + 587 196 2 0.42380 16.56800 1.69800 1.37200 # H587 + 588 196 2 0.42380 16.67500 3.13600 1.64500 # H588 + 589 197 1 -0.84760 7.17700 20.82500 4.18200 # O589 + 590 197 2 0.42380 6.56700 20.63700 3.36400 # H590 + 591 197 2 0.42380 6.70600 20.48200 5.01000 # H591 + 592 198 1 -0.84760 4.18400 1.70500 18.56600 # O592 + 593 198 2 0.42380 3.78400 2.57600 18.83800 # H593 + 594 198 2 0.42380 4.59200 1.51500 19.45500 # H594 + 595 199 1 -0.84760 18.38500 22.81800 15.15900 # O595 + 596 199 2 0.42380 19.16100 22.19800 14.80500 # H596 + 597 199 2 0.42380 18.92800 23.67600 15.39900 # H597 + 598 200 1 -0.84760 15.22900 1.69200 22.93500 # O598 + 599 200 2 0.42380 16.02600 1.07100 22.84900 # H599 + 600 200 2 0.42380 15.04700 2.20500 22.07000 # H600 + 601 201 1 -0.84760 9.24200 2.19500 20.13300 # O601 + 602 201 2 0.42380 9.04300 3.12300 19.68400 # H602 + 603 201 2 0.42380 9.18200 2.31400 21.15400 # H603 + 604 202 1 -0.84760 21.21100 0.75700 16.20800 # O604 + 605 202 2 0.42380 20.90300 1.03400 15.29800 # H605 + 606 202 2 0.42380 21.47400 1.70800 16.57100 # H606 + 607 203 1 -0.84760 25.26300 28.96100 3.71900 # O607 + 608 203 2 0.42380 25.43100 28.36300 2.94900 # H608 + 609 203 2 0.42380 25.45900 28.49800 4.62100 # H609 + 610 204 1 -0.84760 15.04500 19.70400 3.35100 # O610 + 611 204 2 0.42380 14.63900 18.78900 3.54500 # H611 + 612 204 2 0.42380 14.46700 20.51800 3.50700 # H612 + 613 205 1 -0.84760 13.49100 8.86700 17.02500 # O613 + 614 205 2 0.42380 13.34500 7.90100 17.19000 # H614 + 615 205 2 0.42380 12.54900 8.93000 16.57100 # H615 + 616 206 1 -0.84760 18.49100 8.02900 13.87300 # O616 + 617 206 2 0.42380 17.91200 8.63900 13.22400 # H617 + 618 206 2 0.42380 18.49600 7.14700 13.40200 # H618 + 619 207 1 -0.84760 18.45500 14.51300 0.43900 # O619 + 620 207 2 0.42380 19.26700 15.00700 0.61200 # H620 + 621 207 2 0.42380 18.25500 13.99200 1.24800 # H621 + 622 208 1 -0.84760 26.66300 14.98700 10.26100 # O622 + 623 208 2 0.42380 27.33900 14.54400 9.72300 # H623 + 624 208 2 0.42380 27.08300 15.84000 10.58400 # H624 + 625 209 1 -0.84760 0.79200 15.54100 14.80600 # O625 + 626 209 2 0.42380 0.60500 16.35400 15.34800 # H626 + 627 209 2 0.42380 1.59700 14.98900 14.96500 # H627 + 628 210 1 -0.84760 1.24800 0.86300 19.58500 # O628 + 629 210 2 0.42380 1.74900 0.15400 19.04400 # H629 + 630 210 2 0.42380 2.03000 1.40800 19.93700 # H630 + 631 211 1 -0.84760 17.41700 16.44900 16.91800 # O631 + 632 211 2 0.42380 16.58900 16.94000 17.26700 # H632 + 633 211 2 0.42380 17.15400 15.56300 16.52200 # H633 + 634 212 1 -0.84760 12.61100 -0.74700 0.78800 # O634 + 635 212 2 0.42380 12.83900 -1.64100 0.34800 # H635 + 636 212 2 0.42380 12.99300 -0.02800 0.20700 # H636 + 637 213 1 -0.84760 8.58700 15.73300 6.99100 # O637 + 638 213 2 0.42380 8.15000 15.39900 7.84600 # H638 + 639 213 2 0.42380 7.96000 16.26500 6.44500 # H639 + 640 214 1 -0.84760 0.10100 8.84700 9.14300 # O640 + 641 214 2 0.42380 -0.12000 9.30800 8.23200 # H641 + 642 214 2 0.42380 0.69600 8.07800 9.00700 # H642 + 643 215 1 -0.84760 7.83200 19.40900 21.52800 # O643 + 644 215 2 0.42380 8.82700 19.74400 21.54000 # H644 + 645 215 2 0.42380 7.43400 19.98000 22.32400 # H645 + 646 216 1 -0.84760 24.88300 29.08000 22.32600 # O646 + 647 216 2 0.42380 24.29100 28.30800 22.06700 # H647 + 648 216 2 0.42380 25.66200 28.58900 22.73400 # H648 + 649 217 1 -0.84760 24.14600 8.41900 28.35000 # O649 + 650 217 2 0.42380 23.90300 8.83400 29.30300 # H650 + 651 217 2 0.42380 23.28300 8.15100 27.78300 # H651 + 652 218 1 -0.84760 2.71000 17.46700 19.40600 # O652 + 653 218 2 0.42380 1.81500 17.24400 19.87200 # H653 + 654 218 2 0.42380 3.27000 16.66600 19.10500 # H654 + 655 219 1 -0.84760 12.08300 1.09900 6.91900 # O655 + 656 219 2 0.42380 12.36500 1.00800 5.93100 # H656 + 657 219 2 0.42380 11.27700 0.50000 6.95900 # H657 + 658 220 1 -0.84760 28.61200 16.27300 0.63700 # O658 + 659 220 2 0.42380 28.64200 16.41000 -0.42900 # H659 + 660 220 2 0.42380 27.66700 16.38600 0.82000 # H660 + 661 221 1 -0.84760 17.84500 10.35800 3.96000 # O661 + 662 221 2 0.42380 18.37600 9.84700 3.27200 # H662 + 663 221 2 0.42380 17.85400 11.36300 3.75400 # H663 + 664 222 1 -0.84760 16.54600 5.94400 19.90700 # O664 + 665 222 2 0.42380 15.56700 5.95500 20.04000 # H665 + 666 222 2 0.42380 16.76000 5.76200 18.98500 # H666 + 667 223 1 -0.84760 4.76800 14.87500 19.66700 # O667 + 668 223 2 0.42380 5.24500 14.35600 18.98900 # H668 + 669 223 2 0.42380 5.00700 14.59300 20.60100 # H669 + 670 224 1 -0.84760 8.43700 26.26700 11.43900 # O670 + 671 224 2 0.42380 7.88700 26.77400 12.09200 # H671 + 672 224 2 0.42380 9.34900 26.38400 11.65300 # H672 + 673 225 1 -0.84760 11.14500 19.49700 12.30900 # O673 + 674 225 2 0.42380 10.87400 19.72900 11.26700 # H674 + 675 225 2 0.42380 10.49900 20.14800 12.82300 # H675 + 676 226 1 -0.84760 7.26600 12.28000 9.35500 # O676 + 677 226 2 0.42380 8.30600 12.10900 9.33800 # H677 + 678 226 2 0.42380 7.02700 11.30500 9.55000 # H678 + 679 227 1 -0.84760 13.84500 4.44700 12.97500 # O679 + 680 227 2 0.42380 13.17600 4.57100 13.69400 # H680 + 681 227 2 0.42380 13.26800 4.78100 12.26700 # H681 + 682 228 1 -0.84760 20.55100 18.83200 2.05500 # O682 + 683 228 2 0.42380 21.07300 18.02000 1.61900 # H683 + 684 228 2 0.42380 21.19500 19.63100 2.07600 # H684 + 685 229 1 -0.84760 14.23000 17.12400 4.30100 # O685 + 686 229 2 0.42380 14.08000 16.88000 5.37800 # H686 + 687 229 2 0.42380 14.54000 16.21400 3.84000 # H687 + 688 230 1 -0.84760 26.33600 20.79200 15.28500 # O688 + 689 230 2 0.42380 26.85300 19.93300 15.49500 # H689 + 690 230 2 0.42380 25.33100 20.70500 15.65100 # H690 + 691 231 1 -0.84760 1.50800 24.66300 11.21700 # O691 + 692 231 2 0.42380 1.28100 23.70300 11.23800 # H692 + 693 231 2 0.42380 0.99500 25.08100 10.41200 # H693 + 694 232 1 -0.84760 27.31800 3.87100 4.07400 # O694 + 695 232 2 0.42380 26.88700 4.80700 3.98200 # H695 + 696 232 2 0.42380 28.30400 4.07500 3.91500 # H696 + 697 233 1 -0.84760 5.79500 1.18200 23.83900 # O697 + 698 233 2 0.42380 6.32100 0.52000 23.26500 # H698 + 699 233 2 0.42380 5.98300 0.97000 24.82700 # H699 + 700 234 1 -0.84760 16.77600 21.32800 17.80500 # O700 + 701 234 2 0.42380 16.36100 20.57400 18.36000 # H701 + 702 234 2 0.42380 16.67300 21.14200 16.82600 # H702 + 703 235 1 -0.84760 20.02700 15.39000 25.15800 # O703 + 704 235 2 0.42380 20.28900 15.71600 26.05600 # H704 + 705 235 2 0.42380 19.97300 16.10800 24.42100 # H705 + 706 236 1 -0.84760 13.95400 5.18600 18.25200 # O706 + 707 236 2 0.42380 13.30500 4.41500 18.35000 # H707 + 708 236 2 0.42380 14.61300 4.73700 17.60200 # H708 + 709 237 1 -0.84760 24.35500 23.90700 15.00000 # O709 + 710 237 2 0.42380 24.29100 23.18100 14.25900 # H710 + 711 237 2 0.42380 23.98000 24.80500 14.76100 # H711 + 712 238 1 -0.84760 3.90100 27.25500 28.80500 # O712 + 713 238 2 0.42380 4.67800 27.81800 28.66400 # H713 + 714 238 2 0.42380 3.09100 27.67000 28.47200 # H714 + 715 239 1 -0.84760 16.57600 21.68400 25.60700 # O715 + 716 239 2 0.42380 16.06100 22.35300 24.98200 # H716 + 717 239 2 0.42380 15.94100 21.09800 26.11700 # H717 + 718 240 1 -0.84760 7.58300 17.01700 28.05600 # O718 + 719 240 2 0.42380 7.26900 16.56200 27.14400 # H719 + 720 240 2 0.42380 7.01900 17.03500 28.94900 # H720 + 721 241 1 -0.84760 21.80100 19.55400 -0.55100 # O721 + 722 241 2 0.42380 22.56300 20.00900 -1.05400 # H722 + 723 241 2 0.42380 21.04200 19.66900 -1.12700 # H723 + 724 242 1 -0.84760 11.93600 26.16500 8.75000 # O724 + 725 242 2 0.42380 12.22500 25.69700 7.91000 # H725 + 726 242 2 0.42380 12.48100 25.67400 9.46100 # H726 + 727 243 1 -0.84760 24.78600 16.49500 21.39500 # O727 + 728 243 2 0.42380 25.51200 17.20000 21.44300 # H728 + 729 243 2 0.42380 23.94700 16.91800 21.63000 # H729 + 730 244 1 -0.84760 3.50900 13.39600 23.81400 # O730 + 731 244 2 0.42380 4.27000 13.86100 24.41900 # H731 + 732 244 2 0.42380 4.17400 13.00200 23.09100 # H732 + 733 245 1 -0.84760 3.99900 7.64700 26.09300 # O733 + 734 245 2 0.42380 3.20700 7.98200 25.55400 # H734 + 735 245 2 0.42380 4.49200 7.02900 25.41300 # H735 + 736 246 1 -0.84760 3.63900 18.61900 0.59400 # O736 + 737 246 2 0.42380 3.58800 17.70200 1.06800 # H737 + 738 246 2 0.42380 4.38500 19.08900 1.14800 # H738 + 739 247 1 -0.84760 9.93300 24.42200 14.24800 # O739 + 740 247 2 0.42380 9.85900 23.58500 13.65200 # H740 + 741 247 2 0.42380 10.29100 25.28000 13.75100 # H741 + 742 248 1 -0.84760 22.78300 16.98000 10.33400 # O742 + 743 248 2 0.42380 23.14700 18.00000 10.38200 # H743 + 744 248 2 0.42380 23.13600 16.33100 11.02700 # H744 + 745 249 1 -0.84760 13.75100 1.02900 4.36700 # O745 + 746 249 2 0.42380 14.40500 1.74900 4.74500 # H746 + 747 249 2 0.42380 13.79800 1.33000 3.41000 # H747 + 748 250 1 -0.84760 16.52300 15.46700 29.56300 # O748 + 749 250 2 0.42380 17.30400 15.15000 30.08700 # H749 + 750 250 2 0.42380 15.77700 15.33700 30.25500 # H750 + 751 251 1 -0.84760 7.67200 15.32000 9.68800 # O751 + 752 251 2 0.42380 8.27100 14.50600 9.57200 # H752 + 753 251 2 0.42380 7.26500 14.97900 10.54300 # H753 + 754 252 1 -0.84760 6.73400 22.83300 21.74700 # O754 + 755 252 2 0.42380 7.33500 22.75500 20.95200 # H755 + 756 252 2 0.42380 5.96400 22.17600 21.58500 # H756 + 757 253 1 -0.84760 28.66600 24.11400 27.25200 # O757 + 758 253 2 0.42380 27.71300 23.88100 27.24200 # H758 + 759 253 2 0.42380 28.89400 24.55400 26.37100 # H759 + 760 254 1 -0.84760 26.62600 29.42200 28.66300 # O760 + 761 254 2 0.42380 25.96300 30.18600 28.63900 # H761 + 762 254 2 0.42380 26.07700 28.84600 29.29100 # H762 + 763 255 1 -0.84760 0.35000 28.67800 21.88200 # O763 + 764 255 2 0.42380 -0.61400 28.49300 22.10200 # H764 + 765 255 2 0.42380 0.71500 27.78300 22.11200 # H765 + 766 256 1 -0.84760 13.47000 24.34000 12.88700 # O766 + 767 256 2 0.42380 12.85500 23.61900 13.27400 # H767 + 768 256 2 0.42380 14.36400 24.22600 13.39700 # H768 + 769 257 1 -0.84760 20.34800 10.80800 6.75400 # O769 + 770 257 2 0.42380 21.27500 11.06400 6.42500 # H770 + 771 257 2 0.42380 19.68700 10.52300 6.07000 # H771 + 772 258 1 -0.84760 20.78700 -0.46400 11.51100 # O772 + 773 258 2 0.42380 20.27100 -0.28600 10.67000 # H773 + 774 258 2 0.42380 20.39300 0.20300 12.08400 # H774 + 775 259 1 -0.84760 26.29300 1.73100 16.35100 # O775 + 776 259 2 0.42380 26.54100 2.29000 17.11200 # H776 + 777 259 2 0.42380 27.03300 1.92500 15.72700 # H777 + 778 260 1 -0.84760 10.51000 5.51600 5.06600 # O778 + 779 260 2 0.42380 9.74700 4.89000 5.30200 # H779 + 780 260 2 0.42380 10.10300 6.19800 4.43700 # H780 + 781 261 1 -0.84760 7.03700 0.34600 28.66100 # O781 + 782 261 2 0.42380 7.74000 -0.34500 29.00900 # H782 + 783 261 2 0.42380 6.17800 -0.27900 28.71000 # H783 + 784 262 1 -0.84760 25.07900 11.33600 14.72800 # O784 + 785 262 2 0.42380 25.02600 10.82200 13.80600 # H785 + 786 262 2 0.42380 24.09800 11.64100 14.83400 # H786 + 787 263 1 -0.84760 9.44600 15.10300 3.13100 # O787 + 788 263 2 0.42380 9.72700 15.64800 2.34300 # H788 + 789 263 2 0.42380 8.51700 14.82100 2.82600 # H789 + 790 264 1 -0.84760 5.46300 18.74100 25.34700 # O790 + 791 264 2 0.42380 4.44600 18.97500 25.39300 # H791 + 792 264 2 0.42380 5.53900 17.73800 25.29600 # H792 + 793 265 1 -0.84760 5.50600 21.82300 -0.10800 # O793 + 794 265 2 0.42380 5.80500 22.77800 0.12400 # H794 + 795 265 2 0.42380 6.27700 21.17000 -0.14800 # H795 + 796 266 1 -0.84760 27.88000 19.46000 1.95200 # O796 + 797 266 2 0.42380 28.65700 19.78600 2.50700 # H797 + 798 266 2 0.42380 28.17900 18.89800 1.14400 # H798 + 799 267 1 -0.84760 12.35600 11.97700 4.46700 # O799 + 800 267 2 0.42380 13.03300 12.66500 4.21900 # H800 + 801 267 2 0.42380 12.80600 11.10300 4.25800 # H801 + 802 268 1 -0.84760 8.60200 7.28900 27.67900 # O802 + 803 268 2 0.42380 9.23400 8.09600 27.78600 # H803 + 804 268 2 0.42380 7.68200 7.66600 27.42600 # H804 + 805 269 1 -0.84760 21.65600 22.83600 27.51100 # O805 + 806 269 2 0.42380 20.93800 22.65300 28.20100 # H806 + 807 269 2 0.42380 22.06400 23.73300 27.79100 # H807 + 808 270 1 -0.84760 6.79500 15.28800 13.91400 # O808 + 809 270 2 0.42380 7.07600 14.91500 13.01700 # H809 + 810 270 2 0.42380 7.64200 15.16900 14.42100 # H810 + 811 271 1 -0.84760 12.64500 14.94700 2.61200 # O811 + 812 271 2 0.42380 11.98400 14.93400 1.79800 # H812 + 813 271 2 0.42380 12.50400 15.78800 3.01800 # H813 + 814 272 1 -0.84760 8.83000 13.27800 17.89900 # O814 + 815 272 2 0.42380 8.79600 13.47000 16.91900 # H815 + 816 272 2 0.42380 8.35400 14.09700 18.28200 # H816 + 817 273 1 -0.84760 7.36400 28.18200 29.66300 # O817 + 818 273 2 0.42380 7.16700 27.28100 30.10900 # H818 + 819 273 2 0.42380 6.58700 28.71500 30.02900 # H819 + 820 274 1 -0.84760 11.41400 15.36700 5.37700 # O820 + 821 274 2 0.42380 11.00600 15.34100 4.47400 # H821 + 822 274 2 0.42380 11.83300 14.46500 5.56600 # H822 + 823 275 1 -0.84760 7.50000 6.68900 7.27700 # O823 + 824 275 2 0.42380 7.03700 6.96300 8.11200 # H824 + 825 275 2 0.42380 7.58800 5.62800 7.34100 # H825 + 826 276 1 -0.84760 18.35000 21.26300 7.02600 # O826 + 827 276 2 0.42380 17.47500 21.76600 7.07300 # H827 + 828 276 2 0.42380 18.88500 21.62400 6.23600 # H828 + 829 277 1 -0.84760 7.05400 18.76100 16.03800 # O829 + 830 277 2 0.42380 7.99900 19.08000 16.04100 # H830 + 831 277 2 0.42380 6.72900 18.90100 15.03800 # H831 + 832 278 1 -0.84760 26.64600 -0.00100 12.35200 # O832 + 833 278 2 0.42380 27.50600 0.08800 12.98300 # H833 + 834 278 2 0.42380 26.75300 -0.86100 11.80100 # H834 + 835 279 1 -0.84760 11.29900 2.98400 18.61200 # O835 + 836 279 2 0.42380 10.33400 3.12200 19.00900 # H836 + 837 279 2 0.42380 11.56100 2.04800 18.91200 # H837 + 838 280 1 -0.84760 14.25000 29.32200 3.30900 # O838 + 839 280 2 0.42380 14.47600 29.58400 2.31600 # H839 + 840 280 2 0.42380 14.00300 30.16000 3.83400 # H840 + 841 281 1 -0.84760 2.88800 15.57300 6.31500 # O841 + 842 281 2 0.42380 3.27900 16.03900 7.15900 # H842 + 843 281 2 0.42380 3.69500 15.58300 5.59100 # H843 + 844 282 1 -0.84760 14.10700 27.19700 0.95500 # O844 + 845 282 2 0.42380 13.09600 27.01300 0.93800 # H845 + 846 282 2 0.42380 14.50500 26.49800 0.34800 # H846 + 847 283 1 -0.84760 29.93100 26.37900 12.24200 # O847 + 848 283 2 0.42380 30.61700 25.63200 12.21600 # H848 + 849 283 2 0.42380 29.00900 26.00000 12.44300 # H849 + 850 284 1 -0.84760 20.41800 -0.23900 1.17700 # O850 + 851 284 2 0.42380 21.43400 -0.19600 1.28600 # H851 + 852 284 2 0.42380 20.00400 0.53800 0.63300 # H852 + 853 285 1 -0.84760 3.20000 28.88300 4.45300 # O853 + 854 285 2 0.42380 3.24400 29.45200 5.22400 # H854 + 855 285 2 0.42380 3.93100 29.20000 3.78600 # H855 + 856 286 1 -0.84760 -0.96000 10.89700 2.95300 # O856 + 857 286 2 0.42380 -1.58100 10.26700 2.47900 # H857 + 858 286 2 0.42380 -0.13200 11.06300 2.48500 # H858 + 859 287 1 -0.84760 29.44500 1.96000 28.74200 # O859 + 860 287 2 0.42380 29.30000 2.55700 27.88200 # H860 + 861 287 2 0.42380 29.30700 2.41700 29.62200 # H861 + 862 288 1 -0.84760 6.86600 18.53900 8.97700 # O862 + 863 288 2 0.42380 6.36800 19.37400 8.91900 # H863 + 864 288 2 0.42380 6.93200 18.25100 9.96800 # H864 + 865 289 1 -0.84760 9.40600 25.21400 25.61800 # O865 + 866 289 2 0.42380 9.40600 25.19900 26.57500 # H866 + 867 289 2 0.42380 8.82900 25.94500 25.19200 # H867 + 868 290 1 -0.84760 23.07700 5.97600 14.12900 # O868 + 869 290 2 0.42380 23.87300 5.33000 13.96900 # H869 + 870 290 2 0.42380 23.44400 6.75000 14.68600 # H870 + 871 291 1 -0.84760 21.16600 22.07300 14.37800 # O871 + 872 291 2 0.42380 21.31800 22.99100 13.95800 # H872 + 873 291 2 0.42380 21.11500 21.35300 13.66200 # H873 + 874 292 1 -0.84760 2.10300 14.99800 -0.46700 # O874 + 875 292 2 0.42380 1.72700 14.12300 -0.78300 # H875 + 876 292 2 0.42380 1.43100 15.15900 0.30500 # H876 + 877 293 1 -0.84760 8.38900 17.59500 4.31200 # O877 + 878 293 2 0.42380 8.91100 16.81800 4.64300 # H878 + 879 293 2 0.42380 8.98400 18.08900 3.63000 # H879 + 880 294 1 -0.84760 18.42900 27.29600 5.48400 # O880 + 881 294 2 0.42380 19.42900 27.07200 5.80500 # H881 + 882 294 2 0.42380 18.44000 27.08100 4.50600 # H882 + 883 295 1 -0.84760 5.28600 16.08800 23.52700 # O883 + 884 295 2 0.42380 5.08800 16.61300 22.63400 # H884 + 885 295 2 0.42380 5.91600 16.69700 24.08900 # H885 + 886 296 1 -0.84760 22.21500 11.50700 11.66000 # O886 + 887 296 2 0.42380 23.23100 11.37700 11.71900 # H887 + 888 296 2 0.42380 21.95700 12.52000 11.73500 # H888 + 889 297 1 -0.84760 23.78400 2.26600 17.45100 # O889 + 890 297 2 0.42380 24.25800 1.87800 16.63300 # H890 + 891 297 2 0.42380 24.18000 3.13900 17.75600 # H891 + 892 298 1 -0.84760 10.09800 27.30300 3.37700 # O892 + 893 298 2 0.42380 10.32900 28.30300 3.50800 # H893 + 894 298 2 0.42380 9.26600 27.23200 2.81500 # H894 + 895 299 1 -0.84760 27.03800 14.30500 1.95500 # O895 + 896 299 2 0.42380 27.93500 14.71000 2.17500 # H896 + 897 299 2 0.42380 27.00900 13.49900 2.52800 # H897 + 898 300 1 -0.84760 3.36300 12.13500 19.58000 # O898 + 899 300 2 0.42380 3.96700 12.44100 20.31600 # H899 + 900 300 2 0.42380 3.46300 12.66700 18.69400 # H900 + 901 301 1 -0.84760 7.72000 7.48400 13.41300 # O901 + 902 301 2 0.42380 6.85900 7.04600 13.17400 # H902 + 903 301 2 0.42380 7.49500 8.46700 13.48200 # H903 + 904 302 1 -0.84760 15.09900 24.83500 27.68500 # O904 + 905 302 2 0.42380 15.50500 24.24700 28.42200 # H905 + 906 302 2 0.42380 14.13100 24.96300 27.90200 # H906 + 907 303 1 -0.84760 1.15900 12.92500 25.01400 # O907 + 908 303 2 0.42380 0.46100 12.64100 24.31800 # H908 + 909 303 2 0.42380 1.99400 12.37900 24.70200 # H909 + 910 304 1 -0.84760 11.69100 28.58700 17.76100 # O910 + 911 304 2 0.42380 12.03600 28.82800 16.81000 # H911 + 912 304 2 0.42380 11.25100 27.68000 17.68600 # H912 + 913 305 1 -0.84760 13.08000 28.33900 12.69100 # O913 + 914 305 2 0.42380 13.19900 27.38200 12.38900 # H914 + 915 305 2 0.42380 13.90500 28.71700 12.19500 # H915 + 916 306 1 -0.84760 20.10500 25.56600 21.26000 # O916 + 917 306 2 0.42380 19.87100 25.64200 20.19200 # H917 + 918 306 2 0.42380 19.48700 24.83800 21.61800 # H918 + 919 307 1 -0.84760 8.55100 12.04000 3.45200 # O919 + 920 307 2 0.42380 8.69600 12.98800 3.84800 # H920 + 921 307 2 0.42380 9.40800 11.52000 3.29700 # H921 + 922 308 1 -0.84760 15.03500 17.08200 19.02600 # O922 + 923 308 2 0.42380 15.80600 16.84700 18.48000 # H923 + 924 308 2 0.42380 15.26100 17.64800 19.75700 # H924 + 925 309 1 -0.84760 4.23500 10.48300 16.44300 # O925 + 926 309 2 0.42380 5.10200 10.19500 16.91000 # H926 + 927 309 2 0.42380 3.70600 9.60100 16.63900 # H927 + 928 310 1 -0.84760 29.07000 27.60700 14.61500 # O928 + 929 310 2 0.42380 28.95400 26.92600 15.29300 # H929 + 930 310 2 0.42380 29.37400 27.20200 13.76400 # H930 + 931 311 1 -0.84760 15.33700 18.69300 6.76700 # O931 + 932 311 2 0.42380 16.14700 18.23800 6.46600 # H932 + 933 311 2 0.42380 14.62200 18.99200 6.12800 # H933 + 934 312 1 -0.84760 22.97900 18.23400 23.67500 # O934 + 935 312 2 0.42380 22.66200 17.35600 23.20300 # H935 + 936 312 2 0.42380 22.95000 17.75400 24.56600 # H936 + 937 313 1 -0.84760 24.79800 10.88400 12.01000 # O937 + 938 313 2 0.42380 25.17800 11.54100 12.72900 # H938 + 939 313 2 0.42380 25.52600 10.71300 11.34900 # H939 + 940 314 1 -0.84760 7.72800 3.02500 22.95200 # O940 + 941 314 2 0.42380 7.65300 2.53200 23.83700 # H941 + 942 314 2 0.42380 7.45000 2.31600 22.31200 # H942 + 943 315 1 -0.84760 7.45000 0.58000 21.64100 # O943 + 944 315 2 0.42380 7.98500 0.31800 20.82300 # H944 + 945 315 2 0.42380 6.87900 -0.33000 21.70100 # H945 + 946 316 1 -0.84760 14.89500 15.51400 21.60900 # O946 + 947 316 2 0.42380 15.45200 16.34100 21.44600 # H947 + 948 316 2 0.42380 14.80800 15.21400 20.66100 # H948 + 949 317 1 -0.84760 16.88600 7.36400 8.30200 # O949 + 950 317 2 0.42380 16.06400 7.10400 7.87100 # H950 + 951 317 2 0.42380 17.61000 7.00500 7.70700 # H951 + 952 318 1 -0.84760 24.01500 10.35700 -0.67600 # O952 + 953 318 2 0.42380 24.22600 11.33500 -0.95900 # H953 + 954 318 2 0.42380 24.72100 10.11000 0.04100 # H954 + 955 319 1 -0.84760 16.52900 -0.79400 5.02600 # O955 + 956 319 2 0.42380 15.75700 -0.94200 4.35400 # H956 + 957 319 2 0.42380 17.35200 -0.67400 4.46300 # H957 + 958 320 1 -0.84760 7.85100 8.52700 20.99800 # O958 + 959 320 2 0.42380 8.34000 9.18900 21.59800 # H959 + 960 320 2 0.42380 7.44200 9.20700 20.42300 # H960 + 961 321 1 -0.84760 23.76700 19.80700 27.82800 # O961 + 962 321 2 0.42380 24.40800 20.44300 28.28000 # H962 + 963 321 2 0.42380 24.11000 19.80100 26.85900 # H963 + 964 322 1 -0.84760 5.43500 2.46800 11.12500 # O964 + 965 322 2 0.42380 4.73200 2.69100 11.81900 # H965 + 966 322 2 0.42380 5.34400 1.41000 10.87000 # H966 + 967 323 1 -0.84760 17.50600 29.21200 18.78000 # O967 + 968 323 2 0.42380 18.33200 29.41100 18.18900 # H968 + 969 323 2 0.42380 17.88500 29.41700 19.65600 # H969 + 970 324 1 -0.84760 24.19600 23.30400 25.38900 # O970 + 971 324 2 0.42380 24.00200 23.88900 26.15400 # H971 + 972 324 2 0.42380 24.64000 24.00300 24.75200 # H972 + 973 325 1 -0.84760 29.26600 8.08600 3.69900 # O973 + 974 325 2 0.42380 28.31500 8.14300 4.10900 # H974 + 975 325 2 0.42380 29.48200 7.20400 4.19300 # H975 + 976 326 1 -0.84760 15.93000 21.73000 10.78600 # O976 + 977 326 2 0.42380 15.61600 22.15200 11.64400 # H977 + 978 326 2 0.42380 16.91700 21.86400 10.65100 # H978 + 979 327 1 -0.84760 29.24600 28.17200 7.94900 # O979 + 980 327 2 0.42380 28.26200 28.45300 7.92100 # H980 + 981 327 2 0.42380 29.75000 28.70900 8.61600 # H981 + 982 328 1 -0.84760 24.84300 26.44100 10.97300 # O982 + 983 328 2 0.42380 24.85600 25.60700 10.37700 # H983 + 984 328 2 0.42380 24.38700 25.96700 11.80500 # H984 + 985 329 1 -0.84760 20.21700 23.06900 18.59300 # O985 + 986 329 2 0.42380 19.52300 22.36400 18.53700 # H986 + 987 329 2 0.42380 20.51900 23.05000 19.52800 # H987 + 988 330 1 -0.84760 20.38700 0.83200 29.45000 # O988 + 989 330 2 0.42380 21.14000 0.19900 29.11800 # H989 + 990 330 2 0.42380 19.56100 0.73400 28.88200 # H990 + 991 331 1 -0.84760 25.26800 10.20400 9.04900 # O991 + 992 331 2 0.42380 25.80200 9.35600 9.28400 # H992 + 993 331 2 0.42380 25.86700 10.67200 8.30300 # H993 + 994 332 1 -0.84760 20.71700 3.35500 16.84700 # O994 + 995 332 2 0.42380 21.56800 3.97200 16.97900 # H995 + 996 332 2 0.42380 20.63900 3.43900 15.87100 # H996 + 997 333 1 -0.84760 17.93100 1.63100 18.09300 # O997 + 998 333 2 0.42380 18.87100 1.99400 18.26300 # H998 + 999 333 2 0.42380 17.95900 1.84200 17.04600 # H999 + 1000 334 1 -0.84760 22.75800 29.11600 13.69100 # O1000 + 1001 334 2 0.42380 23.29100 29.73400 14.37200 # H1001 + 1002 334 2 0.42380 22.72300 29.65400 12.84700 # H1002 + 1003 335 1 -0.84760 5.20200 9.38600 29.21700 # O1003 + 1004 335 2 0.42380 4.26800 9.07500 29.46600 # H1004 + 1005 335 2 0.42380 5.87200 8.65100 29.51000 # H1005 + 1006 336 1 -0.84760 28.49100 3.00300 0.80300 # O1006 + 1007 336 2 0.42380 29.04200 2.30600 1.27400 # H1007 + 1008 336 2 0.42380 27.98500 3.55400 1.49300 # H1008 + 1009 337 1 -0.84760 9.66100 9.85800 12.27100 # O1009 + 1010 337 2 0.42380 9.79900 10.88000 12.24300 # H1010 + 1011 337 2 0.42380 10.12200 9.43700 11.51200 # H1011 + 1012 338 1 -0.84760 30.11800 19.36200 28.74500 # O1012 + 1013 338 2 0.42380 30.20200 18.72800 27.94500 # H1013 + 1014 338 2 0.42380 29.11500 19.37800 28.93800 # H1014 + 1015 339 1 -0.84760 20.70400 16.06700 3.88200 # O1015 + 1016 339 2 0.42380 21.36600 15.80000 3.10700 # H1016 + 1017 339 2 0.42380 20.46000 16.94400 3.46600 # H1017 + 1018 340 1 -0.84760 8.01600 19.14400 26.06200 # O1018 + 1019 340 2 0.42380 7.53500 19.31700 26.95300 # H1019 + 1020 340 2 0.42380 7.22000 18.86000 25.50600 # H1020 + 1021 341 1 -0.84760 11.98500 14.63800 22.91000 # O1021 + 1022 341 2 0.42380 12.47500 13.80900 23.08200 # H1022 + 1023 341 2 0.42380 11.50800 14.51300 22.00700 # H1023 + 1024 342 1 -0.84760 17.50100 12.14200 11.47000 # O1024 + 1025 342 2 0.42380 17.41100 12.56400 12.38600 # H1025 + 1026 342 2 0.42380 16.89500 12.58900 10.80200 # H1026 + 1027 343 1 -0.84760 12.78500 11.75100 25.70700 # O1027 + 1028 343 2 0.42380 12.36000 10.93700 26.23100 # H1028 + 1029 343 2 0.42380 12.22700 12.09800 24.94100 # H1029 + 1030 344 1 -0.84760 0.27100 24.57400 0.56700 # O1030 + 1031 344 2 0.42380 -0.54700 24.92000 0.04400 # H1031 + 1032 344 2 0.42380 0.50700 23.61800 0.19100 # H1032 + 1033 345 1 -0.84760 -0.57600 26.36400 19.94200 # O1033 + 1034 345 2 0.42380 -0.41900 25.31800 19.96900 # H1034 + 1035 345 2 0.42380 -0.85300 26.64200 20.85900 # H1035 + 1036 346 1 -0.84760 12.47900 9.73900 14.02500 # O1036 + 1037 346 2 0.42380 12.47700 9.94500 13.05600 # H1037 + 1038 346 2 0.42380 11.82600 10.37200 14.46100 # H1038 + 1039 347 1 -0.84760 16.60000 4.07000 29.89800 # O1039 + 1040 347 2 0.42380 15.72400 4.05800 29.31500 # H1040 + 1041 347 2 0.42380 17.10600 3.20200 29.89500 # H1041 + 1042 348 1 -0.84760 4.20800 23.49800 6.84200 # O1042 + 1043 348 2 0.42380 4.26800 24.43100 7.16900 # H1043 + 1044 348 2 0.42380 3.29200 23.07200 7.02000 # H1044 + 1045 349 1 -0.84760 9.50300 10.75300 27.56700 # O1045 + 1046 349 2 0.42380 8.81500 11.47300 27.41100 # H1046 + 1047 349 2 0.42380 9.72600 10.65600 28.55900 # H1047 + 1048 350 1 -0.84760 20.86900 21.37400 25.64300 # O1048 + 1049 350 2 0.42380 21.20700 22.09800 26.36100 # H1049 + 1050 350 2 0.42380 20.14900 21.87100 25.18600 # H1050 + 1051 351 1 -0.84760 13.25500 24.15700 10.13400 # O1051 + 1052 351 2 0.42380 13.45700 24.04700 11.14900 # H1052 + 1053 351 2 0.42380 12.81900 23.32400 9.75500 # H1053 + 1054 352 1 -0.84760 23.20100 19.39400 7.34300 # O1054 + 1055 352 2 0.42380 23.05800 20.40000 7.14300 # H1055 + 1056 352 2 0.42380 24.19600 19.23200 7.50400 # H1056 + 1057 353 1 -0.84760 14.79900 9.60300 3.02500 # O1057 + 1058 353 2 0.42380 15.68500 9.78900 3.46700 # H1058 + 1059 353 2 0.42380 14.82600 10.33200 2.28600 # H1059 + 1060 354 1 -0.84760 6.62700 24.26700 1.11000 # O1060 + 1061 354 2 0.42380 7.30600 23.51600 1.20300 # H1061 + 1062 354 2 0.42380 7.12300 24.93000 0.46800 # H1062 + 1063 355 1 -0.84760 22.06800 8.84400 12.59900 # O1063 + 1064 355 2 0.42380 21.91500 8.13000 13.31000 # H1064 + 1065 355 2 0.42380 21.47800 9.66900 12.76000 # H1065 + 1066 356 1 -0.84760 18.88100 18.17900 11.83700 # O1066 + 1067 356 2 0.42380 19.72500 18.78800 11.68400 # H1067 + 1068 356 2 0.42380 18.46300 18.44000 10.94700 # H1068 + 1069 357 1 -0.84760 18.86900 28.64200 22.10200 # O1069 + 1070 357 2 0.42380 19.12900 28.88200 21.12300 # H1070 + 1071 357 2 0.42380 18.22100 29.39800 22.32900 # H1071 + 1072 358 1 -0.84760 10.61300 20.02400 24.94600 # O1072 + 1073 358 2 0.42380 11.27500 19.71200 25.60900 # H1073 + 1074 358 2 0.42380 9.69800 19.89000 25.50600 # H1074 + 1075 359 1 -0.84760 29.45500 -0.63900 11.82500 # O1075 + 1076 359 2 0.42380 29.53900 0.35400 12.02800 # H1076 + 1077 359 2 0.42380 30.23600 -0.88300 11.24600 # H1077 + 1078 360 1 -0.84760 7.16700 15.98000 16.89600 # O1078 + 1079 360 2 0.42380 6.53600 16.15100 16.07900 # H1079 + 1080 360 2 0.42380 7.15600 16.91300 17.34200 # H1080 + 1081 361 1 -0.84760 16.94000 24.19400 17.84000 # O1081 + 1082 361 2 0.42380 16.59000 23.30400 17.63800 # H1082 + 1083 361 2 0.42380 16.73400 24.06400 18.83000 # H1083 + 1084 362 1 -0.84760 4.98300 17.16400 4.50800 # O1084 + 1085 362 2 0.42380 3.99400 17.48300 4.55000 # H1085 + 1086 362 2 0.42380 5.51400 17.50700 5.24100 # H1086 + 1087 363 1 -0.84760 17.97700 26.88800 19.70900 # O1087 + 1088 363 2 0.42380 17.71500 27.78900 19.23900 # H1088 + 1089 363 2 0.42380 17.12200 26.41300 19.34500 # H1089 + 1090 364 1 -0.84760 10.38600 29.24400 8.56100 # O1090 + 1091 364 2 0.42380 10.47000 28.57400 7.82000 # H1091 + 1092 364 2 0.42380 11.32600 29.46500 8.65700 # H1092 + 1093 365 1 -0.84760 3.07100 23.74600 27.92700 # O1093 + 1094 365 2 0.42380 2.15100 24.06200 27.61800 # H1094 + 1095 365 2 0.42380 3.22100 22.82900 27.49100 # H1095 + 1096 366 1 -0.84760 28.17200 6.11100 17.85200 # O1096 + 1097 366 2 0.42380 27.29200 6.40600 18.22600 # H1097 + 1098 366 2 0.42380 28.64900 6.99600 17.89000 # H1098 + 1099 367 1 -0.84760 18.70400 22.92800 27.25100 # O1099 + 1100 367 2 0.42380 17.77000 22.61500 26.86600 # H1100 + 1101 367 2 0.42380 19.41600 22.48600 26.68800 # H1101 + 1102 368 1 -0.84760 -0.57900 11.79900 8.34000 # O1102 + 1103 368 2 0.42380 -0.17000 11.37800 7.56800 # H1103 + 1104 368 2 0.42380 0.22000 12.28100 8.74800 # H1104 + 1105 369 1 -0.84760 0.55900 21.23800 27.27300 # O1105 + 1106 369 2 0.42380 -0.17700 21.29900 27.91400 # H1106 + 1107 369 2 0.42380 1.33800 20.79800 27.78300 # H1107 + 1108 370 1 -0.84760 21.58100 0.25900 4.21100 # O1108 + 1109 370 2 0.42380 21.83300 1.12300 3.79900 # H1109 + 1110 370 2 0.42380 21.37200 0.47600 5.17400 # H1110 + 1111 371 1 -0.84760 23.78600 4.43600 23.18800 # O1111 + 1112 371 2 0.42380 23.49500 3.49100 23.42900 # H1112 + 1113 371 2 0.42380 23.79400 4.36700 22.19500 # H1113 + 1114 372 1 -0.84760 9.17200 10.44400 1.08200 # O1114 + 1115 372 2 0.42380 8.22500 10.37500 0.74800 # H1115 + 1116 372 2 0.42380 9.39800 11.31600 1.31900 # H1116 + 1117 373 1 -0.84760 0.90500 0.14500 5.09300 # O1117 + 1118 373 2 0.42380 0.50800 -0.63200 4.52300 # H1118 + 1119 373 2 0.42380 1.18100 -0.42900 5.94500 # H1119 + 1120 374 1 -0.84760 26.47800 7.71700 9.00300 # O1120 + 1121 374 2 0.42380 26.59200 7.05100 8.27500 # H1121 + 1122 374 2 0.42380 26.08900 7.14000 9.77600 # H1122 + 1123 375 1 -0.84760 8.66100 4.75200 26.59400 # O1123 + 1124 375 2 0.42380 8.98400 5.18900 27.43700 # H1124 + 1125 375 2 0.42380 7.94500 5.38600 26.38100 # H1125 + 1126 376 1 -0.84760 20.33100 1.47700 9.50500 # O1126 + 1127 376 2 0.42380 19.35700 1.70100 9.80700 # H1127 + 1128 376 2 0.42380 20.92600 2.28200 9.69600 # H1128 + 1129 377 1 -0.84760 7.38900 10.06400 24.58700 # O1129 + 1130 377 2 0.42380 6.55500 10.06900 25.22700 # H1130 + 1131 377 2 0.42380 7.67300 9.06200 24.52600 # H1131 + 1132 378 1 -0.84760 23.35100 22.97800 17.20600 # O1132 + 1133 378 2 0.42380 23.82300 23.40700 16.36500 # H1133 + 1134 378 2 0.42380 23.24000 22.05500 16.81000 # H1134 + 1135 379 1 -0.84760 4.21700 27.30100 1.28100 # O1135 + 1136 379 2 0.42380 4.41000 26.43900 1.76000 # H1136 + 1137 379 2 0.42380 4.15900 26.86600 0.36700 # H1137 + 1138 380 1 -0.84760 19.77100 26.73800 0.47800 # O1138 + 1139 380 2 0.42380 19.56000 27.55700 1.04800 # H1139 + 1140 380 2 0.42380 19.85300 27.03500 -0.48600 # H1140 + 1141 381 1 -0.84760 13.11800 3.74900 23.60400 # O1141 + 1142 381 2 0.42380 13.10500 2.76400 23.96600 # H1142 + 1143 381 2 0.42380 14.01000 3.80500 23.10400 # H1143 + 1144 382 1 -0.84760 13.97600 -0.98600 6.60800 # O1144 + 1145 382 2 0.42380 13.86600 -1.79900 7.27300 # H1145 + 1146 382 2 0.42380 13.97000 -0.13300 7.23100 # H1146 + 1147 383 1 -0.84760 9.04800 6.04500 0.50100 # O1147 + 1148 383 2 0.42380 9.82500 5.30800 0.45000 # H1148 + 1149 383 2 0.42380 9.18300 6.38000 -0.43000 # H1149 + 1150 384 1 -0.84760 29.73900 14.83200 29.44100 # O1150 + 1151 384 2 0.42380 30.28800 15.53600 29.06500 # H1151 + 1152 384 2 0.42380 30.34200 13.99700 29.41800 # H1152 + 1153 385 1 -0.84760 19.73600 14.54500 17.95300 # O1153 + 1154 385 2 0.42380 19.45400 14.17700 17.05900 # H1154 + 1155 385 2 0.42380 19.24100 13.84400 18.45700 # H1155 + 1156 386 1 -0.84760 17.17000 9.62900 12.35200 # O1156 + 1157 386 2 0.42380 16.33500 9.71900 13.01100 # H1157 + 1158 386 2 0.42380 17.46500 10.35300 11.77000 # H1158 + 1159 387 1 -0.84760 5.81700 10.65400 5.56600 # O1159 + 1160 387 2 0.42380 5.48500 9.95900 6.21400 # H1160 + 1161 387 2 0.42380 5.06600 11.21700 5.19400 # H1161 + 1162 388 1 -0.84760 0.99700 23.71800 19.96400 # O1162 + 1163 388 2 0.42380 1.20100 23.77300 18.95900 # H1163 + 1164 388 2 0.42380 1.61700 24.26700 20.57000 # H1164 + 1165 389 1 -0.84760 1.50700 23.69800 17.30500 # O1165 + 1166 389 2 0.42380 2.45900 23.51000 17.62400 # H1166 + 1167 389 2 0.42380 1.00600 22.85400 17.52300 # H1167 + 1168 390 1 -0.84760 8.25400 15.41500 -0.26100 # O1168 + 1169 390 2 0.42380 8.23800 14.56900 -0.84600 # H1169 + 1170 390 2 0.42380 8.65500 16.01500 -0.93300 # H1170 + 1171 391 1 -0.84760 11.96500 13.76900 14.57800 # O1171 + 1172 391 2 0.42380 11.05800 13.30900 14.58900 # H1172 + 1173 391 2 0.42380 11.98400 14.04500 15.53100 # H1173 + 1174 392 1 -0.84760 16.35300 2.97200 4.61600 # O1174 + 1175 392 2 0.42380 15.80500 3.76500 5.02000 # H1175 + 1176 392 2 0.42380 16.61500 3.31000 3.69600 # H1176 + 1177 393 1 -0.84760 30.05500 22.84500 5.38300 # O1177 + 1178 393 2 0.42380 29.32600 22.74900 6.12700 # H1178 + 1179 393 2 0.42380 30.72200 22.02600 5.55000 # H1179 + 1180 394 1 -0.84760 11.94500 -0.97500 14.78500 # O1180 + 1181 394 2 0.42380 12.71500 -0.73100 15.36300 # H1181 + 1182 394 2 0.42380 12.35600 -1.53700 14.07800 # H1182 + 1183 395 1 -0.84760 8.86600 4.23900 11.67400 # O1183 + 1184 395 2 0.42380 7.98000 4.84000 11.72600 # H1184 + 1185 395 2 0.42380 8.64800 3.30600 11.34900 # H1185 + 1186 396 1 -0.84760 7.88500 3.45500 14.24200 # O1186 + 1187 396 2 0.42380 8.56900 3.49100 13.40700 # H1187 + 1188 396 2 0.42380 6.91400 3.20100 14.00900 # H1188 + 1189 397 1 -0.84760 19.58900 13.99900 12.53800 # O1189 + 1190 397 2 0.42380 20.59900 14.01700 12.66000 # H1190 + 1191 397 2 0.42380 19.41200 14.29100 11.57000 # H1191 + 1192 398 1 -0.84760 16.24000 15.62700 12.17600 # O1192 + 1193 398 2 0.42380 15.39400 15.77000 12.67600 # H1193 + 1194 398 2 0.42380 16.74100 16.48500 12.21600 # H1194 + 1195 399 1 -0.84760 7.81900 26.05800 14.78500 # O1195 + 1196 399 2 0.42380 8.63100 25.37900 14.82500 # H1196 + 1197 399 2 0.42380 8.02100 27.04900 15.09600 # H1197 + 1198 400 1 -0.84760 19.41600 16.88200 14.39100 # O1198 + 1199 400 2 0.42380 18.73900 16.83000 13.76400 # H1199 + 1200 400 2 0.42380 19.83500 17.78900 14.16800 # H1200 + 1201 401 1 -0.84760 27.37700 11.85400 23.44500 # O1201 + 1202 401 2 0.42380 26.72800 12.63200 23.44200 # H1202 + 1203 401 2 0.42380 27.77200 11.92800 24.36300 # H1203 + 1204 402 1 -0.84760 13.96500 26.27800 22.16800 # O1204 + 1205 402 2 0.42380 13.28400 27.00300 21.96100 # H1205 + 1206 402 2 0.42380 13.81100 26.05100 23.15000 # H1206 + 1207 403 1 -0.84760 6.86100 5.57000 11.48100 # O1207 + 1208 403 2 0.42380 7.73800 6.14400 11.31000 # H1208 + 1209 403 2 0.42380 6.72000 5.00200 10.64200 # H1209 + 1210 404 1 -0.84760 8.93700 8.22400 18.11400 # O1210 + 1211 404 2 0.42380 9.49200 8.44700 17.26800 # H1211 + 1212 404 2 0.42380 8.45500 7.33300 17.80600 # H1212 + 1213 405 1 -0.84760 16.21000 4.70100 14.59300 # O1213 + 1214 405 2 0.42380 16.68000 3.87100 14.20300 # H1214 + 1215 405 2 0.42380 15.25400 4.58300 14.27900 # H1215 + 1216 406 1 -0.84760 10.39700 7.31800 2.82500 # O1216 + 1217 406 2 0.42380 10.50000 6.65300 2.01600 # H1217 + 1218 406 2 0.42380 9.76400 8.16100 2.61800 # H1218 + 1219 407 1 -0.84760 10.50000 13.98200 9.00200 # O1219 + 1220 407 2 0.42380 10.99800 13.18100 8.63800 # H1220 + 1221 407 2 0.42380 11.16400 14.68800 8.74400 # H1221 + 1222 408 1 -0.84760 24.67100 4.05200 15.70500 # O1222 + 1223 408 2 0.42380 24.66200 4.89300 16.35700 # H1223 + 1224 408 2 0.42380 25.45500 4.38700 15.11500 # H1224 + 1225 409 1 -0.84760 26.50400 9.01000 4.26500 # O1225 + 1226 409 2 0.42380 26.77800 8.39500 5.00000 # H1226 + 1227 409 2 0.42380 25.93200 8.37700 3.68900 # H1227 + 1228 410 1 -0.84760 26.74200 18.07600 20.67100 # O1228 + 1229 410 2 0.42380 26.02800 18.15300 19.93300 # H1229 + 1230 410 2 0.42380 26.27100 18.67800 21.43300 # H1230 + 1231 411 1 -0.84760 17.24500 27.97100 1.85200 # O1231 + 1232 411 2 0.42380 16.32700 27.94800 2.24200 # H1232 + 1233 411 2 0.42380 17.62000 27.18900 2.37400 # H1233 + 1234 412 1 -0.84760 24.23400 10.88100 27.04900 # O1234 + 1235 412 2 0.42380 23.30800 10.89200 27.43200 # H1235 + 1236 412 2 0.42380 24.58900 10.01400 26.70000 # H1236 + 1237 413 1 -0.84760 29.26200 20.88400 15.33500 # O1237 + 1238 413 2 0.42380 28.85300 20.96700 16.24100 # H1238 + 1239 413 2 0.42380 28.72800 21.55400 14.79300 # H1239 + 1240 414 1 -0.84760 17.51200 26.36600 8.92400 # O1240 + 1241 414 2 0.42380 17.00300 25.85000 9.67200 # H1241 + 1242 414 2 0.42380 18.06100 25.66700 8.51100 # H1242 + 1243 415 1 -0.84760 21.13200 11.87100 24.81700 # O1243 + 1244 415 2 0.42380 21.05200 11.74600 25.81900 # H1244 + 1245 415 2 0.42380 20.96600 11.03400 24.28000 # H1245 + 1246 416 1 -0.84760 4.54400 20.37700 9.27100 # O1246 + 1247 416 2 0.42380 5.35000 20.97900 9.13900 # H1247 + 1248 416 2 0.42380 4.07400 20.63100 10.13500 # H1248 + 1249 417 1 -0.84760 6.19500 24.18200 10.15100 # O1249 + 1250 417 2 0.42380 5.52300 24.74800 9.70700 # H1250 + 1251 417 2 0.42380 6.99700 24.15200 9.57500 # H1251 + 1252 418 1 -0.84760 20.59200 28.17200 20.04100 # O1252 + 1253 418 2 0.42380 20.19100 27.25600 19.95500 # H1253 + 1254 418 2 0.42380 21.28800 28.19300 20.69900 # H1254 + 1255 419 1 -0.84760 28.15100 22.55200 7.17600 # O1255 + 1256 419 2 0.42380 27.58100 23.31900 6.91400 # H1256 + 1257 419 2 0.42380 28.34700 22.50000 8.17000 # H1257 + 1258 420 1 -0.84760 6.91000 10.14000 17.29200 # O1258 + 1259 420 2 0.42380 7.79000 9.72600 17.58900 # H1259 + 1260 420 2 0.42380 7.01900 11.09500 17.03600 # H1260 + 1261 421 1 -0.84760 -0.11000 23.57300 13.20400 # O1261 + 1262 421 2 0.42380 -0.73200 22.81500 13.37200 # H1262 + 1263 421 2 0.42380 0.34000 23.88300 14.05600 # H1263 + 1264 422 1 -0.84760 30.08100 6.38000 7.74300 # O1264 + 1265 422 2 0.42380 29.16800 6.51500 7.98400 # H1265 + 1266 422 2 0.42380 30.50100 5.88600 8.49900 # H1266 + 1267 423 1 -0.84760 25.34800 15.08400 0.21900 # O1267 + 1268 423 2 0.42380 25.83300 14.46400 0.85700 # H1268 + 1269 423 2 0.42380 25.81300 15.31100 -0.68600 # H1269 + 1270 424 1 -0.84760 0.45600 28.55400 1.07200 # O1270 + 1271 424 2 0.42380 1.19400 28.87700 0.46500 # H1271 + 1272 424 2 0.42380 -0.44900 28.89700 0.85500 # H1272 + 1273 425 1 -0.84760 6.94000 3.35700 20.59800 # O1273 + 1274 425 2 0.42380 7.37700 3.85300 21.37300 # H1274 + 1275 425 2 0.42380 7.53900 2.50800 20.45700 # H1275 + 1276 426 1 -0.84760 14.06700 25.47800 18.42900 # O1276 + 1277 426 2 0.42380 14.45800 26.38700 18.33300 # H1277 + 1278 426 2 0.42380 13.04000 25.51500 18.41900 # H1278 + 1279 427 1 -0.84760 14.74100 2.35200 26.58200 # O1279 + 1280 427 2 0.42380 15.70100 2.64300 26.44600 # H1280 + 1281 427 2 0.42380 14.46100 2.26300 25.61400 # H1281 + 1282 428 1 -0.84760 21.31800 7.20800 20.93500 # O1282 + 1283 428 2 0.42380 20.32100 7.28400 20.60200 # H1283 + 1284 428 2 0.42380 21.19300 6.31400 21.41100 # H1284 + 1285 429 1 -0.84760 13.40500 16.30600 24.48200 # O1285 + 1286 429 2 0.42380 13.85900 16.80300 23.73300 # H1286 + 1287 429 2 0.42380 12.97300 15.51400 24.09000 # H1287 + 1288 430 1 -0.84760 9.79400 20.26400 -0.14800 # O1288 + 1289 430 2 0.42380 10.69000 20.20200 -0.63300 # H1289 + 1290 430 2 0.42380 9.36000 21.13900 -0.37400 # H1290 + 1291 431 1 -0.84760 26.77500 22.48500 17.53200 # O1291 + 1292 431 2 0.42380 26.73400 22.35300 16.53600 # H1292 + 1293 431 2 0.42380 26.07600 23.14800 17.90000 # H1293 + 1294 432 1 -0.84760 15.54100 4.93000 10.52600 # O1294 + 1295 432 2 0.42380 14.85000 4.34000 10.03100 # H1295 + 1296 432 2 0.42380 15.31900 4.85300 11.49700 # H1296 + 1297 433 1 -0.84760 28.26200 3.88700 15.33600 # O1297 + 1298 433 2 0.42380 29.17200 4.25000 15.03200 # H1298 + 1299 433 2 0.42380 28.00100 4.48400 16.09700 # H1299 + 1300 434 1 -0.84760 18.63900 22.63800 11.06300 # O1300 + 1301 434 2 0.42380 19.53800 22.85300 11.42400 # H1301 + 1302 434 2 0.42380 17.98200 22.80800 11.80100 # H1302 + 1303 435 1 -0.84760 4.29500 -0.94000 25.45200 # O1303 + 1304 435 2 0.42380 3.70800 -0.11200 25.34500 # H1304 + 1305 435 2 0.42380 3.59200 -1.68700 25.54400 # H1305 + 1306 436 1 -0.84760 4.17200 0.09300 11.42200 # O1306 + 1307 436 2 0.42380 5.15300 -0.04400 11.55800 # H1307 + 1308 436 2 0.42380 3.78200 -0.72300 10.98000 # H1308 + 1309 437 1 -0.84760 19.50700 3.96600 26.63400 # O1309 + 1310 437 2 0.42380 19.04500 4.78100 26.29700 # H1310 + 1311 437 2 0.42380 18.81400 3.23100 26.42300 # H1311 + 1312 438 1 -0.84760 24.93300 24.23200 1.65400 # O1312 + 1313 438 2 0.42380 24.79300 23.45100 2.29100 # H1313 + 1314 438 2 0.42380 25.85500 24.56800 1.94500 # H1314 + 1315 439 1 -0.84760 4.59600 19.29000 18.01900 # O1315 + 1316 439 2 0.42380 4.90000 18.58800 17.43500 # H1316 + 1317 439 2 0.42380 4.82000 20.10800 17.42200 # H1317 + 1318 440 1 -0.84760 8.32300 28.59400 6.68700 # O1318 + 1319 440 2 0.42380 7.66000 29.28300 6.32000 # H1319 + 1320 440 2 0.42380 8.30700 27.91700 5.95300 # H1320 + 1321 441 1 -0.84760 15.91200 1.28400 13.86200 # O1321 + 1322 441 2 0.42380 15.29600 0.57200 13.49400 # H1322 + 1323 441 2 0.42380 15.53400 2.19300 13.59900 # H1323 + 1324 442 1 -0.84760 13.41300 24.14300 20.74000 # O1324 + 1325 442 2 0.42380 13.70500 24.61200 19.91300 # H1325 + 1326 442 2 0.42380 13.31600 24.89100 21.35200 # H1326 + 1327 443 1 -0.84760 29.87400 4.65600 12.60000 # O1327 + 1328 443 2 0.42380 30.05500 5.65100 12.49600 # H1328 + 1329 443 2 0.42380 29.46300 4.33800 11.68900 # H1329 + 1330 444 1 -0.84760 3.44500 25.61900 4.84600 # O1330 + 1331 444 2 0.42380 3.04900 25.33500 5.75400 # H1331 + 1332 444 2 0.42380 3.47300 26.68100 4.84000 # H1332 + 1333 445 1 -0.84760 2.81100 17.33000 12.95300 # O1333 + 1334 445 2 0.42380 2.28300 18.12300 13.24700 # H1334 + 1335 445 2 0.42380 3.78100 17.62800 12.73800 # H1335 + 1336 446 1 -0.84760 29.11700 6.06600 25.91400 # O1336 + 1337 446 2 0.42380 28.77500 6.87900 25.35200 # H1337 + 1338 446 2 0.42380 29.32700 5.35100 25.20600 # H1338 + 1339 447 1 -0.84760 25.31000 28.91300 0.60400 # O1339 + 1340 447 2 0.42380 24.58400 28.88600 -0.07600 # H1340 + 1341 447 2 0.42380 25.28900 29.87100 0.95100 # H1341 + 1342 448 1 -0.84760 28.99600 8.82900 7.25800 # O1342 + 1343 448 2 0.42380 29.30000 7.94200 7.57500 # H1343 + 1344 448 2 0.42380 29.77300 9.38800 6.88500 # H1344 + 1345 449 1 -0.84760 11.24700 24.27000 18.81000 # O1345 + 1346 449 2 0.42380 11.54700 24.38600 17.82300 # H1346 + 1347 449 2 0.42380 11.82200 23.59300 19.26500 # H1347 + 1348 450 1 -0.84760 13.90300 7.50400 14.11100 # O1348 + 1349 450 2 0.42380 13.07100 8.05600 14.39600 # H1349 + 1350 450 2 0.42380 14.59600 8.13000 14.41700 # H1350 + 1351 451 1 -0.84760 14.79100 -0.76200 17.57100 # O1351 + 1352 451 2 0.42380 14.43600 -0.29800 18.40200 # H1352 + 1353 451 2 0.42380 15.67700 -1.17400 17.79900 # H1353 + 1354 452 1 -0.84760 24.68100 6.60600 10.88500 # O1354 + 1355 452 2 0.42380 23.92500 7.25600 10.66800 # H1355 + 1356 452 2 0.42380 24.18000 5.79000 11.19400 # H1356 + 1357 453 1 -0.84760 16.58900 10.36000 26.61000 # O1357 + 1358 453 2 0.42380 16.94400 10.96200 25.84300 # H1358 + 1359 453 2 0.42380 15.63800 10.22800 26.26700 # H1359 + 1360 454 1 -0.84760 19.45900 0.43300 21.36400 # O1360 + 1361 454 2 0.42380 19.43100 1.00700 22.28700 # H1361 + 1362 454 2 0.42380 20.36400 0.52600 20.82500 # H1362 + 1363 455 1 -0.84760 29.38600 9.37200 17.37000 # O1363 + 1364 455 2 0.42380 28.87000 10.25800 17.20900 # H1364 + 1365 455 2 0.42380 29.92800 9.52900 18.22600 # H1365 + 1366 456 1 -0.84760 13.77500 19.61000 9.09000 # O1366 + 1367 456 2 0.42380 13.87800 20.32700 8.28000 # H1367 + 1368 456 2 0.42380 12.83000 19.62700 9.54000 # H1368 + 1369 457 1 -0.84760 3.81900 17.62800 7.97900 # O1369 + 1370 457 2 0.42380 4.44100 18.35100 8.15300 # H1370 + 1371 457 2 0.42380 3.33400 17.38400 8.90000 # H1371 + 1372 458 1 -0.84760 22.20500 10.96900 1.67700 # O1372 + 1373 458 2 0.42380 23.14700 11.41700 1.65000 # H1373 + 1374 458 2 0.42380 22.36500 10.03600 2.06000 # H1374 + 1375 459 1 -0.84760 22.98400 1.49200 29.71400 # O1375 + 1376 459 2 0.42380 23.16000 2.42100 29.99300 # H1376 + 1377 459 2 0.42380 22.22900 1.18000 30.30300 # H1377 + 1378 460 1 -0.84760 23.10700 24.60800 23.42100 # O1378 + 1379 460 2 0.42380 23.07100 23.74300 23.96400 # H1379 + 1380 460 2 0.42380 23.97400 25.04400 23.68800 # H1380 + 1381 461 1 -0.84760 19.01900 22.83700 24.20100 # O1381 + 1382 461 2 0.42380 18.19800 23.44100 24.14100 # H1382 + 1383 461 2 0.42380 18.74600 21.87500 24.02200 # H1383 + 1384 462 1 -0.84760 6.96400 17.85900 19.45400 # O1384 + 1385 462 2 0.42380 7.49400 18.14500 20.27300 # H1385 + 1386 462 2 0.42380 6.54900 18.70200 19.03700 # H1386 + 1387 463 1 -0.84760 28.06000 19.59000 23.10600 # O1387 + 1388 463 2 0.42380 28.02200 20.30300 22.44200 # H1388 + 1389 463 2 0.42380 27.14900 19.37800 23.56100 # H1389 + 1390 464 1 -0.84760 15.97300 6.77600 16.93600 # O1390 + 1391 464 2 0.42380 15.89100 6.78400 17.98300 # H1391 + 1392 464 2 0.42380 15.70200 5.79400 16.75900 # H1392 + 1393 465 1 -0.84760 16.72900 8.33300 0.69000 # O1393 + 1394 465 2 0.42380 17.26200 8.89900 -0.01400 # H1394 + 1395 465 2 0.42380 15.73400 8.46700 0.67400 # H1395 + 1396 466 1 -0.84760 17.38700 21.06000 23.05700 # O1396 + 1397 466 2 0.42380 17.01800 21.97800 23.15900 # H1397 + 1398 466 2 0.42380 17.11200 20.57100 23.87400 # H1398 + 1399 467 1 -0.84760 5.81900 28.61200 9.00300 # O1399 + 1400 467 2 0.42380 6.23600 28.11100 9.77800 # H1400 + 1401 467 2 0.42380 4.80700 28.32000 9.08100 # H1401 + 1402 468 1 -0.84760 6.39500 12.40500 28.77000 # O1402 + 1403 468 2 0.42380 6.90500 12.55900 29.64800 # H1403 + 1404 468 2 0.42380 5.48800 12.22600 29.21700 # H1404 + 1405 469 1 -0.84760 17.84300 20.43600 30.00400 # O1405 + 1406 469 2 0.42380 17.49000 19.70300 30.57700 # H1406 + 1407 469 2 0.42380 18.10100 21.10500 30.73400 # H1407 + 1408 470 1 -0.84760 11.63300 4.51000 28.17300 # O1408 + 1409 470 2 0.42380 11.76700 5.16900 27.36400 # H1409 + 1410 470 2 0.42380 10.66400 4.54300 28.48200 # H1410 + 1411 471 1 -0.84760 1.53100 28.54900 26.79000 # O1411 + 1412 471 2 0.42380 2.18000 28.14500 27.39500 # H1412 + 1413 471 2 0.42380 0.97900 27.71900 26.39300 # H1413 + 1414 472 1 -0.84760 -0.45000 28.31100 18.15600 # O1414 + 1415 472 2 0.42380 -0.60600 27.54200 18.86900 # H1415 + 1416 472 2 0.42380 -1.36700 28.75700 18.28400 # H1416 + 1417 473 1 -0.84760 21.14500 28.57000 3.69000 # O1417 + 1418 473 2 0.42380 20.83700 29.35100 4.26800 # H1418 + 1419 473 2 0.42380 21.46400 29.04000 2.88000 # H1419 + 1420 474 1 -0.84760 9.31500 18.47400 1.84300 # O1420 + 1421 474 2 0.42380 9.00000 18.21900 0.90700 # H1421 + 1422 474 2 0.42380 9.40000 19.43700 1.66400 # H1422 + 1423 475 1 -0.84760 25.20200 28.37100 25.77700 # O1423 + 1424 475 2 0.42380 26.20600 28.52800 25.73000 # H1424 + 1425 475 2 0.42380 24.82500 29.24300 26.03500 # H1425 + 1426 476 1 -0.84760 0.13700 23.55500 25.87000 # O1426 + 1427 476 2 0.42380 0.12600 22.74300 26.49000 # H1427 + 1428 476 2 0.42380 0.37700 23.40200 24.92400 # H1428 + 1429 477 1 -0.84760 20.94600 16.46500 8.37600 # O1429 + 1430 477 2 0.42380 20.22800 16.68200 7.79100 # H1430 + 1431 477 2 0.42380 21.77000 17.05100 8.26600 # H1431 + 1432 478 1 -0.84760 17.16600 14.75700 25.68300 # O1432 + 1433 478 2 0.42380 17.51200 14.40700 24.79100 # H1433 + 1434 478 2 0.42380 16.70400 15.70800 25.62800 # H1434 + 1435 479 1 -0.84760 28.38100 18.43900 17.44500 # O1435 + 1436 479 2 0.42380 29.14400 18.55600 16.75000 # H1436 + 1437 479 2 0.42380 28.70600 17.68700 18.08100 # H1437 + 1438 480 1 -0.84760 14.41700 4.29000 28.70700 # O1438 + 1439 480 2 0.42380 13.52000 4.59300 28.21800 # H1439 + 1440 480 2 0.42380 14.34100 3.32100 28.30000 # H1440 + 1441 481 1 -0.84760 6.78400 -0.21300 6.56700 # O1441 + 1442 481 2 0.42380 6.25800 -0.69400 7.32500 # H1442 + 1443 481 2 0.42380 6.16700 0.17900 5.88000 # H1443 + 1444 482 1 -0.84760 27.01300 6.79200 29.77400 # O1444 + 1445 482 2 0.42380 26.86900 7.75900 29.82600 # H1445 + 1446 482 2 0.42380 27.08300 6.62300 28.80600 # H1446 + 1447 483 1 -0.84760 25.24100 5.90100 17.87500 # O1447 + 1448 483 2 0.42380 26.25300 5.84100 17.97000 # H1448 + 1449 483 2 0.42380 25.03800 6.76100 18.28700 # H1449 + 1450 484 1 -0.84760 7.95600 -0.13900 19.25800 # O1450 + 1451 484 2 0.42380 7.78500 0.87300 19.12300 # H1451 + 1452 484 2 0.42380 7.13800 -0.70000 18.88700 # H1452 + 1453 485 1 -0.84760 20.00000 20.12800 22.02500 # O1453 + 1454 485 2 0.42380 19.92700 19.16900 22.25900 # H1454 + 1455 485 2 0.42380 19.04300 20.44200 22.05000 # H1455 + 1456 486 1 -0.84760 8.55100 24.64100 2.85400 # O1456 + 1457 486 2 0.42380 9.33400 24.99100 3.33700 # H1457 + 1458 486 2 0.42380 7.78000 25.33100 2.83600 # H1458 + 1459 487 1 -0.84760 12.07900 5.92500 25.86400 # O1459 + 1460 487 2 0.42380 11.54800 6.72500 25.48900 # H1460 + 1461 487 2 0.42380 12.87400 5.78900 25.20200 # H1461 + 1462 488 1 -0.84760 0.41400 27.88300 5.96800 # O1462 + 1463 488 2 0.42380 0.03400 28.46600 5.07900 # H1463 + 1464 488 2 0.42380 -0.27500 27.90400 6.78100 # H1464 + 1465 489 1 -0.84760 -0.73800 3.84500 19.94800 # O1465 + 1466 489 2 0.42380 0.11800 4.09700 19.43300 # H1466 + 1467 489 2 0.42380 -0.64100 3.53400 20.93400 # H1467 + 1468 490 1 -0.84760 29.77100 1.05100 18.82300 # O1468 + 1469 490 2 0.42380 30.76000 0.93500 18.85100 # H1469 + 1470 490 2 0.42380 29.61200 1.99000 19.09800 # H1470 + 1471 491 1 -0.84760 16.90600 -0.10900 20.84700 # O1471 + 1472 491 2 0.42380 17.96900 -0.20100 21.13600 # H1472 + 1473 491 2 0.42380 17.11400 0.85400 20.45500 # H1473 + 1474 492 1 -0.84760 5.90200 11.67300 13.97600 # O1474 + 1475 492 2 0.42380 5.58300 11.85100 14.88600 # H1475 + 1476 492 2 0.42380 6.40700 10.84900 13.99800 # H1476 + 1477 493 1 -0.84760 20.80100 17.08000 20.64000 # O1477 + 1478 493 2 0.42380 21.32100 16.53300 21.29900 # H1478 + 1479 493 2 0.42380 21.30800 17.13400 19.70000 # H1479 + 1480 494 1 -0.84760 21.95200 9.67500 22.27800 # O1480 + 1481 494 2 0.42380 21.32400 9.41800 23.05500 # H1481 + 1482 494 2 0.42380 22.06900 8.77400 21.71200 # H1482 + 1483 495 1 -0.84760 20.37200 13.01400 2.97400 # O1483 + 1484 495 2 0.42380 20.26300 13.19800 3.95800 # H1484 + 1485 495 2 0.42380 21.34000 12.71200 2.87400 # H1485 + 1486 496 1 -0.84760 7.92700 -0.11500 9.60000 # O1486 + 1487 496 2 0.42380 8.31900 -0.39700 10.45200 # H1487 + 1488 496 2 0.42380 7.70600 -0.98700 9.22300 # H1488 + 1489 497 1 -0.84760 2.86300 8.55600 21.49300 # O1489 + 1490 497 2 0.42380 2.15400 9.30000 21.55300 # H1490 + 1491 497 2 0.42380 3.54500 8.96600 20.90100 # H1491 + 1492 498 1 -0.84760 6.16100 5.61600 28.38000 # O1492 + 1493 498 2 0.42380 5.49700 5.83600 27.60600 # H1493 + 1494 498 2 0.42380 5.76600 5.49100 29.33400 # H1494 + 1495 499 1 -0.84760 14.02900 21.85200 22.68600 # O1495 + 1496 499 2 0.42380 14.84000 22.41100 22.93600 # H1496 + 1497 499 2 0.42380 13.26400 22.50000 22.47300 # H1497 + 1498 500 1 -0.84760 15.66700 24.19300 8.93900 # O1498 + 1499 500 2 0.42380 15.28700 23.41500 9.36600 # H1499 + 1500 500 2 0.42380 14.93600 24.89200 8.96900 # H1500 + 1501 501 1 -0.84760 4.37700 22.31100 2.32500 # O1501 + 1502 501 2 0.42380 4.81500 22.01900 1.38200 # H1502 + 1503 501 2 0.42380 3.68900 21.53200 2.50700 # H1503 + 1504 502 1 -0.84760 26.78600 23.05100 23.45400 # O1504 + 1505 502 2 0.42380 27.16700 22.42800 24.20200 # H1505 + 1506 502 2 0.42380 26.85900 22.54700 22.53700 # H1506 + 1507 503 1 -0.84760 2.22700 3.28000 8.34400 # O1507 + 1508 503 2 0.42380 3.08700 2.75400 8.37800 # H1508 + 1509 503 2 0.42380 2.40000 3.78300 7.49200 # H1509 + 1510 504 1 -0.84760 16.11100 10.52500 21.72300 # O1510 + 1511 504 2 0.42380 15.69700 10.22700 20.79100 # H1511 + 1512 504 2 0.42380 15.66400 9.82200 22.29300 # H1512 + 1513 505 1 -0.84760 27.91700 24.73800 14.35400 # O1513 + 1514 505 2 0.42380 27.63500 24.69400 15.29600 # H1514 + 1515 505 2 0.42380 28.86200 24.46200 14.29500 # H1515 + 1516 506 1 -0.84760 24.03000 21.38200 0.57200 # O1516 + 1517 506 2 0.42380 23.28900 22.07000 0.65800 # H1517 + 1518 506 2 0.42380 24.46200 21.12900 1.47200 # H1518 + 1519 507 1 -0.84760 9.64100 8.37300 15.27900 # O1519 + 1520 507 2 0.42380 8.78200 7.94500 14.89000 # H1520 + 1521 507 2 0.42380 10.26000 8.25200 14.43800 # H1521 + 1522 508 1 -0.84760 26.09800 16.71600 24.63300 # O1522 + 1523 508 2 0.42380 26.61600 17.53400 25.03200 # H1523 + 1524 508 2 0.42380 26.06500 16.06700 25.43100 # H1524 + 1525 509 1 -0.84760 11.84600 8.83000 26.81600 # O1525 + 1526 509 2 0.42380 12.73900 8.38400 26.71900 # H1526 + 1527 509 2 0.42380 11.12000 8.13400 26.88400 # H1527 + 1528 510 1 -0.84760 21.94300 2.01200 19.78200 # O1528 + 1529 510 2 0.42380 22.20900 1.04600 19.78200 # H1529 + 1530 510 2 0.42380 22.06200 2.21500 18.80700 # H1530 + 1531 511 1 -0.84760 20.76100 22.81700 21.03200 # O1531 + 1532 511 2 0.42380 19.80900 22.57300 21.41300 # H1532 + 1533 511 2 0.42380 21.24400 23.36500 21.77100 # H1533 + 1534 512 1 -0.84760 25.22000 13.48900 11.95200 # O1534 + 1535 512 2 0.42380 25.70900 14.36400 11.71700 # H1535 + 1536 512 2 0.42380 24.73400 13.38900 11.09400 # H1536 + 1537 513 1 -0.84760 8.04900 18.98900 11.03100 # O1537 + 1538 513 2 0.42380 7.95000 19.99900 11.07300 # H1538 + 1539 513 2 0.42380 8.49400 18.75500 10.16800 # H1539 + 1540 514 1 -0.84760 7.83100 11.23700 19.74400 # O1540 + 1541 514 2 0.42380 7.93400 12.15200 19.21900 # H1541 + 1542 514 2 0.42380 8.17100 10.34500 19.28300 # H1542 + 1543 515 1 -0.84760 29.22700 17.01700 5.31100 # O1543 + 1544 515 2 0.42380 28.49200 17.67900 5.60700 # H1544 + 1545 515 2 0.42380 29.17900 16.83600 4.36700 # H1545 + 1546 516 1 -0.84760 15.65300 29.24700 22.18300 # O1546 + 1547 516 2 0.42380 16.08300 29.85300 21.48000 # H1547 + 1548 516 2 0.42380 15.63600 28.33200 21.80000 # H1548 + 1549 517 1 -0.84760 13.75600 7.55800 7.02500 # O1549 + 1550 517 2 0.42380 14.45100 6.96100 6.56900 # H1550 + 1551 517 2 0.42380 13.09000 7.88500 6.34000 # H1551 + 1552 518 1 -0.84760 16.28000 1.61600 10.70500 # O1552 + 1553 518 2 0.42380 16.38300 2.59800 11.04500 # H1553 + 1554 518 2 0.42380 16.53500 1.66700 9.75400 # H1554 + 1555 519 1 -0.84760 25.61800 26.29400 27.77900 # O1555 + 1556 519 2 0.42380 25.65000 27.12700 27.20200 # H1556 + 1557 519 2 0.42380 24.79500 26.22500 28.38700 # H1557 + 1558 520 1 -0.84760 16.71600 8.30200 23.52000 # O1558 + 1559 520 2 0.42380 17.69400 8.52800 23.29100 # H1559 + 1560 520 2 0.42380 16.20800 9.15200 23.81500 # H1560 + 1561 521 1 -0.84760 7.85000 21.48800 10.10000 # O1561 + 1562 521 2 0.42380 7.39300 22.12800 10.71300 # H1562 + 1563 521 2 0.42380 8.17000 22.06900 9.27900 # H1563 + 1564 522 1 -0.84760 0.06000 3.51400 22.63100 # O1564 + 1565 522 2 0.42380 -0.37000 3.37400 23.54300 # H1565 + 1566 522 2 0.42380 0.70500 2.71700 22.62500 # H1566 + 1567 523 1 -0.84760 28.88200 27.38500 27.81000 # O1567 + 1568 523 2 0.42380 28.99600 28.18200 28.37700 # H1568 + 1569 523 2 0.42380 29.24800 26.67900 28.43100 # H1569 + 1570 524 1 -0.84760 1.14600 20.55500 1.57400 # O1570 + 1571 524 2 0.42380 0.31700 20.90700 1.05800 # H1571 + 1572 524 2 0.42380 1.89300 21.18300 1.48500 # H1572 + 1573 525 1 -0.84760 12.32400 28.67100 23.20500 # O1573 + 1574 525 2 0.42380 13.28800 28.95500 23.32300 # H1574 + 1575 525 2 0.42380 11.99500 29.36700 22.66400 # H1575 + 1576 526 1 -0.84760 0.98100 26.99200 29.83100 # O1576 + 1577 526 2 0.42380 0.93700 26.01200 30.07900 # H1577 + 1578 526 2 0.42380 0.13700 27.45000 29.96100 # H1578 + 1579 527 1 -0.84760 6.68500 1.77700 16.64300 # O1579 + 1580 527 2 0.42380 6.51500 2.60600 16.23600 # H1580 + 1581 527 2 0.42380 7.55500 1.68500 17.16100 # H1581 + 1582 528 1 -0.84760 1.73400 3.69500 27.76300 # O1582 + 1583 528 2 0.42380 2.34300 3.09000 28.39600 # H1583 + 1584 528 2 0.42380 1.32100 4.44500 28.34700 # H1584 + 1585 529 1 -0.84760 26.37900 13.23100 25.82200 # O1585 + 1586 529 2 0.42380 27.03100 14.02100 25.89600 # H1586 + 1587 529 2 0.42380 26.63500 12.64800 26.62300 # H1587 + 1588 530 1 -0.84760 25.90900 2.83400 7.77700 # O1588 + 1589 530 2 0.42380 26.74300 2.32000 7.97900 # H1589 + 1590 530 2 0.42380 25.68400 3.44900 8.59800 # H1590 + 1591 531 1 -0.84760 3.59100 1.74900 28.26500 # O1591 + 1592 531 2 0.42380 4.55500 2.09700 28.49900 # H1592 + 1593 531 2 0.42380 3.29500 2.13100 27.36500 # H1593 + 1594 532 1 -0.84760 7.59200 13.94200 22.65200 # O1594 + 1595 532 2 0.42380 8.27100 14.48300 23.29900 # H1595 + 1596 532 2 0.42380 6.66500 14.33300 22.99400 # H1596 + 1597 533 1 -0.84760 15.31700 9.22900 15.36300 # O1597 + 1598 533 2 0.42380 14.65400 9.16500 16.13800 # H1598 + 1599 533 2 0.42380 15.82100 8.32900 15.55000 # H1599 + 1600 534 1 -0.84760 6.28000 29.19600 3.42400 # O1600 + 1601 534 2 0.42380 6.90800 29.45800 2.67700 # H1601 + 1602 534 2 0.42380 5.83100 28.38900 2.98300 # H1602 + 1603 535 1 -0.84760 -0.54700 11.31600 23.78400 # O1603 + 1604 535 2 0.42380 -1.09100 10.85900 24.50600 # H1604 + 1605 535 2 0.42380 -1.26300 11.74200 23.23500 # H1605 + 1606 536 1 -0.84760 18.48100 19.62300 19.59100 # O1606 + 1607 536 2 0.42380 18.81300 18.97800 20.30500 # H1607 + 1608 536 2 0.42380 19.18900 19.62200 18.83900 # H1608 + 1609 537 1 -0.84760 5.17600 25.45100 12.68900 # O1609 + 1610 537 2 0.42380 4.85900 24.53100 12.93600 # H1610 + 1611 537 2 0.42380 6.11500 25.53000 13.09900 # H1611 + 1612 538 1 -0.84760 9.98200 -0.25900 23.19800 # O1612 + 1613 538 2 0.42380 9.97300 0.75600 22.99000 # H1613 + 1614 538 2 0.42380 9.10200 -0.72700 23.53100 # H1614 + 1615 539 1 -0.84760 6.33600 9.61400 1.54300 # O1615 + 1616 539 2 0.42380 6.19300 9.12300 2.47400 # H1616 + 1617 539 2 0.42380 5.40600 9.58400 1.15300 # H1617 + 1618 540 1 -0.84760 26.16600 8.35500 12.93600 # O1618 + 1619 540 2 0.42380 25.78400 7.43100 12.98900 # H1619 + 1620 540 2 0.42380 25.38900 8.69600 12.38600 # H1620 + 1621 541 1 -0.84760 1.52000 12.64700 29.34400 # O1621 + 1622 541 2 0.42380 0.89400 12.32200 30.12700 # H1622 + 1623 541 2 0.42380 1.69700 11.72300 28.95800 # H1623 + 1624 542 1 -0.84760 8.73200 19.72200 7.42000 # O1624 + 1625 542 2 0.42380 8.11400 20.48900 7.15800 # H1625 + 1626 542 2 0.42380 8.06200 18.95900 7.47000 # H1626 + 1627 543 1 -0.84760 15.89200 2.41800 20.26900 # O1627 + 1628 543 2 0.42380 14.98500 2.88300 20.13800 # H1628 + 1629 543 2 0.42380 16.32300 2.03000 19.42200 # H1629 + 1630 544 1 -0.84760 23.63900 27.48000 1.95200 # O1630 + 1631 544 2 0.42380 22.89800 27.63300 2.66900 # H1631 + 1632 544 2 0.42380 23.84000 28.33400 1.42200 # H1632 + 1633 545 1 -0.84760 15.26200 20.42800 0.70500 # O1633 + 1634 545 2 0.42380 15.48500 19.56900 0.15700 # H1634 + 1635 545 2 0.42380 15.22700 19.97500 1.63900 # H1635 + 1636 546 1 -0.84760 17.01100 22.45900 4.46700 # O1636 + 1637 546 2 0.42380 17.83100 22.03600 4.02400 # H1637 + 1638 546 2 0.42380 16.38300 22.92600 3.84900 # H1638 + 1639 547 1 -0.84760 29.37100 15.17500 8.87600 # O1639 + 1640 547 2 0.42380 29.31700 14.19500 8.65400 # H1640 + 1641 547 2 0.42380 29.88200 15.35400 9.76500 # H1641 + 1642 548 1 -0.84760 28.84500 13.35800 21.50700 # O1642 + 1643 548 2 0.42380 28.19200 13.64000 22.24600 # H1643 + 1644 548 2 0.42380 29.76800 13.68400 21.97800 # H1644 + 1645 549 1 -0.84760 1.34100 3.38200 17.78200 # O1645 + 1646 549 2 0.42380 1.08400 3.65500 16.91000 # H1646 + 1647 549 2 0.42380 1.14200 2.38000 17.64900 # H1647 + 1648 550 1 -0.84760 19.33700 10.61800 20.89500 # O1648 + 1649 550 2 0.42380 18.71800 9.98000 20.38500 # H1649 + 1650 550 2 0.42380 19.87100 9.98700 21.49000 # H1650 + 1651 551 1 -0.84760 23.10600 20.60100 10.94400 # O1651 + 1652 551 2 0.42380 22.68600 21.50600 11.17000 # H1652 + 1653 551 2 0.42380 24.06800 20.87900 10.65200 # H1653 + 1654 552 1 -0.84760 6.03500 16.99200 11.67900 # O1654 + 1655 552 2 0.42380 6.02900 16.91100 12.69600 # H1655 + 1656 552 2 0.42380 7.02600 16.87900 11.58200 # H1656 + 1657 553 1 -0.84760 14.94700 21.44300 6.62700 # O1657 + 1658 553 2 0.42380 14.92500 22.43000 6.25800 # H1658 + 1659 553 2 0.42380 15.58100 20.90500 6.05900 # H1659 + 1660 554 1 -0.84760 14.98000 4.90300 7.11400 # O1660 + 1661 554 2 0.42380 15.82400 4.70100 7.63700 # H1661 + 1662 554 2 0.42380 14.34700 4.96400 7.92100 # H1662 + 1663 555 1 -0.84760 20.05200 -0.99600 17.74900 # O1663 + 1664 555 2 0.42380 20.19200 -0.12900 17.18600 # H1664 + 1665 555 2 0.42380 20.89300 -1.06300 18.29900 # H1665 + 1666 556 1 -0.84760 11.85200 4.54000 2.25200 # O1666 + 1667 556 2 0.42380 12.44000 5.36300 2.08600 # H1667 + 1668 556 2 0.42380 11.90700 4.29600 3.22700 # H1668 + 1669 557 1 -0.84760 17.98800 22.11400 20.54600 # O1669 + 1670 557 2 0.42380 17.67100 22.39100 19.61700 # H1670 + 1671 557 2 0.42380 18.24200 21.12100 20.49900 # H1671 + 1672 558 1 -0.84760 3.61300 26.15900 18.97200 # O1672 + 1673 558 2 0.42380 3.64900 26.51100 18.02800 # H1673 + 1674 558 2 0.42380 3.07800 26.89500 19.40700 # H1674 + 1675 559 1 -0.84760 29.72700 15.51600 11.32600 # O1675 + 1676 559 2 0.42380 29.41100 14.63400 11.75400 # H1676 + 1677 559 2 0.42380 29.08400 16.31800 11.43700 # H1677 + 1678 560 1 -0.84760 21.70000 9.16900 18.44000 # O1678 + 1679 560 2 0.42380 21.31600 9.64500 19.23400 # H1679 + 1680 560 2 0.42380 22.73100 9.33200 18.52500 # H1680 + 1681 561 1 -0.84760 9.36300 27.67800 22.83200 # O1681 + 1682 561 2 0.42380 10.18300 27.84700 23.43400 # H1682 + 1683 561 2 0.42380 9.14100 28.62500 22.48900 # H1683 + 1684 562 1 -0.84760 7.20200 4.15200 8.89600 # O1684 + 1685 562 2 0.42380 6.28100 4.29900 9.26200 # H1685 + 1686 562 2 0.42380 7.07300 3.41500 8.14800 # H1686 + 1687 563 1 -0.84760 19.74900 11.57900 0.04800 # O1687 + 1688 563 2 0.42380 20.20600 11.19700 0.83900 # H1688 + 1689 563 2 0.42380 19.59500 12.54200 0.06600 # H1689 + 1690 564 1 -0.84760 1.82800 25.07400 14.50200 # O1690 + 1691 564 2 0.42380 2.07200 25.93600 13.92700 # H1691 + 1692 564 2 0.42380 1.96400 25.33100 15.47900 # H1692 + 1693 565 1 -0.84760 15.39500 13.78000 27.35300 # O1693 + 1694 565 2 0.42380 15.48800 14.31200 28.16700 # H1694 + 1695 565 2 0.42380 16.19800 13.89700 26.76500 # H1695 + 1696 566 1 -0.84760 11.47800 5.90100 21.32000 # O1696 + 1697 566 2 0.42380 11.10500 6.70400 21.76600 # H1697 + 1698 566 2 0.42380 12.11000 5.51400 21.99900 # H1698 + 1699 567 1 -0.84760 3.04600 10.66200 27.52000 # O1699 + 1700 567 2 0.42380 3.89800 10.08100 27.75900 # H1700 + 1701 567 2 0.42380 3.28400 11.62800 27.78000 # H1701 + 1702 568 1 -0.84760 17.51600 16.71200 20.54400 # O1702 + 1703 568 2 0.42380 18.12600 16.00700 20.16800 # H1703 + 1704 568 2 0.42380 17.18100 17.00200 19.67700 # H1704 + 1705 569 1 -0.84760 15.33900 19.82700 19.86000 # O1705 + 1706 569 2 0.42380 16.21900 19.60700 20.24300 # H1706 + 1707 569 2 0.42380 14.67100 19.92500 20.60900 # H1707 + 1708 570 1 -0.84760 14.67200 18.60600 25.48000 # O1708 + 1709 570 2 0.42380 14.99000 18.00700 26.27900 # H1709 + 1710 570 2 0.42380 14.11600 19.33500 25.98600 # H1710 + 1711 571 1 -0.84760 24.92900 24.16600 7.08500 # O1711 + 1712 571 2 0.42380 24.35000 24.04400 6.25700 # H1712 + 1713 571 2 0.42380 25.26500 23.23000 7.22300 # H1713 + 1714 572 1 -0.84760 12.40400 3.51600 7.29300 # O1714 + 1715 572 2 0.42380 12.40400 2.45400 7.21500 # H1715 + 1716 572 2 0.42380 11.84900 3.63600 8.09200 # H1716 + 1717 573 1 -0.84760 25.58900 2.66900 11.46900 # O1717 + 1718 573 2 0.42380 26.00800 1.83000 11.87600 # H1718 + 1719 573 2 0.42380 26.42500 3.17600 11.22200 # H1719 + 1720 574 1 -0.84760 2.77300 5.68000 6.76500 # O1720 + 1721 574 2 0.42380 1.88000 6.04800 6.83700 # H1721 + 1722 574 2 0.42380 3.16700 5.89800 7.64900 # H1722 + 1723 575 1 -0.84760 16.17700 5.49100 1.78900 # O1723 + 1724 575 2 0.42380 16.55900 5.20300 0.90100 # H1724 + 1725 575 2 0.42380 15.14800 5.43800 1.63700 # H1725 + 1726 576 1 -0.84760 13.81100 21.52500 17.30200 # O1726 + 1727 576 2 0.42380 14.43200 21.33700 16.49600 # H1727 + 1728 576 2 0.42380 14.35300 21.02100 18.06500 # H1728 + 1729 577 1 -0.84760 22.00300 19.23800 14.66100 # O1729 + 1730 577 2 0.42380 22.78700 19.26500 14.01200 # H1730 + 1731 577 2 0.42380 21.40800 19.93100 14.44600 # H1731 + 1732 578 1 -0.84760 22.73300 15.82100 1.69000 # O1732 + 1733 578 2 0.42380 23.09700 14.98000 2.20600 # H1733 + 1734 578 2 0.42380 23.60300 16.28600 1.32800 # H1734 + 1735 579 1 -0.84760 10.08200 20.27600 4.78000 # O1735 + 1736 579 2 0.42380 9.07200 20.17000 4.75100 # H1736 + 1737 579 2 0.42380 10.38700 19.46800 4.21400 # H1737 + 1738 580 1 -0.84760 22.50600 11.35600 20.48900 # O1738 + 1739 580 2 0.42380 22.13700 10.75900 21.24700 # H1739 + 1740 580 2 0.42380 23.38900 10.94600 20.18700 # H1740 + 1741 581 1 -0.84760 17.79800 3.60400 21.72000 # O1741 + 1742 581 2 0.42380 16.96900 3.70100 21.11100 # H1742 + 1743 581 2 0.42380 18.50600 3.98000 21.00300 # H1743 + 1744 582 1 -0.84760 27.14600 14.91000 16.23300 # O1744 + 1745 582 2 0.42380 26.21500 14.93400 16.70800 # H1745 + 1746 582 2 0.42380 27.81000 15.16000 16.98800 # H1746 + 1747 583 1 -0.84760 9.19700 23.05800 19.72700 # O1747 + 1748 583 2 0.42380 10.07300 23.46200 19.27300 # H1748 + 1749 583 2 0.42380 8.68200 22.71000 18.95800 # H1749 + 1750 584 1 -0.84760 15.75300 23.65600 23.56400 # O1750 + 1751 584 2 0.42380 16.48200 24.35700 23.34800 # H1751 + 1752 584 2 0.42380 14.94500 24.23900 23.93200 # H1752 + 1753 585 1 -0.84760 10.50700 5.14600 8.78400 # O1753 + 1754 585 2 0.42380 10.14600 4.50100 8.12300 # H1754 + 1755 585 2 0.42380 10.02600 6.03200 8.65200 # H1755 + 1756 586 1 -0.84760 13.66100 19.26900 13.94000 # O1756 + 1757 586 2 0.42380 12.94800 19.02800 13.29400 # H1757 + 1758 586 2 0.42380 14.61700 19.32400 13.55200 # H1758 + 1759 587 1 -0.84760 5.45300 13.15900 17.18000 # O1759 + 1760 587 2 0.42380 6.16700 13.81100 17.30800 # H1760 + 1761 587 2 0.42380 5.01600 13.57000 16.35300 # H1761 + 1762 588 1 -0.84760 10.59700 27.87900 29.62600 # O1762 + 1763 588 2 0.42380 9.90000 28.34900 30.26100 # H1763 + 1764 588 2 0.42380 11.34600 27.39200 30.13900 # H1764 + 1765 589 1 -0.84760 29.41400 0.32500 6.04600 # O1765 + 1766 589 2 0.42380 29.95800 0.06400 5.27200 # H1766 + 1767 589 2 0.42380 29.91600 1.06500 6.42700 # H1767 + 1768 590 1 -0.84760 27.98300 8.36600 15.01800 # O1768 + 1769 590 2 0.42380 28.91000 8.58500 14.76700 # H1769 + 1770 590 2 0.42380 27.34400 8.99000 14.51800 # H1770 + 1771 591 1 -0.84760 28.20700 15.05600 25.54700 # O1771 + 1772 591 2 0.42380 28.07800 15.96700 25.21900 # H1772 + 1773 591 2 0.42380 28.47600 14.66200 24.64300 # H1773 + 1774 592 1 -0.84760 22.33400 15.75800 23.11100 # O1774 + 1775 592 2 0.42380 21.86400 15.13100 22.51800 # H1775 + 1776 592 2 0.42380 22.22300 15.15000 23.92300 # H1776 + 1777 593 1 -0.84760 23.88600 1.10000 13.45000 # O1777 + 1778 593 2 0.42380 24.23200 1.03400 14.40700 # H1778 + 1779 593 2 0.42380 24.34300 0.46000 12.73900 # H1779 + 1780 594 1 -0.84760 7.85100 10.72000 29.97400 # O1780 + 1781 594 2 0.42380 8.53400 9.94800 29.99700 # H1781 + 1782 594 2 0.42380 6.93000 10.30700 29.94400 # H1782 + 1783 595 1 -0.84760 24.37900 19.47800 12.86200 # O1783 + 1784 595 2 0.42380 23.86900 19.92300 12.05700 # H1784 + 1785 595 2 0.42380 24.72500 18.54500 12.60800 # H1785 + 1786 596 1 -0.84760 8.99300 7.80100 10.34700 # O1786 + 1787 596 2 0.42380 9.99700 7.68000 10.39100 # H1787 + 1788 596 2 0.42380 8.73100 8.59600 10.87200 # H1788 + 1789 597 1 -0.84760 20.30200 23.46300 4.91100 # O1789 + 1790 597 2 0.42380 20.23600 23.93100 5.85400 # H1790 + 1791 597 2 0.42380 20.68400 24.26700 4.37200 # H1791 + 1792 598 1 -0.84760 19.80700 12.01600 18.55300 # O1792 + 1793 598 2 0.42380 20.82600 11.98000 18.61000 # H1793 + 1794 598 2 0.42380 19.58400 11.85300 19.56000 # H1794 + 1795 599 1 -0.84760 7.70400 21.14200 17.75300 # O1795 + 1796 599 2 0.42380 7.49400 20.32500 17.20700 # H1796 + 1797 599 2 0.42380 8.60900 21.04000 18.18300 # H1797 + 1798 600 1 -0.84760 20.03700 20.34500 9.75400 # O1798 + 1799 600 2 0.42380 20.81700 20.30800 9.07400 # H1799 + 1800 600 2 0.42380 19.60100 21.27800 9.58700 # H1800 + 1801 601 1 -0.84760 4.53600 10.29600 24.99400 # O1801 + 1802 601 2 0.42380 4.15600 11.01900 25.62200 # H1802 + 1803 601 2 0.42380 4.22900 10.25000 24.09500 # H1803 + 1804 602 1 -0.84760 20.20300 8.66200 2.26500 # O1804 + 1805 602 2 0.42380 21.17600 8.67800 1.89000 # H1805 + 1806 602 2 0.42380 20.07200 7.67300 2.16600 # H1806 + 1807 603 1 -0.84760 4.48000 20.89100 23.42100 # O1807 + 1808 603 2 0.42380 3.86700 20.27400 23.90400 # H1808 + 1809 603 2 0.42380 5.34700 20.73000 23.80300 # H1809 + 1810 604 1 -0.84760 -0.79400 13.79100 3.31000 # O1810 + 1811 604 2 0.42380 -1.21400 12.91900 3.18900 # H1811 + 1812 604 2 0.42380 -0.59700 13.75300 4.34300 # H1812 + 1813 605 1 -0.84760 16.63200 24.97200 6.11700 # O1813 + 1814 605 2 0.42380 16.24200 24.04600 6.11900 # H1814 + 1815 605 2 0.42380 16.13200 25.48200 6.83100 # H1815 + 1816 606 1 -0.84760 12.84600 14.23600 26.88500 # O1816 + 1817 606 2 0.42380 12.95400 13.25800 26.58000 # H1817 + 1818 606 2 0.42380 13.84900 14.46600 26.93300 # H1818 + 1819 607 1 -0.84760 8.91300 28.57900 16.66400 # O1819 + 1820 607 2 0.42380 9.83600 28.95000 16.89200 # H1820 + 1821 607 2 0.42380 8.60100 27.82600 17.23400 # H1821 + 1822 608 1 -0.84760 4.73900 17.66800 21.47000 # O1822 + 1823 608 2 0.42380 5.43000 17.54300 20.75500 # H1823 + 1824 608 2 0.42380 4.05800 18.20600 21.02000 # H1824 + 1825 609 1 -0.84760 22.72000 14.32000 18.24400 # O1825 + 1826 609 2 0.42380 21.75500 14.16700 18.09800 # H1826 + 1827 609 2 0.42380 23.11800 13.42600 18.50500 # H1827 + 1828 610 1 -0.84760 27.30700 0.98900 21.65200 # O1828 + 1829 610 2 0.42380 27.11700 0.68300 20.71800 # H1829 + 1830 610 2 0.42380 28.22900 0.70400 21.89600 # H1830 + 1831 611 1 -0.84760 12.03700 13.03000 6.86600 # O1831 + 1832 611 2 0.42380 12.97800 12.78900 7.29300 # H1832 + 1833 611 2 0.42380 11.91000 12.28200 6.25600 # H1833 + 1834 612 1 -0.84760 27.48000 24.59700 4.56500 # O1834 + 1835 612 2 0.42380 27.99100 24.89700 5.40600 # H1835 + 1836 612 2 0.42380 26.47500 24.85300 4.64600 # H1836 + 1837 613 1 -0.84760 12.63200 12.30900 19.19100 # O1837 + 1838 613 2 0.42380 11.75700 12.17800 19.61400 # H1838 + 1839 613 2 0.42380 12.38700 12.44000 18.23500 # H1839 + 1840 614 1 -0.84760 0.49800 4.06900 6.11100 # O1840 + 1841 614 2 0.42380 1.47900 3.78500 6.19400 # H1841 + 1842 614 2 0.42380 0.08100 4.01700 7.01900 # H1842 + 1843 615 1 -0.84760 3.37300 23.46700 13.07800 # O1843 + 1844 615 2 0.42380 3.07200 24.19300 12.42400 # H1844 + 1845 615 2 0.42380 2.92700 23.89600 13.86000 # H1845 + 1846 616 1 -0.84760 29.44500 25.39100 29.60400 # O1846 + 1847 616 2 0.42380 28.81600 25.69900 30.25100 # H1847 + 1848 616 2 0.42380 29.11000 24.96700 28.69700 # H1848 + 1849 617 1 -0.84760 9.04800 23.00000 24.32700 # O1849 + 1850 617 2 0.42380 8.86800 23.84500 24.87600 # H1850 + 1851 617 2 0.42380 9.56500 23.11000 23.51100 # H1851 + 1852 618 1 -0.84760 11.70500 6.73500 17.70600 # O1852 + 1853 618 2 0.42380 12.28500 6.44200 18.51700 # H1853 + 1854 618 2 0.42380 12.00300 6.01000 16.99600 # H1854 + 1855 619 1 -0.84760 5.41100 11.28000 21.91600 # O1855 + 1856 619 2 0.42380 5.59900 10.86900 20.90900 # H1856 + 1857 619 2 0.42380 6.28800 11.07700 22.44500 # H1857 + 1858 620 1 -0.84760 21.54600 14.16700 0.21200 # O1858 + 1859 620 2 0.42380 21.16100 13.58400 0.94600 # H1859 + 1860 620 2 0.42380 22.07400 14.87300 0.70100 # H1860 + 1861 621 1 -0.84760 26.64900 18.89600 10.73100 # O1861 + 1862 621 2 0.42380 26.38700 18.65600 11.61700 # H1862 + 1863 621 2 0.42380 26.76700 18.11000 10.22400 # H1863 + 1864 622 1 -0.84760 22.30600 0.74100 7.56700 # O1864 + 1865 622 2 0.42380 21.52400 1.32400 7.87000 # H1865 + 1866 622 2 0.42380 23.09500 1.13300 8.10200 # H1866 + 1867 623 1 -0.84760 18.77200 5.39900 28.79900 # O1867 + 1868 623 2 0.42380 18.08100 4.69100 29.06400 # H1868 + 1869 623 2 0.42380 18.14500 6.17400 28.64200 # H1869 + 1870 624 1 -0.84760 -0.22200 15.25500 23.14000 # O1870 + 1871 624 2 0.42380 0.25800 15.29900 22.23100 # H1871 + 1872 624 2 0.42380 0.07500 16.16500 23.55300 # H1872 + 1873 625 1 -0.84760 10.89700 17.61800 15.00500 # O1873 + 1874 625 2 0.42380 11.76800 17.59100 14.50900 # H1874 + 1875 625 2 0.42380 11.13100 17.97600 15.91500 # H1875 + 1876 626 1 -0.84760 5.31400 21.75700 11.67100 # O1876 + 1877 626 2 0.42380 5.36800 21.97300 12.55700 # H1877 + 1878 626 2 0.42380 5.41100 22.58500 11.23400 # H1878 + 1879 627 1 -0.84760 23.44300 22.96800 3.19800 # O1879 + 1880 627 2 0.42380 23.04000 23.33700 2.35600 # H1880 + 1881 627 2 0.42380 23.93200 22.18200 2.77500 # H1881 + 1882 628 1 -0.84760 25.28400 17.14300 27.44500 # O1882 + 1883 628 2 0.42380 24.70100 17.03000 28.28200 # H1883 + 1884 628 2 0.42380 24.64800 17.61900 26.83200 # H1884 + 1885 629 1 -0.84760 26.65400 3.82400 27.02600 # O1885 + 1886 629 2 0.42380 26.20600 3.53400 26.13500 # H1886 + 1887 629 2 0.42380 25.79300 4.02500 27.60000 # H1887 + 1888 630 1 -0.84760 19.16200 15.67800 10.32100 # O1888 + 1889 630 2 0.42380 19.77100 16.02000 9.59300 # H1889 + 1890 630 2 0.42380 19.16500 16.16900 11.16600 # H1890 + 1891 631 1 -0.84760 27.70400 12.60300 -0.34200 # O1891 + 1892 631 2 0.42380 27.97300 13.58300 -0.54300 # H1892 + 1893 631 2 0.42380 27.61100 12.17100 -1.29500 # H1893 + 1894 632 1 -0.84760 1.17600 21.82200 11.66500 # O1894 + 1895 632 2 0.42380 1.42500 21.09800 12.32700 # H1895 + 1896 632 2 0.42380 0.99000 22.57800 12.35900 # H1896 + 1897 633 1 -0.84760 -0.71000 18.00300 22.82400 # O1897 + 1898 633 2 0.42380 -0.34300 18.92200 23.06400 # H1898 + 1899 633 2 0.42380 -1.57500 18.15700 22.29600 # H1899 + 1900 634 1 -0.84760 21.16000 22.09300 2.65100 # O1900 + 1901 634 2 0.42380 21.77400 21.30800 2.42500 # H1901 + 1902 634 2 0.42380 20.71100 21.82100 3.46300 # H1902 + 1903 635 1 -0.84760 5.96000 -0.54800 17.77200 # O1903 + 1904 635 2 0.42380 6.12100 0.41500 17.41400 # H1904 + 1905 635 2 0.42380 5.22100 -0.59300 18.47600 # H1905 + 1906 636 1 -0.84760 17.11800 4.29100 17.54400 # O1906 + 1907 636 2 0.42380 16.98800 3.52500 18.21000 # H1907 + 1908 636 2 0.42380 17.30000 4.00300 16.56900 # H1908 + 1909 637 1 -0.84760 0.93400 23.30900 23.29900 # O1909 + 1910 637 2 0.42380 0.28800 24.10500 23.16000 # H1910 + 1911 637 2 0.42380 0.70200 22.38100 22.84900 # H1911 + 1912 638 1 -0.84760 19.73200 9.67500 26.94400 # O1912 + 1913 638 2 0.42380 19.27100 10.19500 27.67700 # H1913 + 1914 638 2 0.42380 20.75700 9.68400 27.07100 # H1914 + 1915 639 1 -0.84760 22.17200 -0.58400 22.45600 # O1915 + 1916 639 2 0.42380 21.61000 -0.46900 23.30900 # H1916 + 1917 639 2 0.42380 23.12100 -0.79600 22.77400 # H1917 + 1918 640 1 -0.84760 11.21700 22.37800 22.31500 # O1918 + 1919 640 2 0.42380 11.47000 22.40000 21.36200 # H1919 + 1920 640 2 0.42380 11.39400 23.32300 22.60600 # H1920 + 1921 641 1 -0.84760 7.18900 29.48300 24.33500 # O1921 + 1922 641 2 0.42380 7.52700 29.13400 25.18700 # H1922 + 1923 641 2 0.42380 6.29000 29.74100 24.78900 # H1923 + 1924 642 1 -0.84760 21.71400 1.70700 27.35900 # O1924 + 1925 642 2 0.42380 20.92800 1.86900 28.01200 # H1925 + 1926 642 2 0.42380 22.55400 1.99200 27.83900 # H1926 + 1927 643 1 -0.84760 1.73600 9.28600 17.29200 # O1927 + 1928 643 2 0.42380 0.93500 8.99000 16.69900 # H1928 + 1929 643 2 0.42380 1.85200 8.54800 17.93100 # H1929 + 1930 644 1 -0.84760 2.52400 19.65100 29.36500 # O1930 + 1931 644 2 0.42380 2.62800 20.03500 30.28900 # H1931 + 1932 644 2 0.42380 2.22400 18.68000 29.54500 # H1932 + 1933 645 1 -0.84760 19.07900 2.71800 23.99400 # O1933 + 1934 645 2 0.42380 19.45500 3.57100 24.35100 # H1934 + 1935 645 2 0.42380 18.48200 3.02000 23.26300 # H1935 + 1936 646 1 -0.84760 16.52300 0.10800 29.68600 # O1936 + 1937 646 2 0.42380 15.53000 0.23400 29.70700 # H1937 + 1938 646 2 0.42380 16.81900 -0.87000 29.84600 # H1938 + 1939 647 1 -0.84760 8.02200 1.91400 25.66800 # O1939 + 1940 647 2 0.42380 8.20700 2.49500 26.41500 # H1940 + 1941 647 2 0.42380 8.37100 1.06900 25.91100 # H1941 + 1942 648 1 -0.84760 9.14200 19.99600 14.37100 # O1942 + 1943 648 2 0.42380 9.84400 19.26700 14.18600 # H1943 + 1944 648 2 0.42380 9.72500 20.72300 14.75800 # H1944 + 1945 649 1 -0.84760 23.47000 -0.20900 2.08800 # O1945 + 1946 649 2 0.42380 23.85900 -0.40600 3.00500 # H1946 + 1947 649 2 0.42380 23.95700 0.65000 1.71100 # H1947 + 1948 650 1 -0.84760 5.52300 8.09700 23.48400 # O1948 + 1949 650 2 0.42380 6.19600 7.51400 23.14200 # H1949 + 1950 650 2 0.42380 5.43500 8.79100 22.79800 # H1950 + 1951 651 1 -0.84760 2.62600 16.75000 10.24000 # O1951 + 1952 651 2 0.42380 2.70600 17.32700 11.12900 # H1952 + 1953 651 2 0.42380 2.95900 15.82600 10.49100 # H1953 + 1954 652 1 -0.84760 11.19100 16.79700 26.60100 # O1954 + 1955 652 2 0.42380 11.92700 16.07000 26.47100 # H1955 + 1956 652 2 0.42380 10.82100 17.24400 25.73600 # H1956 + 1957 653 1 -0.84760 -0.48700 25.51900 17.05000 # O1957 + 1958 653 2 0.42380 0.20200 24.74700 17.04600 # H1958 + 1959 653 2 0.42380 -0.03700 26.41000 17.18200 # H1959 + 1960 654 1 -0.84760 5.73400 4.34100 16.50900 # O1960 + 1961 654 2 0.42380 5.54000 5.28500 16.76600 # H1961 + 1962 654 2 0.42380 4.98100 3.72600 16.81300 # H1962 + 1963 655 1 -0.84760 21.92600 5.03600 25.65800 # O1963 + 1964 655 2 0.42380 22.19600 4.70100 24.73700 # H1964 + 1965 655 2 0.42380 21.33700 4.28900 26.14200 # H1965 + 1966 656 1 -0.84760 21.02500 17.42000 17.74300 # O1966 + 1967 656 2 0.42380 20.79000 16.47600 17.94800 # H1967 + 1968 656 2 0.42380 20.95100 17.40600 16.73200 # H1968 + 1969 657 1 -0.84760 5.03800 7.69900 7.73600 # O1969 + 1970 657 2 0.42380 4.30100 8.08800 7.11900 # H1970 + 1971 657 2 0.42380 4.79300 7.91400 8.67800 # H1971 + 1972 658 1 -0.84760 1.44200 5.67000 20.84000 # O1972 + 1973 658 2 0.42380 1.55200 4.82000 20.29900 # H1973 + 1974 658 2 0.42380 0.46000 5.92300 20.72800 # H1974 + 1975 659 1 -0.84760 24.06400 12.94600 2.13700 # O1975 + 1976 659 2 0.42380 24.62100 13.66400 2.60700 # H1976 + 1977 659 2 0.42380 24.82000 12.35200 1.90600 # H1977 + 1978 660 1 -0.84760 2.56700 3.72000 0.18100 # O1978 + 1979 660 2 0.42380 1.70100 4.12400 -0.12600 # H1979 + 1980 660 2 0.42380 2.82500 4.04200 1.07800 # H1980 + 1981 661 1 -0.84760 -0.44700 18.58700 2.12200 # O1981 + 1982 661 2 0.42380 -1.24400 18.67000 1.57800 # H1982 + 1983 661 2 0.42380 0.27000 19.27700 1.88700 # H1983 + 1984 662 1 -0.84760 1.90300 5.38700 13.80700 # O1984 + 1985 662 2 0.42380 1.38900 6.18700 13.51700 # H1985 + 1986 662 2 0.42380 2.19500 5.12600 12.89400 # H1986 + 1987 663 1 -0.84760 18.77800 13.20600 28.18900 # O1987 + 1988 663 2 0.42380 17.91900 13.64100 27.85300 # H1988 + 1989 663 2 0.42380 19.56500 13.79800 27.80900 # H1989 + 1990 664 1 -0.84760 23.21900 4.65900 0.68700 # O1990 + 1991 664 2 0.42380 24.08900 5.05700 0.94000 # H1991 + 1992 664 2 0.42380 22.77600 5.22500 -0.01700 # H1992 + 1993 665 1 -0.84760 -0.08900 26.08200 26.11000 # O1993 + 1994 665 2 0.42380 -0.05800 25.03500 26.12000 # H1994 + 1995 665 2 0.42380 -0.87900 26.39100 25.57400 # H1995 + 1996 666 1 -0.84760 27.84000 27.57100 0.25400 # O1996 + 1997 666 2 0.42380 27.81700 26.64200 0.75400 # H1997 + 1998 666 2 0.42380 26.96200 28.06900 0.63600 # H1998 + 1999 667 1 -0.84760 16.24000 24.30800 20.74600 # O1999 + 2000 667 2 0.42380 16.50500 23.30200 20.84600 # H2000 + 2001 667 2 0.42380 15.27500 24.30800 21.02500 # H2001 + 2002 668 1 -0.84760 29.19000 12.26700 5.28000 # O2002 + 2003 668 2 0.42380 30.00800 11.72600 5.55300 # H2003 + 2004 668 2 0.42380 28.92900 11.83600 4.41100 # H2004 + 2005 669 1 -0.84760 29.74400 9.66200 13.37300 # O2005 + 2006 669 2 0.42380 29.73700 10.38500 14.12300 # H2006 + 2007 669 2 0.42380 30.21400 10.08600 12.56700 # H2007 + 2008 670 1 -0.84760 1.60800 10.91900 22.00000 # O2008 + 2009 670 2 0.42380 2.53100 11.33100 22.12900 # H2009 + 2010 670 2 0.42380 1.05700 10.98000 22.88500 # H2010 + 2011 671 1 -0.84760 18.57000 25.31600 25.79400 # O2011 + 2012 671 2 0.42380 18.52700 24.32700 25.73000 # H2012 + 2013 671 2 0.42380 18.29500 25.74400 24.86000 # H2013 + 2014 672 1 -0.84760 0.14800 2.78100 25.53200 # O2014 + 2015 672 2 0.42380 0.00400 3.72700 25.64800 # H2015 + 2016 672 2 0.42380 1.02400 2.44400 25.86700 # H2016 + 2017 673 1 -0.84760 3.56600 13.36300 28.05200 # O2017 + 2018 673 2 0.42380 3.93200 14.24500 28.48500 # H2018 + 2019 673 2 0.42380 2.76100 13.22000 28.68700 # H2019 + 2020 674 1 -0.84760 28.31900 16.08400 19.28600 # O2020 + 2021 674 2 0.42380 28.69600 15.42400 19.97700 # H2021 + 2022 674 2 0.42380 28.27600 17.06100 19.68400 # H2022 + 2023 675 1 -0.84760 23.17300 30.05100 16.60800 # O2023 + 2024 675 2 0.42380 23.11900 29.01600 16.71800 # H2024 + 2025 675 2 0.42380 22.35500 30.20300 16.00000 # H2025 + 2026 676 1 -0.84760 1.30000 5.69800 24.43900 # O2026 + 2027 676 2 0.42380 0.63300 5.47800 23.74600 # H2027 + 2028 676 2 0.42380 2.08700 6.17600 23.92400 # H2028 + 2029 677 1 -0.84760 1.42000 -0.07600 26.06900 # O2029 + 2030 677 2 0.42380 1.50800 -1.12100 26.17700 # H2030 + 2031 677 2 0.42380 0.81800 0.05000 26.84600 # H2031 + 2032 678 1 -0.84760 7.05800 7.32500 3.90700 # O2032 + 2033 678 2 0.42380 7.47300 7.81700 4.71900 # H2033 + 2034 678 2 0.42380 7.36200 6.41200 4.09600 # H2034 + 2035 679 1 -0.84760 8.60500 11.27400 14.18600 # O2035 + 2036 679 2 0.42380 8.33400 12.22700 13.93500 # H2036 + 2037 679 2 0.42380 8.75800 10.67300 13.31900 # H2037 + 2038 680 1 -0.84760 21.29500 24.95300 16.15300 # O2038 + 2039 680 2 0.42380 21.19600 23.96400 16.32100 # H2039 + 2040 680 2 0.42380 21.97300 25.14200 16.94800 # H2040 + 2041 681 1 -0.84760 26.80700 13.01900 4.59200 # O2041 + 2042 681 2 0.42380 27.58400 12.56500 5.14400 # H2042 + 2043 681 2 0.42380 27.08500 13.93400 5.10700 # H2043 + 2044 682 1 -0.84760 24.38700 1.49700 5.75700 # O2044 + 2045 682 2 0.42380 23.96900 0.58400 5.99000 # H2045 + 2046 682 2 0.42380 25.05300 1.76300 6.39600 # H2046 + 2047 683 1 -0.84760 4.71700 25.48400 26.84500 # O2047 + 2048 683 2 0.42380 4.31700 26.24600 27.32200 # H2048 + 2049 683 2 0.42380 4.37000 24.65700 27.28600 # H2049 + 2050 684 1 -0.84760 25.27600 16.20800 18.59800 # O2050 + 2051 684 2 0.42380 25.56200 16.12600 19.54300 # H2051 + 2052 684 2 0.42380 24.50400 15.60800 18.41800 # H2052 + 2053 685 1 -0.84760 11.03700 14.66000 0.63800 # O2053 + 2054 685 2 0.42380 10.83700 14.73300 -0.35700 # H2054 + 2055 685 2 0.42380 11.06500 13.68600 0.81800 # H2055 + 2056 686 1 -0.84760 25.03400 19.44600 19.16600 # O2056 + 2057 686 2 0.42380 24.05600 19.21800 18.91100 # H2057 + 2058 686 2 0.42380 25.01900 20.37000 18.84000 # H2058 + 2059 687 1 -0.84760 12.16800 4.75800 15.85200 # O2059 + 2060 687 2 0.42380 11.80000 4.04200 16.34200 # H2060 + 2061 687 2 0.42380 11.38700 5.17800 15.35600 # H2061 + 2062 688 1 -0.84760 24.08400 19.84100 16.26600 # O2062 + 2063 688 2 0.42380 24.68200 19.09500 16.57300 # H2063 + 2064 688 2 0.42380 23.32700 19.51500 15.77800 # H2064 + 2065 689 1 -0.84760 3.21400 8.80100 6.31300 # O2065 + 2066 689 2 0.42380 3.95200 8.95100 5.60900 # H2066 + 2067 689 2 0.42380 3.08200 9.67400 6.81500 # H2067 + 2068 690 1 -0.84760 1.95700 14.14100 8.85600 # O2068 + 2069 690 2 0.42380 1.69300 13.55800 9.63800 # H2069 + 2070 690 2 0.42380 1.71200 15.11200 9.09500 # H2070 + 2071 691 1 -0.84760 28.36400 0.14300 14.57400 # O2071 + 2072 691 2 0.42380 28.43900 -0.85100 14.44700 # H2072 + 2073 691 2 0.42380 28.55000 0.23800 15.60900 # H2073 + 2074 692 1 -0.84760 13.02400 22.22100 25.78100 # O2074 + 2075 692 2 0.42380 12.32600 21.85900 25.13800 # H2075 + 2076 692 2 0.42380 12.91600 21.77900 26.66000 # H2076 + 2077 693 1 -0.84760 19.23800 17.50500 29.03900 # O2077 + 2078 693 2 0.42380 19.61700 18.37400 28.63800 # H2078 + 2079 693 2 0.42380 18.97800 16.84100 28.29300 # H2079 + 2080 694 1 -0.84760 20.90500 27.74300 6.43600 # O2080 + 2081 694 2 0.42380 21.06000 27.37400 5.53500 # H2081 + 2082 694 2 0.42380 20.46100 28.57700 6.30200 # H2082 + 2083 695 1 -0.84760 9.97300 0.17100 1.97500 # O2083 + 2084 695 2 0.42380 10.51400 -0.63400 1.74000 # H2084 + 2085 695 2 0.42380 9.03200 -0.04200 2.26900 # H2085 + 2086 696 1 -0.84760 26.75400 14.96000 22.82600 # O2086 + 2087 696 2 0.42380 26.25400 14.87300 21.96100 # H2087 + 2088 696 2 0.42380 26.42300 15.50800 23.58200 # H2088 + 2089 697 1 -0.84760 5.71300 7.36000 16.36000 # O2089 + 2090 697 2 0.42380 5.93400 6.69200 17.06200 # H2090 + 2091 697 2 0.42380 5.57600 8.28600 16.80300 # H2091 + 2092 698 1 -0.84760 3.23300 2.65400 15.49000 # O2092 + 2093 698 2 0.42380 3.45700 2.26800 16.36900 # H2093 + 2094 698 2 0.42380 2.97600 3.56900 15.68500 # H2094 + 2095 699 1 -0.84760 15.03200 12.36000 3.08600 # O2095 + 2096 699 2 0.42380 15.82300 12.77100 3.55700 # H2096 + 2097 699 2 0.42380 14.73200 13.04800 2.43600 # H2097 + 2098 700 1 -0.84760 16.15000 24.98600 0.48600 # O2098 + 2099 700 2 0.42380 15.77800 24.17900 0.09600 # H2099 + 2100 700 2 0.42380 16.78100 25.26600 -0.23000 # H2100 + 2101 701 1 -0.84760 14.43800 7.83800 11.39700 # O2101 + 2102 701 2 0.42380 13.90400 7.24100 12.09000 # H2102 + 2103 701 2 0.42380 14.75200 8.69500 11.95200 # H2103 + 2104 702 1 -0.84760 0.99700 19.52600 18.22200 # O2104 + 2105 702 2 0.42380 1.95800 19.21500 18.40400 # H2105 + 2106 702 2 0.42380 0.69600 19.79400 19.08200 # H2106 + 2107 703 1 -0.84760 16.30600 17.56600 27.41500 # O2107 + 2108 703 2 0.42380 16.05600 16.65700 27.84200 # H2108 + 2109 703 2 0.42380 17.31900 17.77000 27.19500 # H2109 + 2110 704 1 -0.84760 18.94300 28.33100 29.18900 # O2110 + 2111 704 2 0.42380 18.70700 28.75200 30.06700 # H2111 + 2112 704 2 0.42380 19.91900 28.21300 29.11500 # H2112 + 2113 705 1 -0.84760 14.84300 1.93100 17.50600 # O2113 + 2114 705 2 0.42380 15.23800 1.06300 17.17300 # H2114 + 2115 705 2 0.42380 13.87600 1.60700 17.40200 # H2115 + 2116 706 1 -0.84760 17.72600 2.28600 8.42300 # O2116 + 2117 706 2 0.42380 16.90200 2.12000 7.84000 # H2117 + 2118 706 2 0.42380 18.42500 1.53000 8.36600 # H2118 + 2119 707 1 -0.84760 6.45800 29.20900 21.46100 # O2119 + 2120 707 2 0.42380 6.76400 28.87500 22.37000 # H2120 + 2121 707 2 0.42380 6.02700 28.47600 20.97400 # H2121 + 2122 708 1 -0.84760 0.46000 15.49500 1.81800 # O2122 + 2123 708 2 0.42380 -0.05500 14.68300 2.09300 # H2123 + 2124 708 2 0.42380 -0.28600 16.04800 1.46200 # H2124 + 2125 709 1 -0.84760 10.36800 19.79400 22.30400 # O2125 + 2126 709 2 0.42380 10.87200 20.66800 22.39900 # H2126 + 2127 709 2 0.42380 10.23800 19.44900 23.24500 # H2127 + 2128 710 1 -0.84760 2.72600 11.89400 6.09600 # O2128 + 2129 710 2 0.42380 3.50100 11.29700 6.28400 # H2129 + 2130 710 2 0.42380 3.08200 12.83400 6.23900 # H2130 + 2131 711 1 -0.84760 22.46300 22.56100 8.61400 # O2131 + 2132 711 2 0.42380 22.93600 21.72300 8.38100 # H2132 + 2133 711 2 0.42380 22.76600 23.35100 8.09200 # H2133 + 2134 712 1 -0.84760 12.33800 -0.14000 26.01900 # O2134 + 2135 712 2 0.42380 13.13100 0.43300 25.78800 # H2135 + 2136 712 2 0.42380 12.47900 -1.01500 25.54800 # H2136 + 2137 713 1 -0.84760 4.18500 3.69700 5.53000 # O2137 + 2138 713 2 0.42380 3.37600 3.45200 4.96800 # H2138 + 2139 713 2 0.42380 3.76700 4.40000 6.19500 # H2139 + 2140 714 1 -0.84760 24.69100 0.47500 8.96300 # O2140 + 2141 714 2 0.42380 24.71100 -0.45800 9.06100 # H2141 + 2142 714 2 0.42380 25.48000 0.71400 9.50500 # H2142 + 2143 715 1 -0.84760 21.53600 27.81900 17.15700 # O2143 + 2144 715 2 0.42380 20.87600 27.80500 16.37600 # H2144 + 2145 715 2 0.42380 21.06500 27.74000 18.01400 # H2145 + 2146 716 1 -0.84760 27.65300 16.88400 29.37400 # O2146 + 2147 716 2 0.42380 26.59900 16.91700 29.45700 # H2147 + 2148 716 2 0.42380 28.09300 16.07300 28.85600 # H2148 + 2149 717 1 -0.84760 1.93200 9.83300 4.00400 # O2149 + 2150 717 2 0.42380 1.96500 8.91200 4.40500 # H2150 + 2151 717 2 0.42380 1.47600 9.72800 3.07500 # H2151 + 2152 718 1 -0.84760 17.88200 28.63900 10.24600 # O2152 + 2153 718 2 0.42380 17.91600 29.51900 9.71400 # H2153 + 2154 718 2 0.42380 17.91500 27.82900 9.57200 # H2154 + 2155 719 1 -0.84760 25.88300 13.38800 19.14400 # O2155 + 2156 719 2 0.42380 26.77300 13.84800 19.39900 # H2156 + 2157 719 2 0.42380 25.10900 13.90300 18.75000 # H2157 + 2158 720 1 -0.84760 4.55800 12.13900 8.04500 # O2158 + 2159 720 2 0.42380 5.34200 12.80700 7.95000 # H2159 + 2160 720 2 0.42380 3.78800 12.68400 8.43400 # H2160 + 2161 721 1 -0.84760 5.39400 5.51200 25.28900 # O2161 + 2162 721 2 0.42380 5.26800 4.54800 25.58300 # H2162 + 2163 721 2 0.42380 5.43400 5.61700 24.27700 # H2163 + 2164 722 1 -0.84760 17.95400 21.11900 1.63900 # O2164 + 2165 722 2 0.42380 18.37700 21.91900 1.25300 # H2165 + 2166 722 2 0.42380 16.89300 21.23500 1.68800 # H2166 + 2167 723 1 -0.84760 9.57500 10.91500 9.53400 # O2167 + 2168 723 2 0.42380 10.34600 11.30200 9.99100 # H2168 + 2169 723 2 0.42380 10.03600 10.55100 8.69200 # H2169 + 2170 724 1 -0.84760 21.95700 13.22200 15.14600 # O2170 + 2171 724 2 0.42380 21.94000 14.04700 15.78600 # H2171 + 2172 724 2 0.42380 21.12000 12.60400 15.34100 # H2172 + 2173 725 1 -0.84760 19.22700 18.06900 25.35300 # O2173 + 2174 725 2 0.42380 19.31400 19.05000 25.55300 # H2174 + 2175 725 2 0.42380 19.25700 18.07200 24.41100 # H2175 + 2176 726 1 -0.84760 26.90300 1.16300 2.78400 # O2176 + 2177 726 2 0.42380 26.91300 0.15600 2.82300 # H2177 + 2178 726 2 0.42380 26.47500 1.50100 3.62800 # H2178 + 2179 727 1 -0.84760 8.12300 1.10700 4.21900 # O2179 + 2180 727 2 0.42380 7.97900 2.05800 4.07400 # H2180 + 2181 727 2 0.42380 8.62800 0.91100 5.07200 # H2181 + 2182 728 1 -0.84760 24.55400 2.42900 2.15600 # O2182 + 2183 728 2 0.42380 24.95200 2.40500 1.21700 # H2183 + 2184 728 2 0.42380 23.53700 2.37900 2.13500 # H2184 + 2185 729 1 -0.84760 23.25000 11.77400 8.41300 # O2185 + 2186 729 2 0.42380 22.37900 11.33200 8.55600 # H2186 + 2187 729 2 0.42380 23.91500 11.06400 8.31400 # H2187 + 2188 730 1 -0.84760 29.09400 12.51300 27.75300 # O2188 + 2189 730 2 0.42380 29.85100 12.07700 28.19800 # H2189 + 2190 730 2 0.42380 29.45400 13.32300 27.29400 # H2190 + 2191 731 1 -0.84760 28.72900 27.85500 25.34100 # O2191 + 2192 731 2 0.42380 28.50100 28.78200 25.87300 # H2192 + 2193 731 2 0.42380 28.82900 27.15400 26.09000 # H2193 + 2194 732 1 -0.84760 0.96000 16.58500 28.82500 # O2194 + 2195 732 2 0.42380 1.82200 16.24500 29.29200 # H2195 + 2196 732 2 0.42380 0.80000 15.76100 28.17800 # H2196 + 2197 733 1 -0.84760 8.23700 22.27700 1.83900 # O2197 + 2198 733 2 0.42380 8.66700 21.68400 2.53900 # H2198 + 2199 733 2 0.42380 8.38500 23.22200 2.23000 # H2199 + 2200 734 1 -0.84760 17.67200 18.47700 1.64000 # O2200 + 2201 734 2 0.42380 17.45900 19.48000 1.75900 # H2201 + 2202 734 2 0.42380 18.26800 18.28800 2.40200 # H2202 + 2203 735 1 -0.84760 29.62700 8.19700 28.35600 # O2203 + 2204 735 2 0.42380 29.30200 7.65400 27.52200 # H2204 + 2205 735 2 0.42380 30.68500 8.33600 28.20800 # H2205 + 2206 736 1 -0.84760 25.89200 18.12600 16.47200 # O2206 + 2207 736 2 0.42380 26.85700 18.04500 16.73700 # H2207 + 2208 736 2 0.42380 25.44400 17.33500 16.92100 # H2208 + 2209 737 1 -0.84760 18.13100 5.55400 24.78700 # O2209 + 2210 737 2 0.42380 17.46000 5.35000 24.08100 # H2210 + 2211 737 2 0.42380 17.90600 6.55500 25.03600 # H2211 + 2212 738 1 -0.84760 10.08500 11.61900 23.61800 # O2212 + 2213 738 2 0.42380 10.03200 10.74100 24.08500 # H2213 + 2214 738 2 0.42380 9.22900 11.69100 23.19700 # H2214 + 2215 739 1 -0.84760 14.30600 12.17100 15.93200 # O2215 + 2216 739 2 0.42380 14.08000 13.09100 15.44600 # H2216 + 2217 739 2 0.42380 14.24100 11.42100 15.18100 # H2217 + 2218 740 1 -0.84760 8.81000 14.19300 12.51200 # O2218 + 2219 740 2 0.42380 9.49000 14.04600 11.86000 # H2219 + 2220 740 2 0.42380 8.98900 15.15300 12.85600 # H2220 + 2221 741 1 -0.84760 14.78800 18.96800 16.39400 # O2221 + 2222 741 2 0.42380 13.79800 19.21900 16.30600 # H2222 + 2223 741 2 0.42380 14.91100 18.26900 15.65900 # H2223 + 2224 742 1 -0.84760 25.62000 23.70600 9.91100 # O2224 + 2225 742 2 0.42380 26.30400 23.90600 9.20200 # H2225 + 2226 742 2 0.42380 26.18500 23.66700 10.80600 # H2226 + 2227 743 1 -0.84760 22.87900 25.85800 13.06000 # O2227 + 2228 743 2 0.42380 22.57500 25.82100 14.01500 # H2228 + 2229 743 2 0.42380 22.38200 26.65800 12.72100 # H2229 + 2230 744 1 -0.84760 25.45900 28.98700 10.22500 # O2230 + 2231 744 2 0.42380 24.83800 28.34900 10.68800 # H2231 + 2232 744 2 0.42380 25.29000 28.91700 9.24400 # H2232 + 2233 745 1 -0.84760 17.35500 13.40700 8.81500 # O2233 + 2234 745 2 0.42380 17.28700 12.39100 8.92400 # H2234 + 2235 745 2 0.42380 17.99500 13.66200 9.54400 # H2235 + 2236 746 1 -0.84760 0.27800 7.20600 12.23400 # O2236 + 2237 746 2 0.42380 0.51500 8.16300 12.44300 # H2237 + 2238 746 2 0.42380 -0.68300 7.19400 11.95700 # H2238 + 2239 747 1 -0.84760 4.29000 1.99600 8.57700 # O2239 + 2240 747 2 0.42380 4.35100 1.99000 9.60900 # H2240 + 2241 747 2 0.42380 5.18600 1.78900 8.26700 # H2241 + 2242 748 1 -0.84760 28.20800 9.24900 10.44200 # O2242 + 2243 748 2 0.42380 27.36600 8.91200 9.90900 # H2243 + 2244 748 2 0.42380 27.91800 9.80900 11.22500 # H2244 + 2245 749 1 -0.84760 0.19800 10.32300 6.06600 # O2245 + 2246 749 2 0.42380 0.56300 9.43400 5.77200 # H2246 + 2247 749 2 0.42380 0.96300 11.02800 5.96400 # H2247 + 2248 750 1 -0.84760 28.52500 23.77100 20.34400 # O2248 + 2249 750 2 0.42380 27.57300 23.39600 20.36500 # H2249 + 2250 750 2 0.42380 28.50700 24.47300 19.56900 # H2250 + 2251 751 1 -0.84760 -0.59000 11.34800 -0.43000 # O2251 + 2252 751 2 0.42380 -0.47400 10.54200 -1.04400 # H2252 + 2253 751 2 0.42380 -1.46200 11.21200 0.06500 # H2253 + 2254 752 1 -0.84760 9.11900 4.02500 29.65600 # O2254 + 2255 752 2 0.42380 9.59700 3.29400 30.29600 # H2255 + 2256 752 2 0.42380 8.32200 4.28500 30.28800 # H2256 + 2257 753 1 -0.84760 16.63100 16.03300 2.70800 # O2257 + 2258 753 2 0.42380 16.15100 16.86900 2.38500 # H2258 + 2259 753 2 0.42380 17.28800 16.41900 3.39800 # H2259 + 2260 754 1 -0.84760 17.56900 7.61700 28.47100 # O2260 + 2261 754 2 0.42380 17.00200 8.17300 27.84300 # H2261 + 2262 754 2 0.42380 18.09300 8.34300 28.92900 # H2262 + 2263 755 1 -0.84760 6.78000 9.43800 9.19300 # O2263 + 2264 755 2 0.42380 6.99800 8.50200 9.12900 # H2264 + 2265 755 2 0.42380 6.74800 9.98600 8.39100 # H2265 + 2266 756 1 -0.84760 23.70200 16.38200 29.42700 # O2266 + 2267 756 2 0.42380 23.70300 15.86500 30.24000 # H2267 + 2268 756 2 0.42380 23.07000 17.10000 29.51000 # H2268 + 2269 757 1 -0.84760 7.27700 24.84000 27.49900 # O2269 + 2270 757 2 0.42380 6.86700 24.80300 26.56800 # H2270 + 2271 757 2 0.42380 7.77900 23.95200 27.60600 # H2271 + 2272 758 1 -0.84760 29.17300 6.54900 10.39700 # O2272 + 2273 758 2 0.42380 29.60000 7.08900 9.67700 # H2273 + 2274 758 2 0.42380 28.46600 7.27800 10.66900 # H2274 + 2275 759 1 -0.84760 27.49400 12.41100 9.87300 # O2275 + 2276 759 2 0.42380 27.65900 12.10800 10.86300 # H2276 + 2277 759 2 0.42380 27.58300 11.56500 9.30100 # H2277 + 2278 760 1 -0.84760 20.24700 25.57400 18.72900 # O2278 + 2279 760 2 0.42380 19.90600 26.05400 17.85700 # H2279 + 2280 760 2 0.42380 20.22100 24.58600 18.42600 # H2280 + 2281 761 1 -0.84760 16.78400 6.94700 12.26000 # O2281 + 2282 761 2 0.42380 16.08900 6.95400 11.51600 # H2282 + 2283 761 2 0.42380 16.34900 6.59700 13.07400 # H2283 + 2284 762 1 -0.84760 22.57000 25.63700 0.11600 # O2284 + 2285 762 2 0.42380 23.13600 26.11000 0.80500 # H2285 + 2286 762 2 0.42380 21.64300 25.56400 0.45100 # H2286 + 2287 763 1 -0.84760 9.65100 23.82600 5.82400 # O2287 + 2288 763 2 0.42380 9.02800 23.57900 5.10000 # H2288 + 2289 763 2 0.42380 10.50400 23.31300 5.75200 # H2289 + 2290 764 1 -0.84760 1.54200 8.65800 29.46900 # O2290 + 2291 764 2 0.42380 1.21000 7.76400 29.76200 # H2291 + 2292 764 2 0.42380 1.50600 8.84500 28.43500 # H2292 + 2293 765 1 -0.84760 13.12600 1.71300 20.73800 # O2293 + 2294 765 2 0.42380 12.95600 0.70400 20.78200 # H2294 + 2295 765 2 0.42380 12.24100 2.21000 20.64500 # H2295 + 2296 766 1 -0.84760 13.67400 3.09100 9.63500 # O2296 + 2297 766 2 0.42380 13.72300 2.14600 10.03500 # H2297 + 2298 766 2 0.42380 13.26900 3.72900 10.32700 # H2298 + 2299 767 1 -0.84760 19.04500 7.50400 10.40400 # O2299 + 2300 767 2 0.42380 18.37800 7.30500 11.10000 # H2300 + 2301 767 2 0.42380 18.60300 7.77400 9.55500 # H2301 + 2302 768 1 -0.84760 5.18700 9.60000 20.03400 # O2302 + 2303 768 2 0.42380 4.77200 8.74400 19.62400 # H2303 + 2304 768 2 0.42380 4.96100 10.40800 19.45600 # H2304 + 2305 769 1 -0.84760 29.84200 1.89000 12.29500 # O2305 + 2306 769 2 0.42380 29.66700 2.63300 12.96200 # H2306 + 2307 769 2 0.42380 29.95100 2.43200 11.41000 # H2307 + 2308 770 1 -0.84760 26.43400 5.46400 13.90200 # O2308 + 2309 770 2 0.42380 27.13100 4.83800 13.66200 # H2309 + 2310 770 2 0.42380 26.88700 6.36500 13.99100 # H2310 + 2311 771 1 -0.84760 10.16300 -0.07500 27.45300 # O2311 + 2312 771 2 0.42380 9.84400 -0.46600 26.60600 # H2312 + 2313 771 2 0.42380 11.17400 0.14800 27.24500 # H2313 + 2314 772 1 -0.84760 2.82500 18.44100 3.80600 # O2314 + 2315 772 2 0.42380 2.44700 18.35400 2.84900 # H2315 + 2316 772 2 0.42380 1.86400 18.53800 4.13500 # H2316 + 2317 773 1 -0.84760 29.69300 26.85700 2.59000 # O2317 + 2318 773 2 0.42380 29.55900 26.12000 1.91900 # H2318 + 2319 773 2 0.42380 30.68900 26.88200 2.85300 # H2319 + 2320 774 1 -0.84760 25.31800 8.52000 22.93200 # O2320 + 2321 774 2 0.42380 25.81900 9.40600 22.97700 # H2321 + 2322 774 2 0.42380 25.47100 7.88800 23.63800 # H2322 + 2323 775 1 -0.84760 16.39800 -0.97200 8.15100 # O2323 + 2324 775 2 0.42380 15.41400 -0.73300 8.31700 # H2324 + 2325 775 2 0.42380 16.51800 -0.67200 7.17500 # H2325 + 2326 776 1 -0.84760 15.08500 26.48200 11.27100 # O2326 + 2327 776 2 0.42380 15.36000 25.50900 11.09400 # H2327 + 2328 776 2 0.42380 15.77600 26.86200 11.87900 # H2328 + 2329 777 1 -0.84760 20.45900 2.14900 12.30600 # O2329 + 2330 777 2 0.42380 21.17600 2.84600 12.60300 # H2330 + 2331 777 2 0.42380 21.09000 1.39400 12.07500 # H2331 + 2332 778 1 -0.84760 27.47100 27.74100 17.37300 # O2332 + 2333 778 2 0.42380 27.41300 27.84200 16.33900 # H2333 + 2334 778 2 0.42380 26.48700 27.88200 17.63000 # H2334 + 2335 779 1 -0.84760 6.16500 24.19800 4.87200 # O2335 + 2336 779 2 0.42380 5.16600 24.21400 4.48000 # H2336 + 2337 779 2 0.42380 6.45000 23.35700 4.28000 # H2337 + 2338 780 1 -0.84760 10.41500 13.47400 20.92500 # O2338 + 2339 780 2 0.42380 10.18000 14.16800 20.21500 # H2339 + 2340 780 2 0.42380 9.58500 13.27200 21.44000 # H2340 + 2341 781 1 -0.84760 10.36500 1.76800 9.28700 # O2341 + 2342 781 2 0.42380 9.73100 1.82200 8.48800 # H2342 + 2343 781 2 0.42380 9.81200 1.22800 9.89500 # H2343 + 2344 782 1 -0.84760 24.92600 18.04400 4.82600 # O2344 + 2345 782 2 0.42380 25.90500 17.88100 4.93300 # H2345 + 2346 782 2 0.42380 24.46900 17.73600 5.66000 # H2346 + 2347 783 1 -0.84760 3.16800 0.32100 2.13300 # O2347 + 2348 783 2 0.42380 4.10800 0.51900 2.40000 # H2348 + 2349 783 2 0.42380 3.06600 0.24000 1.11500 # H2349 + 2350 784 1 -0.84760 20.54100 5.21300 10.22700 # O2350 + 2351 784 2 0.42380 19.61200 5.06000 10.45600 # H2351 + 2352 784 2 0.42380 20.64300 6.05400 9.62000 # H2352 + 2353 785 1 -0.84760 25.14800 0.38800 24.10600 # O2353 + 2354 785 2 0.42380 25.53200 1.24400 23.93900 # H2354 + 2355 785 2 0.42380 25.83200 -0.29900 24.34900 # H2355 + 2356 786 1 -0.84760 20.18900 8.65200 24.70400 # O2356 + 2357 786 2 0.42380 20.05300 9.08400 25.64700 # H2357 + 2358 786 2 0.42380 20.28800 7.69100 24.91400 # H2358 + 2359 787 1 -0.84760 24.38900 25.85000 3.93000 # O2359 + 2360 787 2 0.42380 24.47700 26.79400 3.56100 # H2360 + 2361 787 2 0.42380 24.27600 25.25700 3.07600 # H2361 + 2362 788 1 -0.84760 19.77700 24.87200 12.92000 # O2362 + 2363 788 2 0.42380 20.36600 24.77200 12.06800 # H2363 + 2364 788 2 0.42380 19.76800 25.84400 13.19100 # H2364 + 2365 789 1 -0.84760 15.63900 20.15800 28.13500 # O2365 + 2366 789 2 0.42380 16.39900 20.42400 28.77800 # H2366 + 2367 789 2 0.42380 15.73800 19.15400 28.01500 # H2367 + 2368 790 1 -0.84760 2.20700 -0.88500 7.84200 # O2368 + 2369 790 2 0.42380 2.11600 -1.65900 8.53000 # H2369 + 2370 790 2 0.42380 1.82500 -0.15900 8.43400 # H2370 + 2371 791 1 -0.84760 9.15600 0.87100 6.79700 # O2371 + 2372 791 2 0.42380 9.67400 0.14500 7.29100 # H2372 + 2373 791 2 0.42380 8.26500 0.99600 7.26200 # H2373 + 2374 792 1 -0.84760 27.60400 9.71600 30.20100 # O2374 + 2375 792 2 0.42380 26.72000 9.97400 29.82900 # H2375 + 2376 792 2 0.42380 28.17000 9.40900 29.38400 # H2376 + 2377 793 1 -0.84760 15.06700 13.85100 19.30500 # O2377 + 2378 793 2 0.42380 15.17200 12.81900 19.29400 # H2378 + 2379 793 2 0.42380 15.94800 14.27900 19.45300 # H2379 + 2380 794 1 -0.84760 12.00100 25.06200 3.34400 # O2380 + 2381 794 2 0.42380 12.45700 25.93300 3.38400 # H2381 + 2382 794 2 0.42380 11.31300 25.08600 3.98700 # H2382 + 2383 795 1 -0.84760 21.85000 25.84600 4.38800 # O2383 + 2384 795 2 0.42380 22.81600 25.80200 3.96600 # H2384 + 2385 795 2 0.42380 22.06700 25.76000 5.33800 # H2385 + 2386 796 1 -0.84760 14.84300 14.95700 16.24800 # O2386 + 2387 796 2 0.42380 14.25900 14.58400 16.91200 # H2387 + 2388 796 2 0.42380 15.27400 14.05200 16.00700 # H2388 + 2389 797 1 -0.84760 4.40700 8.69100 3.47700 # O2389 + 2390 797 2 0.42380 3.50300 9.12300 3.60800 # H2390 + 2391 797 2 0.42380 4.28600 7.85900 2.86600 # H2391 + 2392 798 1 -0.84760 9.76400 1.78200 16.65700 # O2392 + 2393 798 2 0.42380 10.45900 2.34000 17.19600 # H2393 + 2394 798 2 0.42380 9.57900 2.02600 15.70900 # H2394 + 2395 799 1 -0.84760 27.53400 20.92500 12.70800 # O2395 + 2396 799 2 0.42380 26.81200 20.60100 13.39700 # H2396 + 2397 799 2 0.42380 27.41800 20.39900 11.81700 # H2397 + 2398 800 1 -0.84760 0.49400 7.14100 16.48600 # O2398 + 2399 800 2 0.42380 0.89900 6.47500 15.81000 # H2399 + 2400 800 2 0.42380 -0.08300 7.83600 15.97000 # H2400 + 2401 801 1 -0.84760 20.83400 26.81600 10.17700 # O2401 + 2402 801 2 0.42380 20.43000 27.74900 10.48800 # H2402 + 2403 801 2 0.42380 21.56400 26.98200 9.51000 # H2403 + 2404 802 1 -0.84760 25.39300 18.85500 -0.45100 # O2404 + 2405 802 2 0.42380 24.97600 18.21400 0.20700 # H2405 + 2406 802 2 0.42380 24.93700 19.76300 -0.33600 # H2406 + 2407 803 1 -0.84760 9.55500 17.60200 9.10400 # O2407 + 2408 803 2 0.42380 9.37900 18.19200 8.28700 # H2408 + 2409 803 2 0.42380 8.66100 17.02300 9.10100 # H2409 + 2410 804 1 -0.84760 24.74800 27.92500 18.36700 # O2410 + 2411 804 2 0.42380 25.03200 27.46200 19.26100 # H2411 + 2412 804 2 0.42380 23.78700 27.58500 18.16300 # H2412 + 2413 805 1 -0.84760 18.21000 11.16600 24.43800 # O2413 + 2414 805 2 0.42380 19.00700 11.68400 24.78400 # H2414 + 2415 805 2 0.42380 18.62000 10.28100 24.19300 # H2415 + 2416 806 1 -0.84760 0.28700 12.14200 19.06800 # O2416 + 2417 806 2 0.42380 0.90000 11.60000 18.57500 # H2417 + 2418 806 2 0.42380 0.79000 12.31800 19.92900 # H2418 + 2419 807 1 -0.84760 18.90200 29.76000 26.00800 # O2419 + 2420 807 2 0.42380 19.67500 29.95200 25.39100 # H2420 + 2421 807 2 0.42380 17.98700 29.72400 25.49700 # H2421 + 2422 808 1 -0.84760 14.12600 24.98200 25.35100 # O2422 + 2423 808 2 0.42380 13.67800 24.28200 25.99600 # H2423 + 2424 808 2 0.42380 14.86100 25.45000 25.84100 # H2424 + 2425 809 1 -0.84760 22.65300 4.18800 11.66200 # O2425 + 2426 809 2 0.42380 23.49900 3.72000 11.29400 # H2426 + 2427 809 2 0.42380 22.22000 4.81600 10.98700 # H2427 + 2428 810 1 -0.84760 2.52300 19.46000 10.60100 # O2428 + 2429 810 2 0.42380 2.07400 20.33400 10.60800 # H2429 + 2430 810 2 0.42380 2.69100 19.12400 9.63700 # H2430 + 2431 811 1 -0.84760 20.65000 24.89500 7.68000 # O2431 + 2432 811 2 0.42380 20.46400 25.86000 7.35500 # H2432 + 2433 811 2 0.42380 20.27300 24.74800 8.65900 # H2433 + 2434 812 1 -0.84760 14.28500 12.49800 7.87100 # O2434 + 2435 812 2 0.42380 14.82600 11.68400 7.75300 # H2435 + 2436 812 2 0.42380 14.90200 13.33700 7.80100 # H2436 + 2437 813 1 -0.84760 2.22000 18.21700 26.46900 # O2437 + 2438 813 2 0.42380 2.06700 17.86000 27.44400 # H2438 + 2439 813 2 0.42380 1.68800 17.55800 25.80300 # H2439 + 2440 814 1 -0.84760 1.58300 0.32800 23.40600 # O2440 + 2441 814 2 0.42380 0.97500 -0.33000 22.94300 # H2441 + 2442 814 2 0.42380 1.75900 0.02600 24.34900 # H2442 + 2443 815 1 -0.84760 17.40100 9.56500 18.97400 # O2443 + 2444 815 2 0.42380 17.95200 9.42400 18.18600 # H2444 + 2445 815 2 0.42380 16.51700 9.93400 18.67700 # H2445 + 2446 816 1 -0.84760 27.14400 10.87300 7.47200 # O2446 + 2447 816 2 0.42380 27.69100 9.95800 7.56500 # H2447 + 2448 816 2 0.42380 27.31500 11.34200 6.54000 # H2448 + 2449 817 1 -0.84760 3.40000 5.57600 16.90500 # O2449 + 2450 817 2 0.42380 3.11500 6.10600 17.74900 # H2450 + 2451 817 2 0.42380 4.06400 6.16800 16.37100 # H2451 + 2452 818 1 -0.84760 1.75300 9.18800 24.26400 # O2452 + 2453 818 2 0.42380 1.91600 10.23200 24.22000 # H2453 + 2454 818 2 0.42380 2.53000 8.67300 23.80800 # H2454 + 2455 819 1 -0.84760 27.23900 27.46700 12.06200 # O2455 + 2456 819 2 0.42380 27.72400 28.33800 11.90300 # H2456 + 2457 819 2 0.42380 26.42800 27.30100 11.44900 # H2457 + 2458 820 1 -0.84760 14.85200 9.70400 6.13000 # O2458 + 2459 820 2 0.42380 14.67800 8.81200 6.64400 # H2459 + 2460 820 2 0.42380 14.59700 9.49700 5.19000 # H2460 + 2461 821 1 -0.84760 1.88400 12.18800 14.14100 # O2461 + 2462 821 2 0.42380 2.22100 11.37900 14.68200 # H2462 + 2463 821 2 0.42380 0.88300 12.15000 14.17400 # H2463 + 2464 822 1 -0.84760 12.28900 18.61900 -0.04600 # O2464 + 2465 822 2 0.42380 11.31400 18.62000 0.24000 # H2465 + 2466 822 2 0.42380 12.95000 18.13800 0.54600 # H2466 + 2467 823 1 -0.84760 10.60600 20.27600 17.14500 # O2467 + 2468 823 2 0.42380 11.58700 20.30700 17.38000 # H2468 + 2469 823 2 0.42380 10.29100 19.33900 17.33700 # H2469 + 2470 824 1 -0.84760 3.32700 5.02100 22.56700 # O2470 + 2471 824 2 0.42380 2.60100 5.38500 21.91900 # H2471 + 2472 824 2 0.42380 3.73800 4.45500 21.84900 # H2472 + 2473 825 1 -0.84760 6.03300 13.89100 5.74600 # O2473 + 2474 825 2 0.42380 6.43000 12.97900 5.94000 # H2474 + 2475 825 2 0.42380 6.71700 14.61100 5.68500 # H2475 + 2476 826 1 -0.84760 4.79400 6.98500 1.45900 # O2476 + 2477 826 2 0.42380 5.17300 6.04300 1.67200 # H2477 + 2478 826 2 0.42380 3.85400 6.91800 1.03300 # H2478 + 2479 827 1 -0.84760 17.50700 19.86600 15.28100 # O2479 + 2480 827 2 0.42380 18.42700 19.91100 15.68400 # H2480 + 2481 827 2 0.42380 17.27100 18.85600 15.27400 # H2481 + 2482 828 1 -0.84760 4.73400 22.71400 17.97900 # O2482 + 2483 828 2 0.42380 4.83700 21.77300 17.64900 # H2483 + 2484 828 2 0.42380 4.19400 22.56900 18.81300 # H2484 + 2485 829 1 -0.84760 10.66500 3.07600 22.45000 # O2485 + 2486 829 2 0.42380 11.63100 2.91100 22.34500 # H2486 + 2487 829 2 0.42380 10.30400 3.04900 23.44300 # H2487 + 2488 830 1 -0.84760 -0.16900 4.00100 -0.25500 # O2488 + 2489 830 2 0.42380 -0.83000 4.56000 -0.75800 # H2489 + 2490 830 2 0.42380 -0.12300 4.22500 0.72200 # H2490 + 2491 831 1 -0.84760 1.98500 6.69500 9.30800 # O2491 + 2492 831 2 0.42380 2.24600 7.60300 9.68200 # H2492 + 2493 831 2 0.42380 2.31300 6.07600 10.05700 # H2493 + 2494 832 1 -0.84760 21.51000 7.76000 7.87100 # O2494 + 2495 832 2 0.42380 21.81400 8.49300 8.57100 # H2495 + 2496 832 2 0.42380 21.23400 8.14800 6.92700 # H2496 + 2497 833 1 -0.84760 -0.47200 29.18200 4.02100 # O2497 + 2498 833 2 0.42380 0.18300 28.92900 3.28100 # H2498 + 2499 833 2 0.42380 -1.39700 29.43100 3.64500 # H2499 + 2500 834 1 -0.84760 25.51300 13.14900 7.82600 # O2500 + 2501 834 2 0.42380 24.63400 12.84200 8.14000 # H2501 + 2502 834 2 0.42380 26.03900 12.98500 8.60100 # H2502 + 2503 835 1 -0.84760 28.51500 3.16200 6.31200 # O2503 + 2504 835 2 0.42380 27.75800 3.33600 6.92400 # H2504 + 2505 835 2 0.42380 28.08800 2.68500 5.55500 # H2505 + 2506 836 1 -0.84760 5.98400 4.04300 -0.64900 # O2506 + 2507 836 2 0.42380 5.00100 4.17100 -0.29500 # H2507 + 2508 836 2 0.42380 6.07100 4.15500 -1.69900 # H2508 + 2509 837 1 -0.84760 3.72300 27.96800 17.04100 # O2509 + 2510 837 2 0.42380 4.64300 28.32900 16.90200 # H2510 + 2511 837 2 0.42380 3.13500 28.37200 16.26800 # H2511 + 2512 838 1 -0.84760 22.92000 25.10600 18.92300 # O2512 + 2513 838 2 0.42380 22.80100 24.33800 18.27100 # H2513 + 2514 838 2 0.42380 21.95300 25.29700 19.09700 # H2514 + 2515 839 1 -0.84760 15.63900 -1.00500 1.26800 # O2515 + 2516 839 2 0.42380 16.26800 -0.24500 1.29200 # H2516 + 2517 839 2 0.42380 15.81800 -1.40100 0.34000 # H2517 + 2518 840 1 -0.84760 3.41000 28.27500 10.60800 # O2518 + 2519 840 2 0.42380 3.61800 28.16800 11.61100 # H2519 + 2520 840 2 0.42380 3.24800 27.32400 10.37400 # H2520 + 2521 841 1 -0.84760 5.11300 8.92200 12.89300 # O2521 + 2522 841 2 0.42380 4.75500 8.96000 13.80400 # H2522 + 2523 841 2 0.42380 4.98000 9.83300 12.52200 # H2523 + 2524 842 1 -0.84760 17.80000 26.20000 23.17700 # O2524 + 2525 842 2 0.42380 17.87700 26.39800 22.18700 # H2525 + 2526 842 2 0.42380 18.21400 27.06300 23.43600 # H2526 + 2527 843 1 -0.84760 6.21300 8.56200 26.79900 # O2527 + 2528 843 2 0.42380 6.06600 8.63300 27.78400 # H2528 + 2529 843 2 0.42380 5.26000 8.43400 26.29500 # H2529 + 2530 844 1 -0.84760 10.78900 3.22500 25.75200 # O2530 + 2531 844 2 0.42380 11.22000 3.93500 25.20000 # H2531 + 2532 844 2 0.42380 9.99300 3.61300 26.12300 # H2532 + 2533 845 1 -0.84760 9.54500 3.50500 6.39300 # O2533 + 2534 845 2 0.42380 9.79000 3.38500 5.40800 # H2534 + 2535 845 2 0.42380 9.69400 2.56000 6.75900 # H2535 + 2536 846 1 -0.84760 25.36500 25.81600 24.73400 # O2536 + 2537 846 2 0.42380 25.47000 25.54300 25.68700 # H2537 + 2538 846 2 0.42380 25.00400 26.78100 24.70000 # H2538 + 2539 847 1 -0.84760 15.62000 14.36600 6.84800 # O2539 + 2540 847 2 0.42380 16.31600 14.18000 7.59400 # H2540 + 2541 847 2 0.42380 16.04900 15.11700 6.28000 # H2541 + 2542 848 1 -0.84760 12.03600 8.19300 9.68400 # O2542 + 2543 848 2 0.42380 13.00200 8.08500 10.03700 # H2543 + 2544 848 2 0.42380 11.89500 8.16800 8.64800 # H2544 + 2545 849 1 -0.84760 12.69400 27.95400 28.16300 # O2545 + 2546 849 2 0.42380 12.35100 28.93300 28.27800 # H2546 + 2547 849 2 0.42380 11.87400 27.41200 28.42500 # H2547 + 2548 850 1 -0.84760 26.47400 6.65400 6.01500 # O2548 + 2549 850 2 0.42380 26.60600 5.64900 6.19700 # H2549 + 2550 850 2 0.42380 25.51600 6.84100 6.23300 # H2550 + 2551 851 1 -0.84760 18.17300 26.20800 16.73700 # O2551 + 2552 851 2 0.42380 17.62200 25.38800 16.89700 # H2552 + 2553 851 2 0.42380 18.63500 26.09700 15.80200 # H2553 + 2554 852 1 -0.84760 6.09600 12.52200 1.04600 # O2554 + 2555 852 2 0.42380 6.50000 11.69700 1.35400 # H2555 + 2556 852 2 0.42380 5.94100 13.10100 1.91200 # H2556 + 2557 853 1 -0.84760 19.10200 4.96500 1.46300 # O2557 + 2558 853 2 0.42380 18.28400 5.50700 1.72400 # H2558 + 2559 853 2 0.42380 18.91300 4.55100 0.58000 # H2559 + 2560 854 1 -0.84760 20.90300 6.99500 27.14800 # O2560 + 2561 854 2 0.42380 21.07500 6.09400 26.66800 # H2561 + 2562 854 2 0.42380 20.32400 6.69200 28.00300 # H2562 + 2563 855 1 -0.84760 25.48600 20.41100 2.72700 # O2563 + 2564 855 2 0.42380 26.50200 20.18400 2.47600 # H2564 + 2565 855 2 0.42380 25.42600 20.35100 3.72600 # H2565 + 2566 856 1 -0.84760 12.07200 19.23600 6.61000 # O2566 + 2567 856 2 0.42380 11.07500 19.22400 6.80400 # H2567 + 2568 856 2 0.42380 12.14400 19.22200 5.61600 # H2568 + 2569 857 1 -0.84760 11.35200 0.35700 19.15000 # O2569 + 2570 857 2 0.42380 10.95500 0.00800 20.07700 # H2570 + 2571 857 2 0.42380 10.78100 0.27400 18.22500 # H2571 + 2572 858 1 -0.84760 16.95800 22.96900 29.27200 # O2572 + 2573 858 2 0.42380 17.87100 23.37300 29.06500 # H2573 + 2574 858 2 0.42380 16.59600 22.51700 28.45600 # H2574 + 2575 859 1 -0.84760 18.03600 -0.71100 2.79300 # O2575 + 2576 859 2 0.42380 18.03900 -1.60800 2.27500 # H2576 + 2577 859 2 0.42380 18.81300 -0.19500 2.41000 # H2577 + 2578 860 1 -0.84760 11.18400 23.20400 8.36000 # O2578 + 2579 860 2 0.42380 11.98200 23.63300 7.92400 # H2579 + 2580 860 2 0.42380 10.38700 23.46600 7.82400 # H2580 + 2581 861 1 -0.84760 22.50600 21.38000 22.94900 # O2581 + 2582 861 2 0.42380 21.78300 21.77100 22.49500 # H2582 + 2583 861 2 0.42380 22.02900 20.54800 23.39100 # H2583 + 2584 862 1 -0.84760 22.88900 8.41600 5.38400 # O2584 + 2585 862 2 0.42380 22.50600 8.68500 4.43600 # H2585 + 2586 862 2 0.42380 23.24100 9.28900 5.76600 # H2586 + 2587 863 1 -0.84760 12.09400 22.95000 15.20900 # O2587 + 2588 863 2 0.42380 12.24700 22.56200 16.13000 # H2588 + 2589 863 2 0.42380 11.95000 23.91400 15.45100 # H2589 + 2590 864 1 -0.84760 11.93300 16.47400 9.09000 # O2590 + 2591 864 2 0.42380 11.12300 17.10000 9.17100 # H2591 + 2592 864 2 0.42380 12.40200 16.64900 8.15000 # H2592 + 2593 865 1 -0.84760 24.32000 16.40000 8.04600 # O2593 + 2594 865 2 0.42380 23.88200 16.80400 8.85200 # H2594 + 2595 865 2 0.42380 23.54800 15.76700 7.72900 # H2595 + 2596 866 1 -0.84760 3.74400 1.04900 21.57800 # O2596 + 2597 866 2 0.42380 4.14500 1.77800 22.19700 # H2597 + 2598 866 2 0.42380 3.09400 0.52300 22.09200 # H2598 + 2599 867 1 -0.84760 13.89600 -0.22100 28.49400 # O2599 + 2600 867 2 0.42380 13.78900 0.67900 28.06000 # H2600 + 2601 867 2 0.42380 14.61800 -0.82600 28.11500 # H2601 + 2602 868 1 -0.84760 11.65000 26.80100 0.90800 # O2602 + 2603 868 2 0.42380 11.18200 26.82200 1.81100 # H2603 + 2604 868 2 0.42380 11.51200 25.84700 0.50500 # H2604 + 2605 869 1 -0.84760 24.89600 7.87100 0.44100 # O2605 + 2606 869 2 0.42380 23.99000 7.70900 0.91300 # H2606 + 2607 869 2 0.42380 25.13000 7.06600 -0.17400 # H2607 + 2608 870 1 -0.84760 23.05200 14.96800 25.49500 # O2608 + 2609 870 2 0.42380 22.91400 14.20300 26.16100 # H2609 + 2610 870 2 0.42380 23.99300 15.03700 25.31100 # H2610 + 2611 871 1 -0.84760 4.19800 15.95000 1.05600 # O2611 + 2612 871 2 0.42380 4.70800 15.28200 1.64600 # H2612 + 2613 871 2 0.42380 3.53500 15.34300 0.58400 # H2613 + 2614 872 1 -0.84760 9.05600 16.78100 13.10400 # O2614 + 2615 872 2 0.42380 9.39400 17.23100 12.26800 # H2615 + 2616 872 2 0.42380 9.86800 17.04900 13.70500 # H2616 + 2617 873 1 -0.84760 17.81400 25.77000 28.22200 # O2617 + 2618 873 2 0.42380 17.34100 26.53600 27.73100 # H2618 + 2619 873 2 0.42380 17.75900 24.82900 27.79000 # H2619 + 2620 874 1 -0.84760 15.21000 5.91000 22.77600 # O2620 + 2621 874 2 0.42380 15.64100 6.63400 22.19400 # H2621 + 2622 874 2 0.42380 14.21000 6.17900 22.91700 # H2622 + 2623 875 1 -0.84760 19.26100 19.74100 4.14100 # O2623 + 2624 875 2 0.42380 19.89100 19.60600 3.29500 # H2624 + 2625 875 2 0.42380 19.86800 19.79800 4.95700 # H2625 + 2626 876 1 -0.84760 8.04800 15.53500 19.72100 # O2626 + 2627 876 2 0.42380 7.49800 16.39800 19.50200 # H2627 + 2628 876 2 0.42380 8.53300 15.73700 20.56600 # H2628 + 2629 877 1 -0.84760 24.69700 1.96400 21.46200 # O2629 + 2630 877 2 0.42380 24.04800 2.41300 20.79000 # H2630 + 2631 877 2 0.42380 24.85500 1.04900 21.00200 # H2631 + 2632 878 1 -0.84760 8.65600 1.72900 12.29000 # O2632 + 2633 878 2 0.42380 8.27500 1.26800 13.11800 # H2633 + 2634 878 2 0.42380 7.94600 1.83900 11.63900 # H2634 + 2635 879 1 -0.84760 21.85100 5.92100 17.32100 # O2635 + 2636 879 2 0.42380 22.08000 5.93600 16.33600 # H2636 + 2637 879 2 0.42380 21.08500 6.55900 17.26400 # H2637 + 2638 880 1 -0.84760 9.30600 26.60000 7.99000 # O2638 + 2639 880 2 0.42380 9.05800 27.58500 7.65900 # H2639 + 2640 880 2 0.42380 10.29000 26.44900 8.25400 # H2640 + 2641 881 1 -0.84760 13.64500 16.80500 6.93000 # O2641 + 2642 881 2 0.42380 13.98700 17.76400 7.22700 # H2642 + 2643 881 2 0.42380 14.32500 16.03200 7.15900 # H2643 + 2644 882 1 -0.84760 10.23500 7.60200 7.00900 # O2644 + 2645 882 2 0.42380 10.71100 6.76500 6.65200 # H2645 + 2646 882 2 0.42380 9.25700 7.30500 6.90300 # H2646 + 2647 883 1 -0.84760 23.56100 4.97000 20.27300 # O2647 + 2648 883 2 0.42380 22.67400 4.73700 19.89900 # H2648 + 2649 883 2 0.42380 24.01700 5.29800 19.44100 # H2649 + 2650 884 1 -0.84760 22.35800 5.57100 6.01000 # O2650 + 2651 884 2 0.42380 22.53900 6.47800 5.77500 # H2651 + 2652 884 2 0.42380 21.44400 5.60600 6.35700 # H2652 + 2653 885 1 -0.84760 29.29000 6.01200 4.97700 # O2653 + 2654 885 2 0.42380 28.85200 6.17600 5.93700 # H2654 + 2655 885 2 0.42380 29.68200 5.08100 4.95700 # H2655 + 2656 886 1 -0.84760 25.10800 5.82700 25.52100 # O2656 + 2657 886 2 0.42380 24.49400 5.36500 24.81800 # H2657 + 2658 886 2 0.42380 25.76000 5.11900 25.73200 # H2658 + 2659 887 1 -0.84760 20.22900 16.56800 0.25300 # O2659 + 2660 887 2 0.42380 20.00900 17.32700 -0.49600 # H2660 + 2661 887 2 0.42380 20.19100 15.69000 -0.30900 # H2661 + 2662 888 1 -0.84760 1.28100 27.85200 15.99800 # O2662 + 2663 888 2 0.42380 1.65000 27.58800 16.88800 # H2663 + 2664 888 2 0.42380 0.51600 28.42100 16.29100 # H2664 + 2665 889 1 -0.84760 26.77200 6.23000 21.72000 # O2665 + 2666 889 2 0.42380 26.72400 6.31700 20.72000 # H2666 + 2667 889 2 0.42380 25.85100 6.69500 21.97600 # H2667 + 2668 890 1 -0.84760 1.84000 22.39600 7.26800 # O2668 + 2669 890 2 0.42380 1.21700 23.09700 6.87000 # H2669 + 2670 890 2 0.42380 1.62900 21.44200 7.12700 # H2670 + 2671 891 1 -0.84760 4.09500 14.72700 15.45600 # O2671 + 2672 891 2 0.42380 4.88700 14.41100 14.82800 # H2672 + 2673 891 2 0.42380 4.19700 15.77900 15.26700 # H2673 + 2674 892 1 -0.84760 24.47900 10.89300 4.96400 # O2674 + 2675 892 2 0.42380 25.15000 11.55300 4.58100 # H2675 + 2676 892 2 0.42380 25.03900 10.47700 5.69700 # H2676 + 2677 893 1 -0.84760 1.72200 0.25800 30.03500 # O2677 + 2678 893 2 0.42380 0.99000 0.60000 29.36200 # H2678 + 2679 893 2 0.42380 1.80100 1.14500 30.50800 # H2679 + 2680 894 1 -0.84760 25.46700 29.76400 15.08200 # O2680 + 2681 894 2 0.42380 26.03700 30.35300 15.69800 # H2681 + 2682 894 2 0.42380 26.01500 29.72400 14.24600 # H2682 + 2683 895 1 -0.84760 8.64100 4.12700 2.77800 # O2683 + 2684 895 2 0.42380 8.49900 4.79100 1.93200 # H2684 + 2685 895 2 0.42380 8.98400 3.24900 2.39300 # H2685 + 2686 896 1 -0.84760 2.50200 11.78600 0.48900 # O2686 + 2687 896 2 0.42380 2.27200 12.37400 1.30100 # H2687 + 2688 896 2 0.42380 3.52200 11.79200 0.49000 # H2688 + 2689 897 1 -0.84760 3.41100 2.92600 24.99700 # O2689 + 2690 897 2 0.42380 4.29000 2.83200 25.37400 # H2690 + 2691 897 2 0.42380 3.32100 3.78400 24.54100 # H2691 + 2692 898 1 -0.84760 12.77600 28.10100 20.25000 # O2692 + 2693 898 2 0.42380 11.92800 27.98500 19.74800 # H2693 + 2694 898 2 0.42380 13.25100 28.65200 19.57200 # H2694 + 2695 899 1 -0.84760 11.10000 15.30700 11.55700 # O2695 + 2696 899 2 0.42380 10.89100 15.59000 10.55400 # H2696 + 2697 899 2 0.42380 11.97700 14.81600 11.39300 # H2697 + 2698 900 1 -0.84760 3.99600 21.61700 26.08100 # O2698 + 2699 900 2 0.42380 3.57700 21.70500 25.12900 # H2699 + 2700 900 2 0.42380 4.73100 20.86700 26.06400 # H2700 + 2701 901 1 -0.84760 10.98700 24.58400 -0.64200 # O2701 + 2702 901 2 0.42380 11.10200 24.39200 -1.64800 # H2702 + 2703 901 2 0.42380 11.43700 23.92200 -0.07500 # H2703 + 2704 902 1 -0.84760 26.78100 -0.21900 18.90300 # O2704 + 2705 902 2 0.42380 27.51400 -0.50400 18.31300 # H2705 + 2706 902 2 0.42380 26.08500 0.11300 18.30400 # H2706 + 2707 903 1 -0.84760 13.76100 3.04100 1.00800 # O2707 + 2708 903 2 0.42380 13.61800 3.26800 0.04300 # H2708 + 2709 903 2 0.42380 13.14500 3.67200 1.58600 # H2709 + 2710 904 1 -0.84760 26.55900 5.01200 10.12100 # O2710 + 2711 904 2 0.42380 26.30300 5.65200 10.82100 # H2711 + 2712 904 2 0.42380 25.91900 5.05500 9.34000 # H2712 + 2713 905 1 -0.84760 22.06800 11.56800 28.50100 # O2713 + 2714 905 2 0.42380 22.69900 11.44000 29.30000 # H2714 + 2715 905 2 0.42380 21.25200 12.16700 28.74000 # H2715 + 2716 906 1 -0.84760 10.74900 5.90700 13.73800 # O2716 + 2717 906 2 0.42380 10.25300 5.03600 13.40500 # H2717 + 2718 906 2 0.42380 10.16600 6.69600 13.53000 # H2718 + 2719 907 1 -0.84760 12.24200 15.66600 19.62300 # O2719 + 2720 907 2 0.42380 12.89500 16.24400 20.09800 # H2720 + 2721 907 2 0.42380 12.60500 14.73700 19.66000 # H2721 + 2722 908 1 -0.84760 3.29100 21.55100 4.79300 # O2722 + 2723 908 2 0.42380 2.27600 21.50000 4.63700 # H2723 + 2724 908 2 0.42380 3.51300 22.31200 4.14900 # H2724 + 2725 909 1 -0.84760 25.00600 19.42800 22.35500 # O2725 + 2726 909 2 0.42380 25.30200 19.83600 23.18900 # H2726 + 2727 909 2 0.42380 24.28700 18.82100 22.80200 # H2727 + 2728 910 1 -0.84760 11.86400 25.83300 15.79200 # O2728 + 2729 910 2 0.42380 11.51900 26.73100 15.45800 # H2729 + 2730 910 2 0.42380 10.99800 25.32500 15.81300 # H2730 + 2731 911 1 -0.84760 24.45600 7.93800 15.09300 # O2731 + 2732 911 2 0.42380 24.13300 8.79500 14.77400 # H2732 + 2733 911 2 0.42380 25.34700 8.13100 15.55700 # H2733 + 2734 912 1 -0.84760 8.47500 22.53100 29.92000 # O2734 + 2735 912 2 0.42380 7.77100 22.07500 29.41100 # H2735 + 2736 912 2 0.42380 8.44900 22.30300 30.87900 # H2736 + 2737 913 1 -0.84760 22.29300 2.94200 14.79900 # O2737 + 2738 913 2 0.42380 23.07600 3.52700 15.14900 # H2738 + 2739 913 2 0.42380 22.60800 2.09800 14.35400 # H2739 + 2740 914 1 -0.84760 21.35200 5.40100 2.87400 # O2740 + 2741 914 2 0.42380 22.20600 4.94000 2.58800 # H2741 + 2742 914 2 0.42380 20.63300 5.16500 2.21800 # H2742 + 2743 915 1 -0.84760 12.65300 7.73400 23.02000 # O2743 + 2744 915 2 0.42380 12.19100 8.40000 22.38300 # H2744 + 2745 915 2 0.42380 12.39200 8.02700 23.97000 # H2745 + 2746 916 1 -0.84760 -0.66500 21.80500 21.46200 # O2746 + 2747 916 2 0.42380 -1.10200 22.67900 21.07800 # H2747 + 2748 916 2 0.42380 -1.19000 20.99900 21.19000 # H2748 + 2749 917 1 -0.84760 5.73400 5.98600 20.23700 # O2749 + 2750 917 2 0.42380 5.70900 6.88000 20.63400 # H2750 + 2751 917 2 0.42380 6.69000 5.76400 20.57400 # H2751 + 2752 918 1 -0.84760 15.59400 30.01200 11.74600 # O2752 + 2753 918 2 0.42380 15.78000 30.97900 11.49300 # H2753 + 2754 918 2 0.42380 15.90000 29.44300 10.93800 # H2754 + 2755 919 1 -0.84760 22.53800 19.33000 4.21800 # O2755 + 2756 919 2 0.42380 22.12100 19.18100 5.12300 # H2756 + 2757 919 2 0.42380 23.50600 19.10200 4.32400 # H2757 + 2758 920 1 -0.84760 9.56400 25.74000 17.31700 # O2758 + 2759 920 2 0.42380 8.65500 25.27600 17.30500 # H2759 + 2760 920 2 0.42380 10.14900 25.23900 18.02200 # H2760 + 2761 921 1 -0.84760 14.28400 10.92800 27.99100 # O2761 + 2762 921 2 0.42380 13.76500 11.60700 28.49000 # H2762 + 2763 921 2 0.42380 15.24200 10.84100 28.24500 # H2763 + 2764 922 1 -0.84760 9.34600 15.22700 15.34500 # O2764 + 2765 922 2 0.42380 9.19100 15.95800 16.01200 # H2765 + 2766 922 2 0.42380 9.40500 15.73200 14.48100 # H2766 + 2767 923 1 -0.84760 16.63400 29.65300 15.00300 # O2767 + 2768 923 2 0.42380 16.63300 30.66600 15.05700 # H2768 + 2769 923 2 0.42380 17.52000 29.34800 15.36800 # H2769 + 2770 924 1 -0.84760 2.49000 11.86300 10.53200 # O2770 + 2771 924 2 0.42380 2.85800 12.68100 11.17300 # H2771 + 2772 924 2 0.42380 2.50300 11.14700 11.27800 # H2772 + 2773 925 1 -0.84760 1.71400 20.68000 20.46600 # O2773 + 2774 925 2 0.42380 0.92600 21.18300 20.72000 # H2774 + 2775 925 2 0.42380 2.35900 21.12000 21.05300 # H2775 + 2776 926 1 -0.84760 10.98300 7.57400 -0.71500 # O2776 + 2777 926 2 0.42380 11.77700 7.35000 -1.20400 # H2777 + 2778 926 2 0.42380 10.53000 8.33000 -1.23100 # H2778 + 2779 927 1 -0.84760 13.20700 27.26500 4.50000 # O2779 + 2780 927 2 0.42380 13.93500 26.65300 4.78400 # H2780 + 2781 927 2 0.42380 13.55600 28.04400 3.89100 # H2781 + 2782 928 1 -0.84760 -0.46800 11.41300 11.34800 # O2782 + 2783 928 2 0.42380 0.29400 11.96100 10.94600 # H2783 + 2784 928 2 0.42380 -1.25300 11.11600 10.80700 # H2784 + 2785 929 1 -0.84760 15.77800 19.10800 11.50500 # O2785 + 2786 929 2 0.42380 15.80200 20.07200 11.08600 # H2786 + 2787 929 2 0.42380 15.25400 18.50500 10.83500 # H2787 + 2788 930 1 -0.84760 -0.35200 2.12700 2.31100 # O2788 + 2789 930 2 0.42380 -0.23500 1.67700 1.42300 # H2789 + 2790 930 2 0.42380 0.21100 2.95400 2.28200 # H2790 + 2791 931 1 -0.84760 22.05700 3.67900 8.65300 # O2791 + 2792 931 2 0.42380 22.20400 3.81800 7.65000 # H2792 + 2793 931 2 0.42380 21.38600 4.30700 9.15700 # H2793 + 2794 932 1 -0.84760 20.63000 18.22700 6.35000 # O2794 + 2795 932 2 0.42380 20.72300 17.56000 5.60500 # H2795 + 2796 932 2 0.42380 21.59700 18.38300 6.57900 # H2796 + 2797 933 1 -0.84760 10.33600 14.73200 29.18100 # O2797 + 2798 933 2 0.42380 11.21500 15.22900 28.92900 # H2798 + 2799 933 2 0.42380 9.70000 15.50300 28.86600 # H2799 + 2800 934 1 -0.84760 9.00000 16.90300 22.63500 # O2800 + 2801 934 2 0.42380 9.63800 16.85800 23.56400 # H2801 + 2802 934 2 0.42380 9.35700 17.71900 22.10000 # H2802 + 2803 935 1 -0.84760 18.72600 6.63300 17.16400 # O2803 + 2804 935 2 0.42380 17.72400 6.36500 17.23700 # H2804 + 2805 935 2 0.42380 18.76000 7.63400 16.96200 # H2805 + 2806 936 1 -0.84760 6.15100 19.41500 13.49200 # O2806 + 2807 936 2 0.42380 6.97600 19.09700 13.10800 # H2807 + 2808 936 2 0.42380 5.56400 19.60900 12.65100 # H2808 + 2809 937 1 -0.84760 5.17700 20.76200 16.11500 # O2809 + 2810 937 2 0.42380 5.98100 20.79700 15.46000 # H2810 + 2811 937 2 0.42380 4.21100 20.80200 15.58700 # H2811 + 2812 938 1 -0.84760 -0.80000 21.79300 -0.70400 # O2812 + 2813 938 2 0.42380 -1.82200 21.88600 -0.76200 # H2813 + 2814 938 2 0.42380 -0.52100 21.16000 -1.43000 # H2814 + 2815 939 1 -0.84760 24.75000 4.40000 5.35500 # O2815 + 2816 939 2 0.42380 25.42300 4.24100 4.62900 # H2816 + 2817 939 2 0.42380 24.33000 3.50200 5.56200 # H2817 + 2818 940 1 -0.84760 26.38700 10.98700 1.37700 # O2818 + 2819 940 2 0.42380 27.02400 10.83700 2.14100 # H2819 + 2820 940 2 0.42380 26.93000 11.40200 0.57500 # H2820 + 2821 941 1 -0.84760 19.70000 14.87400 6.04800 # O2821 + 2822 941 2 0.42380 19.49700 14.39500 6.90200 # H2822 + 2823 941 2 0.42380 18.85200 14.73300 5.51900 # H2823 + 2824 942 1 -0.84760 0.67000 19.62200 14.19600 # O2824 + 2825 942 2 0.42380 0.16700 19.21600 14.98100 # H2825 + 2826 942 2 0.42380 -0.04400 20.33000 14.02300 # H2826 + 2827 943 1 -0.84760 4.85500 -0.97200 29.97500 # O2827 + 2828 943 2 0.42380 4.13800 -0.34800 29.74500 # H2828 + 2829 943 2 0.42380 4.59500 -1.27000 30.90600 # H2829 + 2830 944 1 -0.84760 11.55400 -0.78500 4.88300 # O2830 + 2831 944 2 0.42380 12.20500 -1.56800 5.10300 # H2831 + 2832 944 2 0.42380 12.04700 -0.13900 4.28200 # H2832 + 2833 945 1 -0.84760 10.68200 20.44500 9.63900 # O2833 + 2834 945 2 0.42380 10.96500 20.47600 8.67100 # H2834 + 2835 945 2 0.42380 10.09900 21.33000 9.66700 # H2835 + 2836 946 1 -0.84760 9.43500 28.83400 12.17100 # O2836 + 2837 946 2 0.42380 10.23300 28.48000 11.62600 # H2837 + 2838 946 2 0.42380 9.70700 29.71600 12.68300 # H2838 + 2839 947 1 -0.84760 24.42900 15.38000 15.37300 # O2839 + 2840 947 2 0.42380 24.73800 16.32600 15.59700 # H2840 + 2841 947 2 0.42380 23.52700 15.54000 14.96200 # H2841 + 2842 948 1 -0.84760 28.53700 3.45800 10.49200 # O2842 + 2843 948 2 0.42380 27.97200 2.81900 9.98300 # H2843 + 2844 948 2 0.42380 28.10700 4.40200 10.39900 # H2844 + 2845 949 1 -0.84760 23.18600 26.69300 21.30600 # O2845 + 2846 949 2 0.42380 22.69800 25.99500 20.76100 # H2846 + 2847 949 2 0.42380 22.67300 26.92500 22.19500 # H2847 + 2848 950 1 -0.84760 29.35700 5.29100 22.99300 # O2848 + 2849 950 2 0.42380 28.53700 5.31300 22.33100 # H2849 + 2850 950 2 0.42380 30.05900 4.53300 22.62400 # H2850 + 2851 951 1 -0.84760 3.26000 21.15000 14.57200 # O2851 + 2852 951 2 0.42380 3.40500 21.94100 13.87800 # H2852 + 2853 951 2 0.42380 2.32200 20.78800 14.44200 # H2853 + 2854 952 1 -0.84760 28.70500 14.28000 13.80700 # O2854 + 2855 952 2 0.42380 29.43900 13.68300 14.21800 # H2855 + 2856 952 2 0.42380 27.90500 14.15300 14.48200 # H2856 + 2857 953 1 -0.84760 3.74200 28.83400 13.44000 # O2857 + 2858 953 2 0.42380 3.46400 29.57900 14.08200 # H2858 + 2859 953 2 0.42380 4.73500 28.72500 13.61800 # H2859 + 2860 954 1 -0.84760 19.87600 20.34500 12.64100 # O2860 + 2861 954 2 0.42380 18.83200 20.11900 12.67200 # H2861 + 2862 954 2 0.42380 19.86900 20.91200 11.81300 # H2862 + 2863 955 1 -0.84760 2.81200 16.09900 24.66600 # O2863 + 2864 955 2 0.42380 3.80800 16.18900 24.40500 # H2864 + 2865 955 2 0.42380 2.31700 15.22100 24.52100 # H2865 + 2866 956 1 -0.84760 22.68500 25.32700 26.75400 # O2866 + 2867 956 2 0.42380 21.85800 25.65800 27.12300 # H2867 + 2868 956 2 0.42380 22.68700 25.72400 25.80900 # H2868 + 2869 957 1 -0.84760 17.33800 18.81900 8.73000 # O2869 + 2870 957 2 0.42380 17.73800 19.57000 8.27200 # H2870 + 2871 957 2 0.42380 16.54400 19.16600 9.08000 # H2871 + 2872 958 1 -0.84760 10.48700 27.75000 14.43100 # O2872 + 2873 958 2 0.42380 10.60400 27.95800 13.42400 # H2873 + 2874 958 2 0.42380 9.52300 27.92700 14.58000 # H2874 + 2875 959 1 -0.84760 28.15200 11.71000 12.35700 # O2875 + 2876 959 2 0.42380 28.31400 12.37800 13.17700 # H2876 + 2877 959 2 0.42380 29.07200 11.76800 11.94400 # H2877 + 2878 960 1 -0.84760 0.81100 14.61800 27.22200 # O2878 + 2879 960 2 0.42380 1.55400 14.19000 26.70400 # H2879 + 2880 960 2 0.42380 0.28100 15.11800 26.54500 # H2880 + 2881 961 1 -0.84760 17.70900 6.95200 3.29300 # O2881 + 2882 961 2 0.42380 16.87100 6.55000 2.84100 # H2882 + 2883 961 2 0.42380 17.50200 7.94400 3.11300 # H2883 + 2884 962 1 -0.84760 4.26200 -0.02300 14.65800 # O2884 + 2885 962 2 0.42380 4.57300 0.57700 15.33200 # H2885 + 2886 962 2 0.42380 3.76700 0.51300 13.90800 # H2886 + 2887 963 1 -0.84760 11.58300 18.56700 3.53200 # O2887 + 2888 963 2 0.42380 11.24100 18.02700 2.77300 # H2888 + 2889 963 2 0.42380 12.54700 18.36500 3.77400 # H2889 + 2890 964 1 -0.84760 0.28400 27.96600 9.95200 # O2890 + 2891 964 2 0.42380 -0.61700 27.77600 10.28200 # H2891 + 2892 964 2 0.42380 0.90600 27.92100 10.73700 # H2892 + 2893 965 1 -0.84760 2.86300 14.32900 3.84300 # O2893 + 2894 965 2 0.42380 2.06700 14.55900 3.34300 # H2894 + 2895 965 2 0.42380 2.47100 14.46900 4.81000 # H2895 + 2896 966 1 -0.84760 5.81300 25.66500 7.58200 # O2896 + 2897 966 2 0.42380 6.54500 26.29500 7.76000 # H2897 + 2898 966 2 0.42380 5.40200 25.95500 6.72600 # H2898 + 2899 967 1 -0.84760 27.61900 18.33900 26.23400 # O2899 + 2900 967 2 0.42380 27.83600 19.09700 26.83800 # H2900 + 2901 967 2 0.42380 27.79900 17.50200 26.76300 # H2901 + 2902 968 1 -0.84760 28.15500 21.08800 10.10000 # O2902 + 2903 968 2 0.42380 27.30900 20.54500 10.01000 # H2903 + 2904 968 2 0.42380 28.66500 20.74000 10.90400 # H2904 + 2905 969 1 -0.84760 14.10600 7.19100 29.21700 # O2905 + 2906 969 2 0.42380 13.84100 7.15400 30.13100 # H2906 + 2907 969 2 0.42380 14.49600 6.26300 29.09300 # H2907 + 2908 970 1 -0.84760 22.89900 28.88500 8.08900 # O2908 + 2909 970 2 0.42380 22.57700 29.91800 8.11200 # H2909 + 2910 970 2 0.42380 22.40100 28.22300 7.48300 # H2910 + 2911 971 1 -0.84760 24.28100 7.57300 19.86400 # O2911 + 2912 971 2 0.42380 24.34900 8.63500 19.85500 # H2912 + 2913 971 2 0.42380 23.36000 7.31300 20.28200 # H2913 + 2914 972 1 -0.84760 14.69000 14.93000 0.56800 # O2914 + 2915 972 2 0.42380 15.34700 15.32300 1.23300 # H2915 + 2916 972 2 0.42380 13.79800 15.01700 1.02400 # H2916 + 2917 973 1 -0.84760 21.92500 20.22800 19.85100 # O2917 + 2918 973 2 0.42380 21.29300 20.93200 20.09200 # H2918 + 2919 973 2 0.42380 22.46800 20.18300 20.68300 # H2919 + 2920 974 1 -0.84760 9.36100 4.79600 19.61900 # O2920 + 2921 974 2 0.42380 8.99500 5.22200 18.77600 # H2921 + 2922 974 2 0.42380 10.19300 5.32400 19.45300 # H2922 + 2923 975 1 -0.84760 27.01500 7.03400 2.22100 # O2923 + 2924 975 2 0.42380 28.04900 7.10900 2.11500 # H2924 + 2925 975 2 0.42380 26.73100 7.87400 1.69400 # H2925 + 2926 976 1 -0.84760 3.68400 25.08900 23.94900 # O2926 + 2927 976 2 0.42380 3.37000 24.45900 24.65800 # H2927 + 2928 976 2 0.42380 4.71700 25.12100 23.99000 # H2928 + 2929 977 1 -0.84760 15.56400 28.53900 -0.79300 # O2929 + 2930 977 2 0.42380 15.51600 27.90500 -1.53600 # H2930 + 2931 977 2 0.42380 15.91500 27.91800 -0.04900 # H2931 + 2932 978 1 -0.84760 11.59700 12.21700 12.44700 # O2932 + 2933 978 2 0.42380 12.58500 11.83500 12.18100 # H2933 + 2934 978 2 0.42380 11.75300 13.09700 12.93800 # H2934 + 2935 979 1 -0.84760 25.65600 21.36700 6.38200 # O2935 + 2936 979 2 0.42380 25.71400 20.46600 6.86800 # H2936 + 2937 979 2 0.42380 26.47900 21.85000 6.73200 # H2937 + 2938 980 1 -0.84760 24.32900 6.83900 3.81300 # O2938 + 2939 980 2 0.42380 24.69400 5.94000 4.12200 # H2939 + 2940 980 2 0.42380 23.34000 6.78200 3.77900 # H2940 + 2941 981 1 -0.84760 0.86900 4.76800 3.28200 # O2941 + 2942 981 2 0.42380 0.96600 3.72200 3.49000 # H2942 + 2943 981 2 0.42380 0.85200 5.28800 4.15800 # H2943 + 2944 982 1 -0.84760 19.13800 23.57400 1.38400 # O2944 + 2945 982 2 0.42380 18.88900 24.55400 1.39300 # H2945 + 2946 982 2 0.42380 19.80700 23.39200 2.11900 # H2946 + 2947 983 1 -0.84760 6.53100 15.10900 26.06900 # O2947 + 2948 983 2 0.42380 6.10100 14.24000 26.22500 # H2948 + 2949 983 2 0.42380 7.57600 15.07800 25.95300 # H2949 + 2950 984 1 -0.84760 26.23600 8.42900 26.73400 # O2950 + 2951 984 2 0.42380 25.48000 8.39800 27.39900 # H2951 + 2952 984 2 0.42380 26.07400 7.74800 26.04700 # H2952 + 2953 985 1 -0.84760 24.57800 5.44600 8.08000 # O2953 + 2954 985 2 0.42380 24.47700 5.17400 7.05100 # H2954 + 2955 985 2 0.42380 23.65100 5.21700 8.41700 # H2955 + 2956 986 1 -0.84760 17.70800 1.71000 25.85400 # O2956 + 2957 986 2 0.42380 17.94700 1.97800 24.88200 # H2957 + 2958 986 2 0.42380 17.92600 0.76600 26.16300 # H2958 + 2959 987 1 -0.84760 3.39800 27.20600 22.11100 # O2959 + 2960 987 2 0.42380 3.61700 27.64700 21.25400 # H2960 + 2961 987 2 0.42380 3.92700 26.36400 22.40100 # H2961 + 2962 988 1 -0.84760 15.53600 18.22400 30.10500 # O2962 + 2963 988 2 0.42380 16.26400 18.07900 29.41900 # H2963 + 2964 988 2 0.42380 15.23600 17.22900 30.21600 # H2964 + 2965 989 1 -0.84760 18.67400 7.96200 20.94000 # O2965 + 2966 989 2 0.42380 18.15100 8.33300 20.11200 # H2966 + 2967 989 2 0.42380 17.97400 7.29100 21.31300 # H2967 + 2968 990 1 -0.84760 25.40900 21.16600 9.51300 # O2968 + 2969 990 2 0.42380 25.79200 21.14100 8.59800 # H2969 + 2970 990 2 0.42380 25.33300 22.17200 9.72600 # H2970 + 2971 991 1 -0.84760 19.75300 11.42500 13.98700 # O2971 + 2972 991 2 0.42380 19.83500 10.56700 14.48800 # H2972 + 2973 991 2 0.42380 19.00900 11.20900 13.35300 # H2973 + 2974 992 1 -0.84760 17.63800 13.68500 22.98500 # O2974 + 2975 992 2 0.42380 18.21300 12.84500 22.86900 # H2975 + 2976 992 2 0.42380 16.73000 13.27000 23.12100 # H2976 + 2977 993 1 -0.84760 27.39900 29.95700 26.23000 # O2977 + 2978 993 2 0.42380 27.70800 30.93500 26.31300 # H2978 + 2979 993 2 0.42380 26.69500 29.76400 26.96200 # H2979 + 2980 994 1 -0.84760 0.92500 26.26400 23.00600 # O2980 + 2981 994 2 0.42380 1.16900 26.54200 22.08600 # H2981 + 2982 994 2 0.42380 1.24800 26.99400 23.63500 # H2982 + 2983 995 1 -0.84760 10.67600 11.09800 15.98400 # O2983 + 2984 995 2 0.42380 10.30800 11.91200 16.45100 # H2984 + 2985 995 2 0.42380 9.86300 10.52900 15.64000 # H2985 + 2986 996 1 -0.84760 10.01800 17.14400 28.86700 # O2986 + 2987 996 2 0.42380 10.79400 17.22300 28.11500 # H2987 + 2988 996 2 0.42380 9.13000 17.25600 28.26800 # H2988 + 2989 997 1 -0.84760 7.67500 20.66300 23.83000 # O2989 + 2990 997 2 0.42380 8.37200 21.38200 24.04200 # H2990 + 2991 997 2 0.42380 7.87000 19.70500 24.21700 # H2991 + 2992 998 1 -0.84760 14.19000 12.06800 12.40500 # O2992 + 2993 998 2 0.42380 14.44200 12.51300 11.44100 # H2993 + 2994 998 2 0.42380 14.95300 12.46900 12.94900 # H2994 + 2995 999 1 -0.84760 29.27300 7.39000 0.56900 # O2995 + 2996 999 2 0.42380 28.67000 6.78700 0.03200 # H2996 + 2997 999 2 0.42380 29.05800 8.33200 0.22800 # H2997 + 2998 1000 1 -0.84760 12.04100 24.19700 27.58700 # O2998 + 2999 1000 2 0.42380 11.79500 24.66500 26.69800 # H2999 + 3000 1000 2 0.42380 11.47500 23.36800 27.42600 # H3000 + +Bonds + + 1 1 1 2 + 2 1 1 3 + 3 1 4 5 + 4 1 4 6 + 5 1 7 8 + 6 1 7 9 + 7 1 10 11 + 8 1 10 12 + 9 1 13 14 + 10 1 13 15 + 11 1 16 17 + 12 1 16 18 + 13 1 19 20 + 14 1 19 21 + 15 1 22 23 + 16 1 22 24 + 17 1 25 26 + 18 1 25 27 + 19 1 28 29 + 20 1 28 30 + 21 1 31 32 + 22 1 31 33 + 23 1 34 35 + 24 1 34 36 + 25 1 37 38 + 26 1 37 39 + 27 1 40 41 + 28 1 40 42 + 29 1 43 44 + 30 1 43 45 + 31 1 46 47 + 32 1 46 48 + 33 1 49 50 + 34 1 49 51 + 35 1 52 53 + 36 1 52 54 + 37 1 55 56 + 38 1 55 57 + 39 1 58 59 + 40 1 58 60 + 41 1 61 62 + 42 1 61 63 + 43 1 64 65 + 44 1 64 66 + 45 1 67 68 + 46 1 67 69 + 47 1 70 71 + 48 1 70 72 + 49 1 73 74 + 50 1 73 75 + 51 1 76 77 + 52 1 76 78 + 53 1 79 80 + 54 1 79 81 + 55 1 82 83 + 56 1 82 84 + 57 1 85 86 + 58 1 85 87 + 59 1 88 89 + 60 1 88 90 + 61 1 91 92 + 62 1 91 93 + 63 1 94 95 + 64 1 94 96 + 65 1 97 98 + 66 1 97 99 + 67 1 100 101 + 68 1 100 102 + 69 1 103 104 + 70 1 103 105 + 71 1 106 107 + 72 1 106 108 + 73 1 109 110 + 74 1 109 111 + 75 1 112 113 + 76 1 112 114 + 77 1 115 116 + 78 1 115 117 + 79 1 118 119 + 80 1 118 120 + 81 1 121 122 + 82 1 121 123 + 83 1 124 125 + 84 1 124 126 + 85 1 127 128 + 86 1 127 129 + 87 1 130 131 + 88 1 130 132 + 89 1 133 134 + 90 1 133 135 + 91 1 136 137 + 92 1 136 138 + 93 1 139 140 + 94 1 139 141 + 95 1 142 143 + 96 1 142 144 + 97 1 145 146 + 98 1 145 147 + 99 1 148 149 + 100 1 148 150 + 101 1 151 152 + 102 1 151 153 + 103 1 154 155 + 104 1 154 156 + 105 1 157 158 + 106 1 157 159 + 107 1 160 161 + 108 1 160 162 + 109 1 163 164 + 110 1 163 165 + 111 1 166 167 + 112 1 166 168 + 113 1 169 170 + 114 1 169 171 + 115 1 172 173 + 116 1 172 174 + 117 1 175 176 + 118 1 175 177 + 119 1 178 179 + 120 1 178 180 + 121 1 181 182 + 122 1 181 183 + 123 1 184 185 + 124 1 184 186 + 125 1 187 188 + 126 1 187 189 + 127 1 190 191 + 128 1 190 192 + 129 1 193 194 + 130 1 193 195 + 131 1 196 197 + 132 1 196 198 + 133 1 199 200 + 134 1 199 201 + 135 1 202 203 + 136 1 202 204 + 137 1 205 206 + 138 1 205 207 + 139 1 208 209 + 140 1 208 210 + 141 1 211 212 + 142 1 211 213 + 143 1 214 215 + 144 1 214 216 + 145 1 217 218 + 146 1 217 219 + 147 1 220 221 + 148 1 220 222 + 149 1 223 224 + 150 1 223 225 + 151 1 226 227 + 152 1 226 228 + 153 1 229 230 + 154 1 229 231 + 155 1 232 233 + 156 1 232 234 + 157 1 235 236 + 158 1 235 237 + 159 1 238 239 + 160 1 238 240 + 161 1 241 242 + 162 1 241 243 + 163 1 244 245 + 164 1 244 246 + 165 1 247 248 + 166 1 247 249 + 167 1 250 251 + 168 1 250 252 + 169 1 253 254 + 170 1 253 255 + 171 1 256 257 + 172 1 256 258 + 173 1 259 260 + 174 1 259 261 + 175 1 262 263 + 176 1 262 264 + 177 1 265 266 + 178 1 265 267 + 179 1 268 269 + 180 1 268 270 + 181 1 271 272 + 182 1 271 273 + 183 1 274 275 + 184 1 274 276 + 185 1 277 278 + 186 1 277 279 + 187 1 280 281 + 188 1 280 282 + 189 1 283 284 + 190 1 283 285 + 191 1 286 287 + 192 1 286 288 + 193 1 289 290 + 194 1 289 291 + 195 1 292 293 + 196 1 292 294 + 197 1 295 296 + 198 1 295 297 + 199 1 298 299 + 200 1 298 300 + 201 1 301 302 + 202 1 301 303 + 203 1 304 305 + 204 1 304 306 + 205 1 307 308 + 206 1 307 309 + 207 1 310 311 + 208 1 310 312 + 209 1 313 314 + 210 1 313 315 + 211 1 316 317 + 212 1 316 318 + 213 1 319 320 + 214 1 319 321 + 215 1 322 323 + 216 1 322 324 + 217 1 325 326 + 218 1 325 327 + 219 1 328 329 + 220 1 328 330 + 221 1 331 332 + 222 1 331 333 + 223 1 334 335 + 224 1 334 336 + 225 1 337 338 + 226 1 337 339 + 227 1 340 341 + 228 1 340 342 + 229 1 343 344 + 230 1 343 345 + 231 1 346 347 + 232 1 346 348 + 233 1 349 350 + 234 1 349 351 + 235 1 352 353 + 236 1 352 354 + 237 1 355 356 + 238 1 355 357 + 239 1 358 359 + 240 1 358 360 + 241 1 361 362 + 242 1 361 363 + 243 1 364 365 + 244 1 364 366 + 245 1 367 368 + 246 1 367 369 + 247 1 370 371 + 248 1 370 372 + 249 1 373 374 + 250 1 373 375 + 251 1 376 377 + 252 1 376 378 + 253 1 379 380 + 254 1 379 381 + 255 1 382 383 + 256 1 382 384 + 257 1 385 386 + 258 1 385 387 + 259 1 388 389 + 260 1 388 390 + 261 1 391 392 + 262 1 391 393 + 263 1 394 395 + 264 1 394 396 + 265 1 397 398 + 266 1 397 399 + 267 1 400 401 + 268 1 400 402 + 269 1 403 404 + 270 1 403 405 + 271 1 406 407 + 272 1 406 408 + 273 1 409 410 + 274 1 409 411 + 275 1 412 413 + 276 1 412 414 + 277 1 415 416 + 278 1 415 417 + 279 1 418 419 + 280 1 418 420 + 281 1 421 422 + 282 1 421 423 + 283 1 424 425 + 284 1 424 426 + 285 1 427 428 + 286 1 427 429 + 287 1 430 431 + 288 1 430 432 + 289 1 433 434 + 290 1 433 435 + 291 1 436 437 + 292 1 436 438 + 293 1 439 440 + 294 1 439 441 + 295 1 442 443 + 296 1 442 444 + 297 1 445 446 + 298 1 445 447 + 299 1 448 449 + 300 1 448 450 + 301 1 451 452 + 302 1 451 453 + 303 1 454 455 + 304 1 454 456 + 305 1 457 458 + 306 1 457 459 + 307 1 460 461 + 308 1 460 462 + 309 1 463 464 + 310 1 463 465 + 311 1 466 467 + 312 1 466 468 + 313 1 469 470 + 314 1 469 471 + 315 1 472 473 + 316 1 472 474 + 317 1 475 476 + 318 1 475 477 + 319 1 478 479 + 320 1 478 480 + 321 1 481 482 + 322 1 481 483 + 323 1 484 485 + 324 1 484 486 + 325 1 487 488 + 326 1 487 489 + 327 1 490 491 + 328 1 490 492 + 329 1 493 494 + 330 1 493 495 + 331 1 496 497 + 332 1 496 498 + 333 1 499 500 + 334 1 499 501 + 335 1 502 503 + 336 1 502 504 + 337 1 505 506 + 338 1 505 507 + 339 1 508 509 + 340 1 508 510 + 341 1 511 512 + 342 1 511 513 + 343 1 514 515 + 344 1 514 516 + 345 1 517 518 + 346 1 517 519 + 347 1 520 521 + 348 1 520 522 + 349 1 523 524 + 350 1 523 525 + 351 1 526 527 + 352 1 526 528 + 353 1 529 530 + 354 1 529 531 + 355 1 532 533 + 356 1 532 534 + 357 1 535 536 + 358 1 535 537 + 359 1 538 539 + 360 1 538 540 + 361 1 541 542 + 362 1 541 543 + 363 1 544 545 + 364 1 544 546 + 365 1 547 548 + 366 1 547 549 + 367 1 550 551 + 368 1 550 552 + 369 1 553 554 + 370 1 553 555 + 371 1 556 557 + 372 1 556 558 + 373 1 559 560 + 374 1 559 561 + 375 1 562 563 + 376 1 562 564 + 377 1 565 566 + 378 1 565 567 + 379 1 568 569 + 380 1 568 570 + 381 1 571 572 + 382 1 571 573 + 383 1 574 575 + 384 1 574 576 + 385 1 577 578 + 386 1 577 579 + 387 1 580 581 + 388 1 580 582 + 389 1 583 584 + 390 1 583 585 + 391 1 586 587 + 392 1 586 588 + 393 1 589 590 + 394 1 589 591 + 395 1 592 593 + 396 1 592 594 + 397 1 595 596 + 398 1 595 597 + 399 1 598 599 + 400 1 598 600 + 401 1 601 602 + 402 1 601 603 + 403 1 604 605 + 404 1 604 606 + 405 1 607 608 + 406 1 607 609 + 407 1 610 611 + 408 1 610 612 + 409 1 613 614 + 410 1 613 615 + 411 1 616 617 + 412 1 616 618 + 413 1 619 620 + 414 1 619 621 + 415 1 622 623 + 416 1 622 624 + 417 1 625 626 + 418 1 625 627 + 419 1 628 629 + 420 1 628 630 + 421 1 631 632 + 422 1 631 633 + 423 1 634 635 + 424 1 634 636 + 425 1 637 638 + 426 1 637 639 + 427 1 640 641 + 428 1 640 642 + 429 1 643 644 + 430 1 643 645 + 431 1 646 647 + 432 1 646 648 + 433 1 649 650 + 434 1 649 651 + 435 1 652 653 + 436 1 652 654 + 437 1 655 656 + 438 1 655 657 + 439 1 658 659 + 440 1 658 660 + 441 1 661 662 + 442 1 661 663 + 443 1 664 665 + 444 1 664 666 + 445 1 667 668 + 446 1 667 669 + 447 1 670 671 + 448 1 670 672 + 449 1 673 674 + 450 1 673 675 + 451 1 676 677 + 452 1 676 678 + 453 1 679 680 + 454 1 679 681 + 455 1 682 683 + 456 1 682 684 + 457 1 685 686 + 458 1 685 687 + 459 1 688 689 + 460 1 688 690 + 461 1 691 692 + 462 1 691 693 + 463 1 694 695 + 464 1 694 696 + 465 1 697 698 + 466 1 697 699 + 467 1 700 701 + 468 1 700 702 + 469 1 703 704 + 470 1 703 705 + 471 1 706 707 + 472 1 706 708 + 473 1 709 710 + 474 1 709 711 + 475 1 712 713 + 476 1 712 714 + 477 1 715 716 + 478 1 715 717 + 479 1 718 719 + 480 1 718 720 + 481 1 721 722 + 482 1 721 723 + 483 1 724 725 + 484 1 724 726 + 485 1 727 728 + 486 1 727 729 + 487 1 730 731 + 488 1 730 732 + 489 1 733 734 + 490 1 733 735 + 491 1 736 737 + 492 1 736 738 + 493 1 739 740 + 494 1 739 741 + 495 1 742 743 + 496 1 742 744 + 497 1 745 746 + 498 1 745 747 + 499 1 748 749 + 500 1 748 750 + 501 1 751 752 + 502 1 751 753 + 503 1 754 755 + 504 1 754 756 + 505 1 757 758 + 506 1 757 759 + 507 1 760 761 + 508 1 760 762 + 509 1 763 764 + 510 1 763 765 + 511 1 766 767 + 512 1 766 768 + 513 1 769 770 + 514 1 769 771 + 515 1 772 773 + 516 1 772 774 + 517 1 775 776 + 518 1 775 777 + 519 1 778 779 + 520 1 778 780 + 521 1 781 782 + 522 1 781 783 + 523 1 784 785 + 524 1 784 786 + 525 1 787 788 + 526 1 787 789 + 527 1 790 791 + 528 1 790 792 + 529 1 793 794 + 530 1 793 795 + 531 1 796 797 + 532 1 796 798 + 533 1 799 800 + 534 1 799 801 + 535 1 802 803 + 536 1 802 804 + 537 1 805 806 + 538 1 805 807 + 539 1 808 809 + 540 1 808 810 + 541 1 811 812 + 542 1 811 813 + 543 1 814 815 + 544 1 814 816 + 545 1 817 818 + 546 1 817 819 + 547 1 820 821 + 548 1 820 822 + 549 1 823 824 + 550 1 823 825 + 551 1 826 827 + 552 1 826 828 + 553 1 829 830 + 554 1 829 831 + 555 1 832 833 + 556 1 832 834 + 557 1 835 836 + 558 1 835 837 + 559 1 838 839 + 560 1 838 840 + 561 1 841 842 + 562 1 841 843 + 563 1 844 845 + 564 1 844 846 + 565 1 847 848 + 566 1 847 849 + 567 1 850 851 + 568 1 850 852 + 569 1 853 854 + 570 1 853 855 + 571 1 856 857 + 572 1 856 858 + 573 1 859 860 + 574 1 859 861 + 575 1 862 863 + 576 1 862 864 + 577 1 865 866 + 578 1 865 867 + 579 1 868 869 + 580 1 868 870 + 581 1 871 872 + 582 1 871 873 + 583 1 874 875 + 584 1 874 876 + 585 1 877 878 + 586 1 877 879 + 587 1 880 881 + 588 1 880 882 + 589 1 883 884 + 590 1 883 885 + 591 1 886 887 + 592 1 886 888 + 593 1 889 890 + 594 1 889 891 + 595 1 892 893 + 596 1 892 894 + 597 1 895 896 + 598 1 895 897 + 599 1 898 899 + 600 1 898 900 + 601 1 901 902 + 602 1 901 903 + 603 1 904 905 + 604 1 904 906 + 605 1 907 908 + 606 1 907 909 + 607 1 910 911 + 608 1 910 912 + 609 1 913 914 + 610 1 913 915 + 611 1 916 917 + 612 1 916 918 + 613 1 919 920 + 614 1 919 921 + 615 1 922 923 + 616 1 922 924 + 617 1 925 926 + 618 1 925 927 + 619 1 928 929 + 620 1 928 930 + 621 1 931 932 + 622 1 931 933 + 623 1 934 935 + 624 1 934 936 + 625 1 937 938 + 626 1 937 939 + 627 1 940 941 + 628 1 940 942 + 629 1 943 944 + 630 1 943 945 + 631 1 946 947 + 632 1 946 948 + 633 1 949 950 + 634 1 949 951 + 635 1 952 953 + 636 1 952 954 + 637 1 955 956 + 638 1 955 957 + 639 1 958 959 + 640 1 958 960 + 641 1 961 962 + 642 1 961 963 + 643 1 964 965 + 644 1 964 966 + 645 1 967 968 + 646 1 967 969 + 647 1 970 971 + 648 1 970 972 + 649 1 973 974 + 650 1 973 975 + 651 1 976 977 + 652 1 976 978 + 653 1 979 980 + 654 1 979 981 + 655 1 982 983 + 656 1 982 984 + 657 1 985 986 + 658 1 985 987 + 659 1 988 989 + 660 1 988 990 + 661 1 991 992 + 662 1 991 993 + 663 1 994 995 + 664 1 994 996 + 665 1 997 998 + 666 1 997 999 + 667 1 1000 1001 + 668 1 1000 1002 + 669 1 1003 1004 + 670 1 1003 1005 + 671 1 1006 1007 + 672 1 1006 1008 + 673 1 1009 1010 + 674 1 1009 1011 + 675 1 1012 1013 + 676 1 1012 1014 + 677 1 1015 1016 + 678 1 1015 1017 + 679 1 1018 1019 + 680 1 1018 1020 + 681 1 1021 1022 + 682 1 1021 1023 + 683 1 1024 1025 + 684 1 1024 1026 + 685 1 1027 1028 + 686 1 1027 1029 + 687 1 1030 1031 + 688 1 1030 1032 + 689 1 1033 1034 + 690 1 1033 1035 + 691 1 1036 1037 + 692 1 1036 1038 + 693 1 1039 1040 + 694 1 1039 1041 + 695 1 1042 1043 + 696 1 1042 1044 + 697 1 1045 1046 + 698 1 1045 1047 + 699 1 1048 1049 + 700 1 1048 1050 + 701 1 1051 1052 + 702 1 1051 1053 + 703 1 1054 1055 + 704 1 1054 1056 + 705 1 1057 1058 + 706 1 1057 1059 + 707 1 1060 1061 + 708 1 1060 1062 + 709 1 1063 1064 + 710 1 1063 1065 + 711 1 1066 1067 + 712 1 1066 1068 + 713 1 1069 1070 + 714 1 1069 1071 + 715 1 1072 1073 + 716 1 1072 1074 + 717 1 1075 1076 + 718 1 1075 1077 + 719 1 1078 1079 + 720 1 1078 1080 + 721 1 1081 1082 + 722 1 1081 1083 + 723 1 1084 1085 + 724 1 1084 1086 + 725 1 1087 1088 + 726 1 1087 1089 + 727 1 1090 1091 + 728 1 1090 1092 + 729 1 1093 1094 + 730 1 1093 1095 + 731 1 1096 1097 + 732 1 1096 1098 + 733 1 1099 1100 + 734 1 1099 1101 + 735 1 1102 1103 + 736 1 1102 1104 + 737 1 1105 1106 + 738 1 1105 1107 + 739 1 1108 1109 + 740 1 1108 1110 + 741 1 1111 1112 + 742 1 1111 1113 + 743 1 1114 1115 + 744 1 1114 1116 + 745 1 1117 1118 + 746 1 1117 1119 + 747 1 1120 1121 + 748 1 1120 1122 + 749 1 1123 1124 + 750 1 1123 1125 + 751 1 1126 1127 + 752 1 1126 1128 + 753 1 1129 1130 + 754 1 1129 1131 + 755 1 1132 1133 + 756 1 1132 1134 + 757 1 1135 1136 + 758 1 1135 1137 + 759 1 1138 1139 + 760 1 1138 1140 + 761 1 1141 1142 + 762 1 1141 1143 + 763 1 1144 1145 + 764 1 1144 1146 + 765 1 1147 1148 + 766 1 1147 1149 + 767 1 1150 1151 + 768 1 1150 1152 + 769 1 1153 1154 + 770 1 1153 1155 + 771 1 1156 1157 + 772 1 1156 1158 + 773 1 1159 1160 + 774 1 1159 1161 + 775 1 1162 1163 + 776 1 1162 1164 + 777 1 1165 1166 + 778 1 1165 1167 + 779 1 1168 1169 + 780 1 1168 1170 + 781 1 1171 1172 + 782 1 1171 1173 + 783 1 1174 1175 + 784 1 1174 1176 + 785 1 1177 1178 + 786 1 1177 1179 + 787 1 1180 1181 + 788 1 1180 1182 + 789 1 1183 1184 + 790 1 1183 1185 + 791 1 1186 1187 + 792 1 1186 1188 + 793 1 1189 1190 + 794 1 1189 1191 + 795 1 1192 1193 + 796 1 1192 1194 + 797 1 1195 1196 + 798 1 1195 1197 + 799 1 1198 1199 + 800 1 1198 1200 + 801 1 1201 1202 + 802 1 1201 1203 + 803 1 1204 1205 + 804 1 1204 1206 + 805 1 1207 1208 + 806 1 1207 1209 + 807 1 1210 1211 + 808 1 1210 1212 + 809 1 1213 1214 + 810 1 1213 1215 + 811 1 1216 1217 + 812 1 1216 1218 + 813 1 1219 1220 + 814 1 1219 1221 + 815 1 1222 1223 + 816 1 1222 1224 + 817 1 1225 1226 + 818 1 1225 1227 + 819 1 1228 1229 + 820 1 1228 1230 + 821 1 1231 1232 + 822 1 1231 1233 + 823 1 1234 1235 + 824 1 1234 1236 + 825 1 1237 1238 + 826 1 1237 1239 + 827 1 1240 1241 + 828 1 1240 1242 + 829 1 1243 1244 + 830 1 1243 1245 + 831 1 1246 1247 + 832 1 1246 1248 + 833 1 1249 1250 + 834 1 1249 1251 + 835 1 1252 1253 + 836 1 1252 1254 + 837 1 1255 1256 + 838 1 1255 1257 + 839 1 1258 1259 + 840 1 1258 1260 + 841 1 1261 1262 + 842 1 1261 1263 + 843 1 1264 1265 + 844 1 1264 1266 + 845 1 1267 1268 + 846 1 1267 1269 + 847 1 1270 1271 + 848 1 1270 1272 + 849 1 1273 1274 + 850 1 1273 1275 + 851 1 1276 1277 + 852 1 1276 1278 + 853 1 1279 1280 + 854 1 1279 1281 + 855 1 1282 1283 + 856 1 1282 1284 + 857 1 1285 1286 + 858 1 1285 1287 + 859 1 1288 1289 + 860 1 1288 1290 + 861 1 1291 1292 + 862 1 1291 1293 + 863 1 1294 1295 + 864 1 1294 1296 + 865 1 1297 1298 + 866 1 1297 1299 + 867 1 1300 1301 + 868 1 1300 1302 + 869 1 1303 1304 + 870 1 1303 1305 + 871 1 1306 1307 + 872 1 1306 1308 + 873 1 1309 1310 + 874 1 1309 1311 + 875 1 1312 1313 + 876 1 1312 1314 + 877 1 1315 1316 + 878 1 1315 1317 + 879 1 1318 1319 + 880 1 1318 1320 + 881 1 1321 1322 + 882 1 1321 1323 + 883 1 1324 1325 + 884 1 1324 1326 + 885 1 1327 1328 + 886 1 1327 1329 + 887 1 1330 1331 + 888 1 1330 1332 + 889 1 1333 1334 + 890 1 1333 1335 + 891 1 1336 1337 + 892 1 1336 1338 + 893 1 1339 1340 + 894 1 1339 1341 + 895 1 1342 1343 + 896 1 1342 1344 + 897 1 1345 1346 + 898 1 1345 1347 + 899 1 1348 1349 + 900 1 1348 1350 + 901 1 1351 1352 + 902 1 1351 1353 + 903 1 1354 1355 + 904 1 1354 1356 + 905 1 1357 1358 + 906 1 1357 1359 + 907 1 1360 1361 + 908 1 1360 1362 + 909 1 1363 1364 + 910 1 1363 1365 + 911 1 1366 1367 + 912 1 1366 1368 + 913 1 1369 1370 + 914 1 1369 1371 + 915 1 1372 1373 + 916 1 1372 1374 + 917 1 1375 1376 + 918 1 1375 1377 + 919 1 1378 1379 + 920 1 1378 1380 + 921 1 1381 1382 + 922 1 1381 1383 + 923 1 1384 1385 + 924 1 1384 1386 + 925 1 1387 1388 + 926 1 1387 1389 + 927 1 1390 1391 + 928 1 1390 1392 + 929 1 1393 1394 + 930 1 1393 1395 + 931 1 1396 1397 + 932 1 1396 1398 + 933 1 1399 1400 + 934 1 1399 1401 + 935 1 1402 1403 + 936 1 1402 1404 + 937 1 1405 1406 + 938 1 1405 1407 + 939 1 1408 1409 + 940 1 1408 1410 + 941 1 1411 1412 + 942 1 1411 1413 + 943 1 1414 1415 + 944 1 1414 1416 + 945 1 1417 1418 + 946 1 1417 1419 + 947 1 1420 1421 + 948 1 1420 1422 + 949 1 1423 1424 + 950 1 1423 1425 + 951 1 1426 1427 + 952 1 1426 1428 + 953 1 1429 1430 + 954 1 1429 1431 + 955 1 1432 1433 + 956 1 1432 1434 + 957 1 1435 1436 + 958 1 1435 1437 + 959 1 1438 1439 + 960 1 1438 1440 + 961 1 1441 1442 + 962 1 1441 1443 + 963 1 1444 1445 + 964 1 1444 1446 + 965 1 1447 1448 + 966 1 1447 1449 + 967 1 1450 1451 + 968 1 1450 1452 + 969 1 1453 1454 + 970 1 1453 1455 + 971 1 1456 1457 + 972 1 1456 1458 + 973 1 1459 1460 + 974 1 1459 1461 + 975 1 1462 1463 + 976 1 1462 1464 + 977 1 1465 1466 + 978 1 1465 1467 + 979 1 1468 1469 + 980 1 1468 1470 + 981 1 1471 1472 + 982 1 1471 1473 + 983 1 1474 1475 + 984 1 1474 1476 + 985 1 1477 1478 + 986 1 1477 1479 + 987 1 1480 1481 + 988 1 1480 1482 + 989 1 1483 1484 + 990 1 1483 1485 + 991 1 1486 1487 + 992 1 1486 1488 + 993 1 1489 1490 + 994 1 1489 1491 + 995 1 1492 1493 + 996 1 1492 1494 + 997 1 1495 1496 + 998 1 1495 1497 + 999 1 1498 1499 + 1000 1 1498 1500 + 1001 1 1501 1502 + 1002 1 1501 1503 + 1003 1 1504 1505 + 1004 1 1504 1506 + 1005 1 1507 1508 + 1006 1 1507 1509 + 1007 1 1510 1511 + 1008 1 1510 1512 + 1009 1 1513 1514 + 1010 1 1513 1515 + 1011 1 1516 1517 + 1012 1 1516 1518 + 1013 1 1519 1520 + 1014 1 1519 1521 + 1015 1 1522 1523 + 1016 1 1522 1524 + 1017 1 1525 1526 + 1018 1 1525 1527 + 1019 1 1528 1529 + 1020 1 1528 1530 + 1021 1 1531 1532 + 1022 1 1531 1533 + 1023 1 1534 1535 + 1024 1 1534 1536 + 1025 1 1537 1538 + 1026 1 1537 1539 + 1027 1 1540 1541 + 1028 1 1540 1542 + 1029 1 1543 1544 + 1030 1 1543 1545 + 1031 1 1546 1547 + 1032 1 1546 1548 + 1033 1 1549 1550 + 1034 1 1549 1551 + 1035 1 1552 1553 + 1036 1 1552 1554 + 1037 1 1555 1556 + 1038 1 1555 1557 + 1039 1 1558 1559 + 1040 1 1558 1560 + 1041 1 1561 1562 + 1042 1 1561 1563 + 1043 1 1564 1565 + 1044 1 1564 1566 + 1045 1 1567 1568 + 1046 1 1567 1569 + 1047 1 1570 1571 + 1048 1 1570 1572 + 1049 1 1573 1574 + 1050 1 1573 1575 + 1051 1 1576 1577 + 1052 1 1576 1578 + 1053 1 1579 1580 + 1054 1 1579 1581 + 1055 1 1582 1583 + 1056 1 1582 1584 + 1057 1 1585 1586 + 1058 1 1585 1587 + 1059 1 1588 1589 + 1060 1 1588 1590 + 1061 1 1591 1592 + 1062 1 1591 1593 + 1063 1 1594 1595 + 1064 1 1594 1596 + 1065 1 1597 1598 + 1066 1 1597 1599 + 1067 1 1600 1601 + 1068 1 1600 1602 + 1069 1 1603 1604 + 1070 1 1603 1605 + 1071 1 1606 1607 + 1072 1 1606 1608 + 1073 1 1609 1610 + 1074 1 1609 1611 + 1075 1 1612 1613 + 1076 1 1612 1614 + 1077 1 1615 1616 + 1078 1 1615 1617 + 1079 1 1618 1619 + 1080 1 1618 1620 + 1081 1 1621 1622 + 1082 1 1621 1623 + 1083 1 1624 1625 + 1084 1 1624 1626 + 1085 1 1627 1628 + 1086 1 1627 1629 + 1087 1 1630 1631 + 1088 1 1630 1632 + 1089 1 1633 1634 + 1090 1 1633 1635 + 1091 1 1636 1637 + 1092 1 1636 1638 + 1093 1 1639 1640 + 1094 1 1639 1641 + 1095 1 1642 1643 + 1096 1 1642 1644 + 1097 1 1645 1646 + 1098 1 1645 1647 + 1099 1 1648 1649 + 1100 1 1648 1650 + 1101 1 1651 1652 + 1102 1 1651 1653 + 1103 1 1654 1655 + 1104 1 1654 1656 + 1105 1 1657 1658 + 1106 1 1657 1659 + 1107 1 1660 1661 + 1108 1 1660 1662 + 1109 1 1663 1664 + 1110 1 1663 1665 + 1111 1 1666 1667 + 1112 1 1666 1668 + 1113 1 1669 1670 + 1114 1 1669 1671 + 1115 1 1672 1673 + 1116 1 1672 1674 + 1117 1 1675 1676 + 1118 1 1675 1677 + 1119 1 1678 1679 + 1120 1 1678 1680 + 1121 1 1681 1682 + 1122 1 1681 1683 + 1123 1 1684 1685 + 1124 1 1684 1686 + 1125 1 1687 1688 + 1126 1 1687 1689 + 1127 1 1690 1691 + 1128 1 1690 1692 + 1129 1 1693 1694 + 1130 1 1693 1695 + 1131 1 1696 1697 + 1132 1 1696 1698 + 1133 1 1699 1700 + 1134 1 1699 1701 + 1135 1 1702 1703 + 1136 1 1702 1704 + 1137 1 1705 1706 + 1138 1 1705 1707 + 1139 1 1708 1709 + 1140 1 1708 1710 + 1141 1 1711 1712 + 1142 1 1711 1713 + 1143 1 1714 1715 + 1144 1 1714 1716 + 1145 1 1717 1718 + 1146 1 1717 1719 + 1147 1 1720 1721 + 1148 1 1720 1722 + 1149 1 1723 1724 + 1150 1 1723 1725 + 1151 1 1726 1727 + 1152 1 1726 1728 + 1153 1 1729 1730 + 1154 1 1729 1731 + 1155 1 1732 1733 + 1156 1 1732 1734 + 1157 1 1735 1736 + 1158 1 1735 1737 + 1159 1 1738 1739 + 1160 1 1738 1740 + 1161 1 1741 1742 + 1162 1 1741 1743 + 1163 1 1744 1745 + 1164 1 1744 1746 + 1165 1 1747 1748 + 1166 1 1747 1749 + 1167 1 1750 1751 + 1168 1 1750 1752 + 1169 1 1753 1754 + 1170 1 1753 1755 + 1171 1 1756 1757 + 1172 1 1756 1758 + 1173 1 1759 1760 + 1174 1 1759 1761 + 1175 1 1762 1763 + 1176 1 1762 1764 + 1177 1 1765 1766 + 1178 1 1765 1767 + 1179 1 1768 1769 + 1180 1 1768 1770 + 1181 1 1771 1772 + 1182 1 1771 1773 + 1183 1 1774 1775 + 1184 1 1774 1776 + 1185 1 1777 1778 + 1186 1 1777 1779 + 1187 1 1780 1781 + 1188 1 1780 1782 + 1189 1 1783 1784 + 1190 1 1783 1785 + 1191 1 1786 1787 + 1192 1 1786 1788 + 1193 1 1789 1790 + 1194 1 1789 1791 + 1195 1 1792 1793 + 1196 1 1792 1794 + 1197 1 1795 1796 + 1198 1 1795 1797 + 1199 1 1798 1799 + 1200 1 1798 1800 + 1201 1 1801 1802 + 1202 1 1801 1803 + 1203 1 1804 1805 + 1204 1 1804 1806 + 1205 1 1807 1808 + 1206 1 1807 1809 + 1207 1 1810 1811 + 1208 1 1810 1812 + 1209 1 1813 1814 + 1210 1 1813 1815 + 1211 1 1816 1817 + 1212 1 1816 1818 + 1213 1 1819 1820 + 1214 1 1819 1821 + 1215 1 1822 1823 + 1216 1 1822 1824 + 1217 1 1825 1826 + 1218 1 1825 1827 + 1219 1 1828 1829 + 1220 1 1828 1830 + 1221 1 1831 1832 + 1222 1 1831 1833 + 1223 1 1834 1835 + 1224 1 1834 1836 + 1225 1 1837 1838 + 1226 1 1837 1839 + 1227 1 1840 1841 + 1228 1 1840 1842 + 1229 1 1843 1844 + 1230 1 1843 1845 + 1231 1 1846 1847 + 1232 1 1846 1848 + 1233 1 1849 1850 + 1234 1 1849 1851 + 1235 1 1852 1853 + 1236 1 1852 1854 + 1237 1 1855 1856 + 1238 1 1855 1857 + 1239 1 1858 1859 + 1240 1 1858 1860 + 1241 1 1861 1862 + 1242 1 1861 1863 + 1243 1 1864 1865 + 1244 1 1864 1866 + 1245 1 1867 1868 + 1246 1 1867 1869 + 1247 1 1870 1871 + 1248 1 1870 1872 + 1249 1 1873 1874 + 1250 1 1873 1875 + 1251 1 1876 1877 + 1252 1 1876 1878 + 1253 1 1879 1880 + 1254 1 1879 1881 + 1255 1 1882 1883 + 1256 1 1882 1884 + 1257 1 1885 1886 + 1258 1 1885 1887 + 1259 1 1888 1889 + 1260 1 1888 1890 + 1261 1 1891 1892 + 1262 1 1891 1893 + 1263 1 1894 1895 + 1264 1 1894 1896 + 1265 1 1897 1898 + 1266 1 1897 1899 + 1267 1 1900 1901 + 1268 1 1900 1902 + 1269 1 1903 1904 + 1270 1 1903 1905 + 1271 1 1906 1907 + 1272 1 1906 1908 + 1273 1 1909 1910 + 1274 1 1909 1911 + 1275 1 1912 1913 + 1276 1 1912 1914 + 1277 1 1915 1916 + 1278 1 1915 1917 + 1279 1 1918 1919 + 1280 1 1918 1920 + 1281 1 1921 1922 + 1282 1 1921 1923 + 1283 1 1924 1925 + 1284 1 1924 1926 + 1285 1 1927 1928 + 1286 1 1927 1929 + 1287 1 1930 1931 + 1288 1 1930 1932 + 1289 1 1933 1934 + 1290 1 1933 1935 + 1291 1 1936 1937 + 1292 1 1936 1938 + 1293 1 1939 1940 + 1294 1 1939 1941 + 1295 1 1942 1943 + 1296 1 1942 1944 + 1297 1 1945 1946 + 1298 1 1945 1947 + 1299 1 1948 1949 + 1300 1 1948 1950 + 1301 1 1951 1952 + 1302 1 1951 1953 + 1303 1 1954 1955 + 1304 1 1954 1956 + 1305 1 1957 1958 + 1306 1 1957 1959 + 1307 1 1960 1961 + 1308 1 1960 1962 + 1309 1 1963 1964 + 1310 1 1963 1965 + 1311 1 1966 1967 + 1312 1 1966 1968 + 1313 1 1969 1970 + 1314 1 1969 1971 + 1315 1 1972 1973 + 1316 1 1972 1974 + 1317 1 1975 1976 + 1318 1 1975 1977 + 1319 1 1978 1979 + 1320 1 1978 1980 + 1321 1 1981 1982 + 1322 1 1981 1983 + 1323 1 1984 1985 + 1324 1 1984 1986 + 1325 1 1987 1988 + 1326 1 1987 1989 + 1327 1 1990 1991 + 1328 1 1990 1992 + 1329 1 1993 1994 + 1330 1 1993 1995 + 1331 1 1996 1997 + 1332 1 1996 1998 + 1333 1 1999 2000 + 1334 1 1999 2001 + 1335 1 2002 2003 + 1336 1 2002 2004 + 1337 1 2005 2006 + 1338 1 2005 2007 + 1339 1 2008 2009 + 1340 1 2008 2010 + 1341 1 2011 2012 + 1342 1 2011 2013 + 1343 1 2014 2015 + 1344 1 2014 2016 + 1345 1 2017 2018 + 1346 1 2017 2019 + 1347 1 2020 2021 + 1348 1 2020 2022 + 1349 1 2023 2024 + 1350 1 2023 2025 + 1351 1 2026 2027 + 1352 1 2026 2028 + 1353 1 2029 2030 + 1354 1 2029 2031 + 1355 1 2032 2033 + 1356 1 2032 2034 + 1357 1 2035 2036 + 1358 1 2035 2037 + 1359 1 2038 2039 + 1360 1 2038 2040 + 1361 1 2041 2042 + 1362 1 2041 2043 + 1363 1 2044 2045 + 1364 1 2044 2046 + 1365 1 2047 2048 + 1366 1 2047 2049 + 1367 1 2050 2051 + 1368 1 2050 2052 + 1369 1 2053 2054 + 1370 1 2053 2055 + 1371 1 2056 2057 + 1372 1 2056 2058 + 1373 1 2059 2060 + 1374 1 2059 2061 + 1375 1 2062 2063 + 1376 1 2062 2064 + 1377 1 2065 2066 + 1378 1 2065 2067 + 1379 1 2068 2069 + 1380 1 2068 2070 + 1381 1 2071 2072 + 1382 1 2071 2073 + 1383 1 2074 2075 + 1384 1 2074 2076 + 1385 1 2077 2078 + 1386 1 2077 2079 + 1387 1 2080 2081 + 1388 1 2080 2082 + 1389 1 2083 2084 + 1390 1 2083 2085 + 1391 1 2086 2087 + 1392 1 2086 2088 + 1393 1 2089 2090 + 1394 1 2089 2091 + 1395 1 2092 2093 + 1396 1 2092 2094 + 1397 1 2095 2096 + 1398 1 2095 2097 + 1399 1 2098 2099 + 1400 1 2098 2100 + 1401 1 2101 2102 + 1402 1 2101 2103 + 1403 1 2104 2105 + 1404 1 2104 2106 + 1405 1 2107 2108 + 1406 1 2107 2109 + 1407 1 2110 2111 + 1408 1 2110 2112 + 1409 1 2113 2114 + 1410 1 2113 2115 + 1411 1 2116 2117 + 1412 1 2116 2118 + 1413 1 2119 2120 + 1414 1 2119 2121 + 1415 1 2122 2123 + 1416 1 2122 2124 + 1417 1 2125 2126 + 1418 1 2125 2127 + 1419 1 2128 2129 + 1420 1 2128 2130 + 1421 1 2131 2132 + 1422 1 2131 2133 + 1423 1 2134 2135 + 1424 1 2134 2136 + 1425 1 2137 2138 + 1426 1 2137 2139 + 1427 1 2140 2141 + 1428 1 2140 2142 + 1429 1 2143 2144 + 1430 1 2143 2145 + 1431 1 2146 2147 + 1432 1 2146 2148 + 1433 1 2149 2150 + 1434 1 2149 2151 + 1435 1 2152 2153 + 1436 1 2152 2154 + 1437 1 2155 2156 + 1438 1 2155 2157 + 1439 1 2158 2159 + 1440 1 2158 2160 + 1441 1 2161 2162 + 1442 1 2161 2163 + 1443 1 2164 2165 + 1444 1 2164 2166 + 1445 1 2167 2168 + 1446 1 2167 2169 + 1447 1 2170 2171 + 1448 1 2170 2172 + 1449 1 2173 2174 + 1450 1 2173 2175 + 1451 1 2176 2177 + 1452 1 2176 2178 + 1453 1 2179 2180 + 1454 1 2179 2181 + 1455 1 2182 2183 + 1456 1 2182 2184 + 1457 1 2185 2186 + 1458 1 2185 2187 + 1459 1 2188 2189 + 1460 1 2188 2190 + 1461 1 2191 2192 + 1462 1 2191 2193 + 1463 1 2194 2195 + 1464 1 2194 2196 + 1465 1 2197 2198 + 1466 1 2197 2199 + 1467 1 2200 2201 + 1468 1 2200 2202 + 1469 1 2203 2204 + 1470 1 2203 2205 + 1471 1 2206 2207 + 1472 1 2206 2208 + 1473 1 2209 2210 + 1474 1 2209 2211 + 1475 1 2212 2213 + 1476 1 2212 2214 + 1477 1 2215 2216 + 1478 1 2215 2217 + 1479 1 2218 2219 + 1480 1 2218 2220 + 1481 1 2221 2222 + 1482 1 2221 2223 + 1483 1 2224 2225 + 1484 1 2224 2226 + 1485 1 2227 2228 + 1486 1 2227 2229 + 1487 1 2230 2231 + 1488 1 2230 2232 + 1489 1 2233 2234 + 1490 1 2233 2235 + 1491 1 2236 2237 + 1492 1 2236 2238 + 1493 1 2239 2240 + 1494 1 2239 2241 + 1495 1 2242 2243 + 1496 1 2242 2244 + 1497 1 2245 2246 + 1498 1 2245 2247 + 1499 1 2248 2249 + 1500 1 2248 2250 + 1501 1 2251 2252 + 1502 1 2251 2253 + 1503 1 2254 2255 + 1504 1 2254 2256 + 1505 1 2257 2258 + 1506 1 2257 2259 + 1507 1 2260 2261 + 1508 1 2260 2262 + 1509 1 2263 2264 + 1510 1 2263 2265 + 1511 1 2266 2267 + 1512 1 2266 2268 + 1513 1 2269 2270 + 1514 1 2269 2271 + 1515 1 2272 2273 + 1516 1 2272 2274 + 1517 1 2275 2276 + 1518 1 2275 2277 + 1519 1 2278 2279 + 1520 1 2278 2280 + 1521 1 2281 2282 + 1522 1 2281 2283 + 1523 1 2284 2285 + 1524 1 2284 2286 + 1525 1 2287 2288 + 1526 1 2287 2289 + 1527 1 2290 2291 + 1528 1 2290 2292 + 1529 1 2293 2294 + 1530 1 2293 2295 + 1531 1 2296 2297 + 1532 1 2296 2298 + 1533 1 2299 2300 + 1534 1 2299 2301 + 1535 1 2302 2303 + 1536 1 2302 2304 + 1537 1 2305 2306 + 1538 1 2305 2307 + 1539 1 2308 2309 + 1540 1 2308 2310 + 1541 1 2311 2312 + 1542 1 2311 2313 + 1543 1 2314 2315 + 1544 1 2314 2316 + 1545 1 2317 2318 + 1546 1 2317 2319 + 1547 1 2320 2321 + 1548 1 2320 2322 + 1549 1 2323 2324 + 1550 1 2323 2325 + 1551 1 2326 2327 + 1552 1 2326 2328 + 1553 1 2329 2330 + 1554 1 2329 2331 + 1555 1 2332 2333 + 1556 1 2332 2334 + 1557 1 2335 2336 + 1558 1 2335 2337 + 1559 1 2338 2339 + 1560 1 2338 2340 + 1561 1 2341 2342 + 1562 1 2341 2343 + 1563 1 2344 2345 + 1564 1 2344 2346 + 1565 1 2347 2348 + 1566 1 2347 2349 + 1567 1 2350 2351 + 1568 1 2350 2352 + 1569 1 2353 2354 + 1570 1 2353 2355 + 1571 1 2356 2357 + 1572 1 2356 2358 + 1573 1 2359 2360 + 1574 1 2359 2361 + 1575 1 2362 2363 + 1576 1 2362 2364 + 1577 1 2365 2366 + 1578 1 2365 2367 + 1579 1 2368 2369 + 1580 1 2368 2370 + 1581 1 2371 2372 + 1582 1 2371 2373 + 1583 1 2374 2375 + 1584 1 2374 2376 + 1585 1 2377 2378 + 1586 1 2377 2379 + 1587 1 2380 2381 + 1588 1 2380 2382 + 1589 1 2383 2384 + 1590 1 2383 2385 + 1591 1 2386 2387 + 1592 1 2386 2388 + 1593 1 2389 2390 + 1594 1 2389 2391 + 1595 1 2392 2393 + 1596 1 2392 2394 + 1597 1 2395 2396 + 1598 1 2395 2397 + 1599 1 2398 2399 + 1600 1 2398 2400 + 1601 1 2401 2402 + 1602 1 2401 2403 + 1603 1 2404 2405 + 1604 1 2404 2406 + 1605 1 2407 2408 + 1606 1 2407 2409 + 1607 1 2410 2411 + 1608 1 2410 2412 + 1609 1 2413 2414 + 1610 1 2413 2415 + 1611 1 2416 2417 + 1612 1 2416 2418 + 1613 1 2419 2420 + 1614 1 2419 2421 + 1615 1 2422 2423 + 1616 1 2422 2424 + 1617 1 2425 2426 + 1618 1 2425 2427 + 1619 1 2428 2429 + 1620 1 2428 2430 + 1621 1 2431 2432 + 1622 1 2431 2433 + 1623 1 2434 2435 + 1624 1 2434 2436 + 1625 1 2437 2438 + 1626 1 2437 2439 + 1627 1 2440 2441 + 1628 1 2440 2442 + 1629 1 2443 2444 + 1630 1 2443 2445 + 1631 1 2446 2447 + 1632 1 2446 2448 + 1633 1 2449 2450 + 1634 1 2449 2451 + 1635 1 2452 2453 + 1636 1 2452 2454 + 1637 1 2455 2456 + 1638 1 2455 2457 + 1639 1 2458 2459 + 1640 1 2458 2460 + 1641 1 2461 2462 + 1642 1 2461 2463 + 1643 1 2464 2465 + 1644 1 2464 2466 + 1645 1 2467 2468 + 1646 1 2467 2469 + 1647 1 2470 2471 + 1648 1 2470 2472 + 1649 1 2473 2474 + 1650 1 2473 2475 + 1651 1 2476 2477 + 1652 1 2476 2478 + 1653 1 2479 2480 + 1654 1 2479 2481 + 1655 1 2482 2483 + 1656 1 2482 2484 + 1657 1 2485 2486 + 1658 1 2485 2487 + 1659 1 2488 2489 + 1660 1 2488 2490 + 1661 1 2491 2492 + 1662 1 2491 2493 + 1663 1 2494 2495 + 1664 1 2494 2496 + 1665 1 2497 2498 + 1666 1 2497 2499 + 1667 1 2500 2501 + 1668 1 2500 2502 + 1669 1 2503 2504 + 1670 1 2503 2505 + 1671 1 2506 2507 + 1672 1 2506 2508 + 1673 1 2509 2510 + 1674 1 2509 2511 + 1675 1 2512 2513 + 1676 1 2512 2514 + 1677 1 2515 2516 + 1678 1 2515 2517 + 1679 1 2518 2519 + 1680 1 2518 2520 + 1681 1 2521 2522 + 1682 1 2521 2523 + 1683 1 2524 2525 + 1684 1 2524 2526 + 1685 1 2527 2528 + 1686 1 2527 2529 + 1687 1 2530 2531 + 1688 1 2530 2532 + 1689 1 2533 2534 + 1690 1 2533 2535 + 1691 1 2536 2537 + 1692 1 2536 2538 + 1693 1 2539 2540 + 1694 1 2539 2541 + 1695 1 2542 2543 + 1696 1 2542 2544 + 1697 1 2545 2546 + 1698 1 2545 2547 + 1699 1 2548 2549 + 1700 1 2548 2550 + 1701 1 2551 2552 + 1702 1 2551 2553 + 1703 1 2554 2555 + 1704 1 2554 2556 + 1705 1 2557 2558 + 1706 1 2557 2559 + 1707 1 2560 2561 + 1708 1 2560 2562 + 1709 1 2563 2564 + 1710 1 2563 2565 + 1711 1 2566 2567 + 1712 1 2566 2568 + 1713 1 2569 2570 + 1714 1 2569 2571 + 1715 1 2572 2573 + 1716 1 2572 2574 + 1717 1 2575 2576 + 1718 1 2575 2577 + 1719 1 2578 2579 + 1720 1 2578 2580 + 1721 1 2581 2582 + 1722 1 2581 2583 + 1723 1 2584 2585 + 1724 1 2584 2586 + 1725 1 2587 2588 + 1726 1 2587 2589 + 1727 1 2590 2591 + 1728 1 2590 2592 + 1729 1 2593 2594 + 1730 1 2593 2595 + 1731 1 2596 2597 + 1732 1 2596 2598 + 1733 1 2599 2600 + 1734 1 2599 2601 + 1735 1 2602 2603 + 1736 1 2602 2604 + 1737 1 2605 2606 + 1738 1 2605 2607 + 1739 1 2608 2609 + 1740 1 2608 2610 + 1741 1 2611 2612 + 1742 1 2611 2613 + 1743 1 2614 2615 + 1744 1 2614 2616 + 1745 1 2617 2618 + 1746 1 2617 2619 + 1747 1 2620 2621 + 1748 1 2620 2622 + 1749 1 2623 2624 + 1750 1 2623 2625 + 1751 1 2626 2627 + 1752 1 2626 2628 + 1753 1 2629 2630 + 1754 1 2629 2631 + 1755 1 2632 2633 + 1756 1 2632 2634 + 1757 1 2635 2636 + 1758 1 2635 2637 + 1759 1 2638 2639 + 1760 1 2638 2640 + 1761 1 2641 2642 + 1762 1 2641 2643 + 1763 1 2644 2645 + 1764 1 2644 2646 + 1765 1 2647 2648 + 1766 1 2647 2649 + 1767 1 2650 2651 + 1768 1 2650 2652 + 1769 1 2653 2654 + 1770 1 2653 2655 + 1771 1 2656 2657 + 1772 1 2656 2658 + 1773 1 2659 2660 + 1774 1 2659 2661 + 1775 1 2662 2663 + 1776 1 2662 2664 + 1777 1 2665 2666 + 1778 1 2665 2667 + 1779 1 2668 2669 + 1780 1 2668 2670 + 1781 1 2671 2672 + 1782 1 2671 2673 + 1783 1 2674 2675 + 1784 1 2674 2676 + 1785 1 2677 2678 + 1786 1 2677 2679 + 1787 1 2680 2681 + 1788 1 2680 2682 + 1789 1 2683 2684 + 1790 1 2683 2685 + 1791 1 2686 2687 + 1792 1 2686 2688 + 1793 1 2689 2690 + 1794 1 2689 2691 + 1795 1 2692 2693 + 1796 1 2692 2694 + 1797 1 2695 2696 + 1798 1 2695 2697 + 1799 1 2698 2699 + 1800 1 2698 2700 + 1801 1 2701 2702 + 1802 1 2701 2703 + 1803 1 2704 2705 + 1804 1 2704 2706 + 1805 1 2707 2708 + 1806 1 2707 2709 + 1807 1 2710 2711 + 1808 1 2710 2712 + 1809 1 2713 2714 + 1810 1 2713 2715 + 1811 1 2716 2717 + 1812 1 2716 2718 + 1813 1 2719 2720 + 1814 1 2719 2721 + 1815 1 2722 2723 + 1816 1 2722 2724 + 1817 1 2725 2726 + 1818 1 2725 2727 + 1819 1 2728 2729 + 1820 1 2728 2730 + 1821 1 2731 2732 + 1822 1 2731 2733 + 1823 1 2734 2735 + 1824 1 2734 2736 + 1825 1 2737 2738 + 1826 1 2737 2739 + 1827 1 2740 2741 + 1828 1 2740 2742 + 1829 1 2743 2744 + 1830 1 2743 2745 + 1831 1 2746 2747 + 1832 1 2746 2748 + 1833 1 2749 2750 + 1834 1 2749 2751 + 1835 1 2752 2753 + 1836 1 2752 2754 + 1837 1 2755 2756 + 1838 1 2755 2757 + 1839 1 2758 2759 + 1840 1 2758 2760 + 1841 1 2761 2762 + 1842 1 2761 2763 + 1843 1 2764 2765 + 1844 1 2764 2766 + 1845 1 2767 2768 + 1846 1 2767 2769 + 1847 1 2770 2771 + 1848 1 2770 2772 + 1849 1 2773 2774 + 1850 1 2773 2775 + 1851 1 2776 2777 + 1852 1 2776 2778 + 1853 1 2779 2780 + 1854 1 2779 2781 + 1855 1 2782 2783 + 1856 1 2782 2784 + 1857 1 2785 2786 + 1858 1 2785 2787 + 1859 1 2788 2789 + 1860 1 2788 2790 + 1861 1 2791 2792 + 1862 1 2791 2793 + 1863 1 2794 2795 + 1864 1 2794 2796 + 1865 1 2797 2798 + 1866 1 2797 2799 + 1867 1 2800 2801 + 1868 1 2800 2802 + 1869 1 2803 2804 + 1870 1 2803 2805 + 1871 1 2806 2807 + 1872 1 2806 2808 + 1873 1 2809 2810 + 1874 1 2809 2811 + 1875 1 2812 2813 + 1876 1 2812 2814 + 1877 1 2815 2816 + 1878 1 2815 2817 + 1879 1 2818 2819 + 1880 1 2818 2820 + 1881 1 2821 2822 + 1882 1 2821 2823 + 1883 1 2824 2825 + 1884 1 2824 2826 + 1885 1 2827 2828 + 1886 1 2827 2829 + 1887 1 2830 2831 + 1888 1 2830 2832 + 1889 1 2833 2834 + 1890 1 2833 2835 + 1891 1 2836 2837 + 1892 1 2836 2838 + 1893 1 2839 2840 + 1894 1 2839 2841 + 1895 1 2842 2843 + 1896 1 2842 2844 + 1897 1 2845 2846 + 1898 1 2845 2847 + 1899 1 2848 2849 + 1900 1 2848 2850 + 1901 1 2851 2852 + 1902 1 2851 2853 + 1903 1 2854 2855 + 1904 1 2854 2856 + 1905 1 2857 2858 + 1906 1 2857 2859 + 1907 1 2860 2861 + 1908 1 2860 2862 + 1909 1 2863 2864 + 1910 1 2863 2865 + 1911 1 2866 2867 + 1912 1 2866 2868 + 1913 1 2869 2870 + 1914 1 2869 2871 + 1915 1 2872 2873 + 1916 1 2872 2874 + 1917 1 2875 2876 + 1918 1 2875 2877 + 1919 1 2878 2879 + 1920 1 2878 2880 + 1921 1 2881 2882 + 1922 1 2881 2883 + 1923 1 2884 2885 + 1924 1 2884 2886 + 1925 1 2887 2888 + 1926 1 2887 2889 + 1927 1 2890 2891 + 1928 1 2890 2892 + 1929 1 2893 2894 + 1930 1 2893 2895 + 1931 1 2896 2897 + 1932 1 2896 2898 + 1933 1 2899 2900 + 1934 1 2899 2901 + 1935 1 2902 2903 + 1936 1 2902 2904 + 1937 1 2905 2906 + 1938 1 2905 2907 + 1939 1 2908 2909 + 1940 1 2908 2910 + 1941 1 2911 2912 + 1942 1 2911 2913 + 1943 1 2914 2915 + 1944 1 2914 2916 + 1945 1 2917 2918 + 1946 1 2917 2919 + 1947 1 2920 2921 + 1948 1 2920 2922 + 1949 1 2923 2924 + 1950 1 2923 2925 + 1951 1 2926 2927 + 1952 1 2926 2928 + 1953 1 2929 2930 + 1954 1 2929 2931 + 1955 1 2932 2933 + 1956 1 2932 2934 + 1957 1 2935 2936 + 1958 1 2935 2937 + 1959 1 2938 2939 + 1960 1 2938 2940 + 1961 1 2941 2942 + 1962 1 2941 2943 + 1963 1 2944 2945 + 1964 1 2944 2946 + 1965 1 2947 2948 + 1966 1 2947 2949 + 1967 1 2950 2951 + 1968 1 2950 2952 + 1969 1 2953 2954 + 1970 1 2953 2955 + 1971 1 2956 2957 + 1972 1 2956 2958 + 1973 1 2959 2960 + 1974 1 2959 2961 + 1975 1 2962 2963 + 1976 1 2962 2964 + 1977 1 2965 2966 + 1978 1 2965 2967 + 1979 1 2968 2969 + 1980 1 2968 2970 + 1981 1 2971 2972 + 1982 1 2971 2973 + 1983 1 2974 2975 + 1984 1 2974 2976 + 1985 1 2977 2978 + 1986 1 2977 2979 + 1987 1 2980 2981 + 1988 1 2980 2982 + 1989 1 2983 2984 + 1990 1 2983 2985 + 1991 1 2986 2987 + 1992 1 2986 2988 + 1993 1 2989 2990 + 1994 1 2989 2991 + 1995 1 2992 2993 + 1996 1 2992 2994 + 1997 1 2995 2996 + 1998 1 2995 2997 + 1999 1 2998 2999 + 2000 1 2998 3000 + +Angles + + 1 1 2 1 3 + 2 1 5 4 6 + 3 1 8 7 9 + 4 1 11 10 12 + 5 1 14 13 15 + 6 1 17 16 18 + 7 1 20 19 21 + 8 1 23 22 24 + 9 1 26 25 27 + 10 1 29 28 30 + 11 1 32 31 33 + 12 1 35 34 36 + 13 1 38 37 39 + 14 1 41 40 42 + 15 1 44 43 45 + 16 1 47 46 48 + 17 1 50 49 51 + 18 1 53 52 54 + 19 1 56 55 57 + 20 1 59 58 60 + 21 1 62 61 63 + 22 1 65 64 66 + 23 1 68 67 69 + 24 1 71 70 72 + 25 1 74 73 75 + 26 1 77 76 78 + 27 1 80 79 81 + 28 1 83 82 84 + 29 1 86 85 87 + 30 1 89 88 90 + 31 1 92 91 93 + 32 1 95 94 96 + 33 1 98 97 99 + 34 1 101 100 102 + 35 1 104 103 105 + 36 1 107 106 108 + 37 1 110 109 111 + 38 1 113 112 114 + 39 1 116 115 117 + 40 1 119 118 120 + 41 1 122 121 123 + 42 1 125 124 126 + 43 1 128 127 129 + 44 1 131 130 132 + 45 1 134 133 135 + 46 1 137 136 138 + 47 1 140 139 141 + 48 1 143 142 144 + 49 1 146 145 147 + 50 1 149 148 150 + 51 1 152 151 153 + 52 1 155 154 156 + 53 1 158 157 159 + 54 1 161 160 162 + 55 1 164 163 165 + 56 1 167 166 168 + 57 1 170 169 171 + 58 1 173 172 174 + 59 1 176 175 177 + 60 1 179 178 180 + 61 1 182 181 183 + 62 1 185 184 186 + 63 1 188 187 189 + 64 1 191 190 192 + 65 1 194 193 195 + 66 1 197 196 198 + 67 1 200 199 201 + 68 1 203 202 204 + 69 1 206 205 207 + 70 1 209 208 210 + 71 1 212 211 213 + 72 1 215 214 216 + 73 1 218 217 219 + 74 1 221 220 222 + 75 1 224 223 225 + 76 1 227 226 228 + 77 1 230 229 231 + 78 1 233 232 234 + 79 1 236 235 237 + 80 1 239 238 240 + 81 1 242 241 243 + 82 1 245 244 246 + 83 1 248 247 249 + 84 1 251 250 252 + 85 1 254 253 255 + 86 1 257 256 258 + 87 1 260 259 261 + 88 1 263 262 264 + 89 1 266 265 267 + 90 1 269 268 270 + 91 1 272 271 273 + 92 1 275 274 276 + 93 1 278 277 279 + 94 1 281 280 282 + 95 1 284 283 285 + 96 1 287 286 288 + 97 1 290 289 291 + 98 1 293 292 294 + 99 1 296 295 297 + 100 1 299 298 300 + 101 1 302 301 303 + 102 1 305 304 306 + 103 1 308 307 309 + 104 1 311 310 312 + 105 1 314 313 315 + 106 1 317 316 318 + 107 1 320 319 321 + 108 1 323 322 324 + 109 1 326 325 327 + 110 1 329 328 330 + 111 1 332 331 333 + 112 1 335 334 336 + 113 1 338 337 339 + 114 1 341 340 342 + 115 1 344 343 345 + 116 1 347 346 348 + 117 1 350 349 351 + 118 1 353 352 354 + 119 1 356 355 357 + 120 1 359 358 360 + 121 1 362 361 363 + 122 1 365 364 366 + 123 1 368 367 369 + 124 1 371 370 372 + 125 1 374 373 375 + 126 1 377 376 378 + 127 1 380 379 381 + 128 1 383 382 384 + 129 1 386 385 387 + 130 1 389 388 390 + 131 1 392 391 393 + 132 1 395 394 396 + 133 1 398 397 399 + 134 1 401 400 402 + 135 1 404 403 405 + 136 1 407 406 408 + 137 1 410 409 411 + 138 1 413 412 414 + 139 1 416 415 417 + 140 1 419 418 420 + 141 1 422 421 423 + 142 1 425 424 426 + 143 1 428 427 429 + 144 1 431 430 432 + 145 1 434 433 435 + 146 1 437 436 438 + 147 1 440 439 441 + 148 1 443 442 444 + 149 1 446 445 447 + 150 1 449 448 450 + 151 1 452 451 453 + 152 1 455 454 456 + 153 1 458 457 459 + 154 1 461 460 462 + 155 1 464 463 465 + 156 1 467 466 468 + 157 1 470 469 471 + 158 1 473 472 474 + 159 1 476 475 477 + 160 1 479 478 480 + 161 1 482 481 483 + 162 1 485 484 486 + 163 1 488 487 489 + 164 1 491 490 492 + 165 1 494 493 495 + 166 1 497 496 498 + 167 1 500 499 501 + 168 1 503 502 504 + 169 1 506 505 507 + 170 1 509 508 510 + 171 1 512 511 513 + 172 1 515 514 516 + 173 1 518 517 519 + 174 1 521 520 522 + 175 1 524 523 525 + 176 1 527 526 528 + 177 1 530 529 531 + 178 1 533 532 534 + 179 1 536 535 537 + 180 1 539 538 540 + 181 1 542 541 543 + 182 1 545 544 546 + 183 1 548 547 549 + 184 1 551 550 552 + 185 1 554 553 555 + 186 1 557 556 558 + 187 1 560 559 561 + 188 1 563 562 564 + 189 1 566 565 567 + 190 1 569 568 570 + 191 1 572 571 573 + 192 1 575 574 576 + 193 1 578 577 579 + 194 1 581 580 582 + 195 1 584 583 585 + 196 1 587 586 588 + 197 1 590 589 591 + 198 1 593 592 594 + 199 1 596 595 597 + 200 1 599 598 600 + 201 1 602 601 603 + 202 1 605 604 606 + 203 1 608 607 609 + 204 1 611 610 612 + 205 1 614 613 615 + 206 1 617 616 618 + 207 1 620 619 621 + 208 1 623 622 624 + 209 1 626 625 627 + 210 1 629 628 630 + 211 1 632 631 633 + 212 1 635 634 636 + 213 1 638 637 639 + 214 1 641 640 642 + 215 1 644 643 645 + 216 1 647 646 648 + 217 1 650 649 651 + 218 1 653 652 654 + 219 1 656 655 657 + 220 1 659 658 660 + 221 1 662 661 663 + 222 1 665 664 666 + 223 1 668 667 669 + 224 1 671 670 672 + 225 1 674 673 675 + 226 1 677 676 678 + 227 1 680 679 681 + 228 1 683 682 684 + 229 1 686 685 687 + 230 1 689 688 690 + 231 1 692 691 693 + 232 1 695 694 696 + 233 1 698 697 699 + 234 1 701 700 702 + 235 1 704 703 705 + 236 1 707 706 708 + 237 1 710 709 711 + 238 1 713 712 714 + 239 1 716 715 717 + 240 1 719 718 720 + 241 1 722 721 723 + 242 1 725 724 726 + 243 1 728 727 729 + 244 1 731 730 732 + 245 1 734 733 735 + 246 1 737 736 738 + 247 1 740 739 741 + 248 1 743 742 744 + 249 1 746 745 747 + 250 1 749 748 750 + 251 1 752 751 753 + 252 1 755 754 756 + 253 1 758 757 759 + 254 1 761 760 762 + 255 1 764 763 765 + 256 1 767 766 768 + 257 1 770 769 771 + 258 1 773 772 774 + 259 1 776 775 777 + 260 1 779 778 780 + 261 1 782 781 783 + 262 1 785 784 786 + 263 1 788 787 789 + 264 1 791 790 792 + 265 1 794 793 795 + 266 1 797 796 798 + 267 1 800 799 801 + 268 1 803 802 804 + 269 1 806 805 807 + 270 1 809 808 810 + 271 1 812 811 813 + 272 1 815 814 816 + 273 1 818 817 819 + 274 1 821 820 822 + 275 1 824 823 825 + 276 1 827 826 828 + 277 1 830 829 831 + 278 1 833 832 834 + 279 1 836 835 837 + 280 1 839 838 840 + 281 1 842 841 843 + 282 1 845 844 846 + 283 1 848 847 849 + 284 1 851 850 852 + 285 1 854 853 855 + 286 1 857 856 858 + 287 1 860 859 861 + 288 1 863 862 864 + 289 1 866 865 867 + 290 1 869 868 870 + 291 1 872 871 873 + 292 1 875 874 876 + 293 1 878 877 879 + 294 1 881 880 882 + 295 1 884 883 885 + 296 1 887 886 888 + 297 1 890 889 891 + 298 1 893 892 894 + 299 1 896 895 897 + 300 1 899 898 900 + 301 1 902 901 903 + 302 1 905 904 906 + 303 1 908 907 909 + 304 1 911 910 912 + 305 1 914 913 915 + 306 1 917 916 918 + 307 1 920 919 921 + 308 1 923 922 924 + 309 1 926 925 927 + 310 1 929 928 930 + 311 1 932 931 933 + 312 1 935 934 936 + 313 1 938 937 939 + 314 1 941 940 942 + 315 1 944 943 945 + 316 1 947 946 948 + 317 1 950 949 951 + 318 1 953 952 954 + 319 1 956 955 957 + 320 1 959 958 960 + 321 1 962 961 963 + 322 1 965 964 966 + 323 1 968 967 969 + 324 1 971 970 972 + 325 1 974 973 975 + 326 1 977 976 978 + 327 1 980 979 981 + 328 1 983 982 984 + 329 1 986 985 987 + 330 1 989 988 990 + 331 1 992 991 993 + 332 1 995 994 996 + 333 1 998 997 999 + 334 1 1001 1000 1002 + 335 1 1004 1003 1005 + 336 1 1007 1006 1008 + 337 1 1010 1009 1011 + 338 1 1013 1012 1014 + 339 1 1016 1015 1017 + 340 1 1019 1018 1020 + 341 1 1022 1021 1023 + 342 1 1025 1024 1026 + 343 1 1028 1027 1029 + 344 1 1031 1030 1032 + 345 1 1034 1033 1035 + 346 1 1037 1036 1038 + 347 1 1040 1039 1041 + 348 1 1043 1042 1044 + 349 1 1046 1045 1047 + 350 1 1049 1048 1050 + 351 1 1052 1051 1053 + 352 1 1055 1054 1056 + 353 1 1058 1057 1059 + 354 1 1061 1060 1062 + 355 1 1064 1063 1065 + 356 1 1067 1066 1068 + 357 1 1070 1069 1071 + 358 1 1073 1072 1074 + 359 1 1076 1075 1077 + 360 1 1079 1078 1080 + 361 1 1082 1081 1083 + 362 1 1085 1084 1086 + 363 1 1088 1087 1089 + 364 1 1091 1090 1092 + 365 1 1094 1093 1095 + 366 1 1097 1096 1098 + 367 1 1100 1099 1101 + 368 1 1103 1102 1104 + 369 1 1106 1105 1107 + 370 1 1109 1108 1110 + 371 1 1112 1111 1113 + 372 1 1115 1114 1116 + 373 1 1118 1117 1119 + 374 1 1121 1120 1122 + 375 1 1124 1123 1125 + 376 1 1127 1126 1128 + 377 1 1130 1129 1131 + 378 1 1133 1132 1134 + 379 1 1136 1135 1137 + 380 1 1139 1138 1140 + 381 1 1142 1141 1143 + 382 1 1145 1144 1146 + 383 1 1148 1147 1149 + 384 1 1151 1150 1152 + 385 1 1154 1153 1155 + 386 1 1157 1156 1158 + 387 1 1160 1159 1161 + 388 1 1163 1162 1164 + 389 1 1166 1165 1167 + 390 1 1169 1168 1170 + 391 1 1172 1171 1173 + 392 1 1175 1174 1176 + 393 1 1178 1177 1179 + 394 1 1181 1180 1182 + 395 1 1184 1183 1185 + 396 1 1187 1186 1188 + 397 1 1190 1189 1191 + 398 1 1193 1192 1194 + 399 1 1196 1195 1197 + 400 1 1199 1198 1200 + 401 1 1202 1201 1203 + 402 1 1205 1204 1206 + 403 1 1208 1207 1209 + 404 1 1211 1210 1212 + 405 1 1214 1213 1215 + 406 1 1217 1216 1218 + 407 1 1220 1219 1221 + 408 1 1223 1222 1224 + 409 1 1226 1225 1227 + 410 1 1229 1228 1230 + 411 1 1232 1231 1233 + 412 1 1235 1234 1236 + 413 1 1238 1237 1239 + 414 1 1241 1240 1242 + 415 1 1244 1243 1245 + 416 1 1247 1246 1248 + 417 1 1250 1249 1251 + 418 1 1253 1252 1254 + 419 1 1256 1255 1257 + 420 1 1259 1258 1260 + 421 1 1262 1261 1263 + 422 1 1265 1264 1266 + 423 1 1268 1267 1269 + 424 1 1271 1270 1272 + 425 1 1274 1273 1275 + 426 1 1277 1276 1278 + 427 1 1280 1279 1281 + 428 1 1283 1282 1284 + 429 1 1286 1285 1287 + 430 1 1289 1288 1290 + 431 1 1292 1291 1293 + 432 1 1295 1294 1296 + 433 1 1298 1297 1299 + 434 1 1301 1300 1302 + 435 1 1304 1303 1305 + 436 1 1307 1306 1308 + 437 1 1310 1309 1311 + 438 1 1313 1312 1314 + 439 1 1316 1315 1317 + 440 1 1319 1318 1320 + 441 1 1322 1321 1323 + 442 1 1325 1324 1326 + 443 1 1328 1327 1329 + 444 1 1331 1330 1332 + 445 1 1334 1333 1335 + 446 1 1337 1336 1338 + 447 1 1340 1339 1341 + 448 1 1343 1342 1344 + 449 1 1346 1345 1347 + 450 1 1349 1348 1350 + 451 1 1352 1351 1353 + 452 1 1355 1354 1356 + 453 1 1358 1357 1359 + 454 1 1361 1360 1362 + 455 1 1364 1363 1365 + 456 1 1367 1366 1368 + 457 1 1370 1369 1371 + 458 1 1373 1372 1374 + 459 1 1376 1375 1377 + 460 1 1379 1378 1380 + 461 1 1382 1381 1383 + 462 1 1385 1384 1386 + 463 1 1388 1387 1389 + 464 1 1391 1390 1392 + 465 1 1394 1393 1395 + 466 1 1397 1396 1398 + 467 1 1400 1399 1401 + 468 1 1403 1402 1404 + 469 1 1406 1405 1407 + 470 1 1409 1408 1410 + 471 1 1412 1411 1413 + 472 1 1415 1414 1416 + 473 1 1418 1417 1419 + 474 1 1421 1420 1422 + 475 1 1424 1423 1425 + 476 1 1427 1426 1428 + 477 1 1430 1429 1431 + 478 1 1433 1432 1434 + 479 1 1436 1435 1437 + 480 1 1439 1438 1440 + 481 1 1442 1441 1443 + 482 1 1445 1444 1446 + 483 1 1448 1447 1449 + 484 1 1451 1450 1452 + 485 1 1454 1453 1455 + 486 1 1457 1456 1458 + 487 1 1460 1459 1461 + 488 1 1463 1462 1464 + 489 1 1466 1465 1467 + 490 1 1469 1468 1470 + 491 1 1472 1471 1473 + 492 1 1475 1474 1476 + 493 1 1478 1477 1479 + 494 1 1481 1480 1482 + 495 1 1484 1483 1485 + 496 1 1487 1486 1488 + 497 1 1490 1489 1491 + 498 1 1493 1492 1494 + 499 1 1496 1495 1497 + 500 1 1499 1498 1500 + 501 1 1502 1501 1503 + 502 1 1505 1504 1506 + 503 1 1508 1507 1509 + 504 1 1511 1510 1512 + 505 1 1514 1513 1515 + 506 1 1517 1516 1518 + 507 1 1520 1519 1521 + 508 1 1523 1522 1524 + 509 1 1526 1525 1527 + 510 1 1529 1528 1530 + 511 1 1532 1531 1533 + 512 1 1535 1534 1536 + 513 1 1538 1537 1539 + 514 1 1541 1540 1542 + 515 1 1544 1543 1545 + 516 1 1547 1546 1548 + 517 1 1550 1549 1551 + 518 1 1553 1552 1554 + 519 1 1556 1555 1557 + 520 1 1559 1558 1560 + 521 1 1562 1561 1563 + 522 1 1565 1564 1566 + 523 1 1568 1567 1569 + 524 1 1571 1570 1572 + 525 1 1574 1573 1575 + 526 1 1577 1576 1578 + 527 1 1580 1579 1581 + 528 1 1583 1582 1584 + 529 1 1586 1585 1587 + 530 1 1589 1588 1590 + 531 1 1592 1591 1593 + 532 1 1595 1594 1596 + 533 1 1598 1597 1599 + 534 1 1601 1600 1602 + 535 1 1604 1603 1605 + 536 1 1607 1606 1608 + 537 1 1610 1609 1611 + 538 1 1613 1612 1614 + 539 1 1616 1615 1617 + 540 1 1619 1618 1620 + 541 1 1622 1621 1623 + 542 1 1625 1624 1626 + 543 1 1628 1627 1629 + 544 1 1631 1630 1632 + 545 1 1634 1633 1635 + 546 1 1637 1636 1638 + 547 1 1640 1639 1641 + 548 1 1643 1642 1644 + 549 1 1646 1645 1647 + 550 1 1649 1648 1650 + 551 1 1652 1651 1653 + 552 1 1655 1654 1656 + 553 1 1658 1657 1659 + 554 1 1661 1660 1662 + 555 1 1664 1663 1665 + 556 1 1667 1666 1668 + 557 1 1670 1669 1671 + 558 1 1673 1672 1674 + 559 1 1676 1675 1677 + 560 1 1679 1678 1680 + 561 1 1682 1681 1683 + 562 1 1685 1684 1686 + 563 1 1688 1687 1689 + 564 1 1691 1690 1692 + 565 1 1694 1693 1695 + 566 1 1697 1696 1698 + 567 1 1700 1699 1701 + 568 1 1703 1702 1704 + 569 1 1706 1705 1707 + 570 1 1709 1708 1710 + 571 1 1712 1711 1713 + 572 1 1715 1714 1716 + 573 1 1718 1717 1719 + 574 1 1721 1720 1722 + 575 1 1724 1723 1725 + 576 1 1727 1726 1728 + 577 1 1730 1729 1731 + 578 1 1733 1732 1734 + 579 1 1736 1735 1737 + 580 1 1739 1738 1740 + 581 1 1742 1741 1743 + 582 1 1745 1744 1746 + 583 1 1748 1747 1749 + 584 1 1751 1750 1752 + 585 1 1754 1753 1755 + 586 1 1757 1756 1758 + 587 1 1760 1759 1761 + 588 1 1763 1762 1764 + 589 1 1766 1765 1767 + 590 1 1769 1768 1770 + 591 1 1772 1771 1773 + 592 1 1775 1774 1776 + 593 1 1778 1777 1779 + 594 1 1781 1780 1782 + 595 1 1784 1783 1785 + 596 1 1787 1786 1788 + 597 1 1790 1789 1791 + 598 1 1793 1792 1794 + 599 1 1796 1795 1797 + 600 1 1799 1798 1800 + 601 1 1802 1801 1803 + 602 1 1805 1804 1806 + 603 1 1808 1807 1809 + 604 1 1811 1810 1812 + 605 1 1814 1813 1815 + 606 1 1817 1816 1818 + 607 1 1820 1819 1821 + 608 1 1823 1822 1824 + 609 1 1826 1825 1827 + 610 1 1829 1828 1830 + 611 1 1832 1831 1833 + 612 1 1835 1834 1836 + 613 1 1838 1837 1839 + 614 1 1841 1840 1842 + 615 1 1844 1843 1845 + 616 1 1847 1846 1848 + 617 1 1850 1849 1851 + 618 1 1853 1852 1854 + 619 1 1856 1855 1857 + 620 1 1859 1858 1860 + 621 1 1862 1861 1863 + 622 1 1865 1864 1866 + 623 1 1868 1867 1869 + 624 1 1871 1870 1872 + 625 1 1874 1873 1875 + 626 1 1877 1876 1878 + 627 1 1880 1879 1881 + 628 1 1883 1882 1884 + 629 1 1886 1885 1887 + 630 1 1889 1888 1890 + 631 1 1892 1891 1893 + 632 1 1895 1894 1896 + 633 1 1898 1897 1899 + 634 1 1901 1900 1902 + 635 1 1904 1903 1905 + 636 1 1907 1906 1908 + 637 1 1910 1909 1911 + 638 1 1913 1912 1914 + 639 1 1916 1915 1917 + 640 1 1919 1918 1920 + 641 1 1922 1921 1923 + 642 1 1925 1924 1926 + 643 1 1928 1927 1929 + 644 1 1931 1930 1932 + 645 1 1934 1933 1935 + 646 1 1937 1936 1938 + 647 1 1940 1939 1941 + 648 1 1943 1942 1944 + 649 1 1946 1945 1947 + 650 1 1949 1948 1950 + 651 1 1952 1951 1953 + 652 1 1955 1954 1956 + 653 1 1958 1957 1959 + 654 1 1961 1960 1962 + 655 1 1964 1963 1965 + 656 1 1967 1966 1968 + 657 1 1970 1969 1971 + 658 1 1973 1972 1974 + 659 1 1976 1975 1977 + 660 1 1979 1978 1980 + 661 1 1982 1981 1983 + 662 1 1985 1984 1986 + 663 1 1988 1987 1989 + 664 1 1991 1990 1992 + 665 1 1994 1993 1995 + 666 1 1997 1996 1998 + 667 1 2000 1999 2001 + 668 1 2003 2002 2004 + 669 1 2006 2005 2007 + 670 1 2009 2008 2010 + 671 1 2012 2011 2013 + 672 1 2015 2014 2016 + 673 1 2018 2017 2019 + 674 1 2021 2020 2022 + 675 1 2024 2023 2025 + 676 1 2027 2026 2028 + 677 1 2030 2029 2031 + 678 1 2033 2032 2034 + 679 1 2036 2035 2037 + 680 1 2039 2038 2040 + 681 1 2042 2041 2043 + 682 1 2045 2044 2046 + 683 1 2048 2047 2049 + 684 1 2051 2050 2052 + 685 1 2054 2053 2055 + 686 1 2057 2056 2058 + 687 1 2060 2059 2061 + 688 1 2063 2062 2064 + 689 1 2066 2065 2067 + 690 1 2069 2068 2070 + 691 1 2072 2071 2073 + 692 1 2075 2074 2076 + 693 1 2078 2077 2079 + 694 1 2081 2080 2082 + 695 1 2084 2083 2085 + 696 1 2087 2086 2088 + 697 1 2090 2089 2091 + 698 1 2093 2092 2094 + 699 1 2096 2095 2097 + 700 1 2099 2098 2100 + 701 1 2102 2101 2103 + 702 1 2105 2104 2106 + 703 1 2108 2107 2109 + 704 1 2111 2110 2112 + 705 1 2114 2113 2115 + 706 1 2117 2116 2118 + 707 1 2120 2119 2121 + 708 1 2123 2122 2124 + 709 1 2126 2125 2127 + 710 1 2129 2128 2130 + 711 1 2132 2131 2133 + 712 1 2135 2134 2136 + 713 1 2138 2137 2139 + 714 1 2141 2140 2142 + 715 1 2144 2143 2145 + 716 1 2147 2146 2148 + 717 1 2150 2149 2151 + 718 1 2153 2152 2154 + 719 1 2156 2155 2157 + 720 1 2159 2158 2160 + 721 1 2162 2161 2163 + 722 1 2165 2164 2166 + 723 1 2168 2167 2169 + 724 1 2171 2170 2172 + 725 1 2174 2173 2175 + 726 1 2177 2176 2178 + 727 1 2180 2179 2181 + 728 1 2183 2182 2184 + 729 1 2186 2185 2187 + 730 1 2189 2188 2190 + 731 1 2192 2191 2193 + 732 1 2195 2194 2196 + 733 1 2198 2197 2199 + 734 1 2201 2200 2202 + 735 1 2204 2203 2205 + 736 1 2207 2206 2208 + 737 1 2210 2209 2211 + 738 1 2213 2212 2214 + 739 1 2216 2215 2217 + 740 1 2219 2218 2220 + 741 1 2222 2221 2223 + 742 1 2225 2224 2226 + 743 1 2228 2227 2229 + 744 1 2231 2230 2232 + 745 1 2234 2233 2235 + 746 1 2237 2236 2238 + 747 1 2240 2239 2241 + 748 1 2243 2242 2244 + 749 1 2246 2245 2247 + 750 1 2249 2248 2250 + 751 1 2252 2251 2253 + 752 1 2255 2254 2256 + 753 1 2258 2257 2259 + 754 1 2261 2260 2262 + 755 1 2264 2263 2265 + 756 1 2267 2266 2268 + 757 1 2270 2269 2271 + 758 1 2273 2272 2274 + 759 1 2276 2275 2277 + 760 1 2279 2278 2280 + 761 1 2282 2281 2283 + 762 1 2285 2284 2286 + 763 1 2288 2287 2289 + 764 1 2291 2290 2292 + 765 1 2294 2293 2295 + 766 1 2297 2296 2298 + 767 1 2300 2299 2301 + 768 1 2303 2302 2304 + 769 1 2306 2305 2307 + 770 1 2309 2308 2310 + 771 1 2312 2311 2313 + 772 1 2315 2314 2316 + 773 1 2318 2317 2319 + 774 1 2321 2320 2322 + 775 1 2324 2323 2325 + 776 1 2327 2326 2328 + 777 1 2330 2329 2331 + 778 1 2333 2332 2334 + 779 1 2336 2335 2337 + 780 1 2339 2338 2340 + 781 1 2342 2341 2343 + 782 1 2345 2344 2346 + 783 1 2348 2347 2349 + 784 1 2351 2350 2352 + 785 1 2354 2353 2355 + 786 1 2357 2356 2358 + 787 1 2360 2359 2361 + 788 1 2363 2362 2364 + 789 1 2366 2365 2367 + 790 1 2369 2368 2370 + 791 1 2372 2371 2373 + 792 1 2375 2374 2376 + 793 1 2378 2377 2379 + 794 1 2381 2380 2382 + 795 1 2384 2383 2385 + 796 1 2387 2386 2388 + 797 1 2390 2389 2391 + 798 1 2393 2392 2394 + 799 1 2396 2395 2397 + 800 1 2399 2398 2400 + 801 1 2402 2401 2403 + 802 1 2405 2404 2406 + 803 1 2408 2407 2409 + 804 1 2411 2410 2412 + 805 1 2414 2413 2415 + 806 1 2417 2416 2418 + 807 1 2420 2419 2421 + 808 1 2423 2422 2424 + 809 1 2426 2425 2427 + 810 1 2429 2428 2430 + 811 1 2432 2431 2433 + 812 1 2435 2434 2436 + 813 1 2438 2437 2439 + 814 1 2441 2440 2442 + 815 1 2444 2443 2445 + 816 1 2447 2446 2448 + 817 1 2450 2449 2451 + 818 1 2453 2452 2454 + 819 1 2456 2455 2457 + 820 1 2459 2458 2460 + 821 1 2462 2461 2463 + 822 1 2465 2464 2466 + 823 1 2468 2467 2469 + 824 1 2471 2470 2472 + 825 1 2474 2473 2475 + 826 1 2477 2476 2478 + 827 1 2480 2479 2481 + 828 1 2483 2482 2484 + 829 1 2486 2485 2487 + 830 1 2489 2488 2490 + 831 1 2492 2491 2493 + 832 1 2495 2494 2496 + 833 1 2498 2497 2499 + 834 1 2501 2500 2502 + 835 1 2504 2503 2505 + 836 1 2507 2506 2508 + 837 1 2510 2509 2511 + 838 1 2513 2512 2514 + 839 1 2516 2515 2517 + 840 1 2519 2518 2520 + 841 1 2522 2521 2523 + 842 1 2525 2524 2526 + 843 1 2528 2527 2529 + 844 1 2531 2530 2532 + 845 1 2534 2533 2535 + 846 1 2537 2536 2538 + 847 1 2540 2539 2541 + 848 1 2543 2542 2544 + 849 1 2546 2545 2547 + 850 1 2549 2548 2550 + 851 1 2552 2551 2553 + 852 1 2555 2554 2556 + 853 1 2558 2557 2559 + 854 1 2561 2560 2562 + 855 1 2564 2563 2565 + 856 1 2567 2566 2568 + 857 1 2570 2569 2571 + 858 1 2573 2572 2574 + 859 1 2576 2575 2577 + 860 1 2579 2578 2580 + 861 1 2582 2581 2583 + 862 1 2585 2584 2586 + 863 1 2588 2587 2589 + 864 1 2591 2590 2592 + 865 1 2594 2593 2595 + 866 1 2597 2596 2598 + 867 1 2600 2599 2601 + 868 1 2603 2602 2604 + 869 1 2606 2605 2607 + 870 1 2609 2608 2610 + 871 1 2612 2611 2613 + 872 1 2615 2614 2616 + 873 1 2618 2617 2619 + 874 1 2621 2620 2622 + 875 1 2624 2623 2625 + 876 1 2627 2626 2628 + 877 1 2630 2629 2631 + 878 1 2633 2632 2634 + 879 1 2636 2635 2637 + 880 1 2639 2638 2640 + 881 1 2642 2641 2643 + 882 1 2645 2644 2646 + 883 1 2648 2647 2649 + 884 1 2651 2650 2652 + 885 1 2654 2653 2655 + 886 1 2657 2656 2658 + 887 1 2660 2659 2661 + 888 1 2663 2662 2664 + 889 1 2666 2665 2667 + 890 1 2669 2668 2670 + 891 1 2672 2671 2673 + 892 1 2675 2674 2676 + 893 1 2678 2677 2679 + 894 1 2681 2680 2682 + 895 1 2684 2683 2685 + 896 1 2687 2686 2688 + 897 1 2690 2689 2691 + 898 1 2693 2692 2694 + 899 1 2696 2695 2697 + 900 1 2699 2698 2700 + 901 1 2702 2701 2703 + 902 1 2705 2704 2706 + 903 1 2708 2707 2709 + 904 1 2711 2710 2712 + 905 1 2714 2713 2715 + 906 1 2717 2716 2718 + 907 1 2720 2719 2721 + 908 1 2723 2722 2724 + 909 1 2726 2725 2727 + 910 1 2729 2728 2730 + 911 1 2732 2731 2733 + 912 1 2735 2734 2736 + 913 1 2738 2737 2739 + 914 1 2741 2740 2742 + 915 1 2744 2743 2745 + 916 1 2747 2746 2748 + 917 1 2750 2749 2751 + 918 1 2753 2752 2754 + 919 1 2756 2755 2757 + 920 1 2759 2758 2760 + 921 1 2762 2761 2763 + 922 1 2765 2764 2766 + 923 1 2768 2767 2769 + 924 1 2771 2770 2772 + 925 1 2774 2773 2775 + 926 1 2777 2776 2778 + 927 1 2780 2779 2781 + 928 1 2783 2782 2784 + 929 1 2786 2785 2787 + 930 1 2789 2788 2790 + 931 1 2792 2791 2793 + 932 1 2795 2794 2796 + 933 1 2798 2797 2799 + 934 1 2801 2800 2802 + 935 1 2804 2803 2805 + 936 1 2807 2806 2808 + 937 1 2810 2809 2811 + 938 1 2813 2812 2814 + 939 1 2816 2815 2817 + 940 1 2819 2818 2820 + 941 1 2822 2821 2823 + 942 1 2825 2824 2826 + 943 1 2828 2827 2829 + 944 1 2831 2830 2832 + 945 1 2834 2833 2835 + 946 1 2837 2836 2838 + 947 1 2840 2839 2841 + 948 1 2843 2842 2844 + 949 1 2846 2845 2847 + 950 1 2849 2848 2850 + 951 1 2852 2851 2853 + 952 1 2855 2854 2856 + 953 1 2858 2857 2859 + 954 1 2861 2860 2862 + 955 1 2864 2863 2865 + 956 1 2867 2866 2868 + 957 1 2870 2869 2871 + 958 1 2873 2872 2874 + 959 1 2876 2875 2877 + 960 1 2879 2878 2880 + 961 1 2882 2881 2883 + 962 1 2885 2884 2886 + 963 1 2888 2887 2889 + 964 1 2891 2890 2892 + 965 1 2894 2893 2895 + 966 1 2897 2896 2898 + 967 1 2900 2899 2901 + 968 1 2903 2902 2904 + 969 1 2906 2905 2907 + 970 1 2909 2908 2910 + 971 1 2912 2911 2913 + 972 1 2915 2914 2916 + 973 1 2918 2917 2919 + 974 1 2921 2920 2922 + 975 1 2924 2923 2925 + 976 1 2927 2926 2928 + 977 1 2930 2929 2931 + 978 1 2933 2932 2934 + 979 1 2936 2935 2937 + 980 1 2939 2938 2940 + 981 1 2942 2941 2943 + 982 1 2945 2944 2946 + 983 1 2948 2947 2949 + 984 1 2951 2950 2952 + 985 1 2954 2953 2955 + 986 1 2957 2956 2958 + 987 1 2960 2959 2961 + 988 1 2963 2962 2964 + 989 1 2966 2965 2967 + 990 1 2969 2968 2970 + 991 1 2972 2971 2973 + 992 1 2975 2974 2976 + 993 1 2978 2977 2979 + 994 1 2981 2980 2982 + 995 1 2984 2983 2985 + 996 1 2987 2986 2988 + 997 1 2990 2989 2991 + 998 1 2993 2992 2994 + 999 1 2996 2995 2997 + 1000 1 2999 2998 3000 + +Pair Coeffs + + 1 0.1553 3.1655 + 2 0.0000 0.0000 + +Bond Coeffs + + 1 450.0000 1.0000 + +Angle Coeffs + + 1 55.0000 109.4700 diff --git a/examples/VISCOSITY/in.cos.1000SPCE b/examples/VISCOSITY/in.cos.1000SPCE new file mode 100644 index 0000000000..415341f0eb --- /dev/null +++ b/examples/VISCOSITY/in.cos.1000SPCE @@ -0,0 +1,53 @@ +# DFF generated Lammps input file + +units real +atom_style full +boundary p p p + +pair_style lj/cut/coul/long 10.0 +pair_modify mix arithmetic +pair_modify tail yes +kspace_style pppm 1.0e-4 +dielectric 1.0 +special_bonds amber +bond_style harmonic +angle_style harmonic +dihedral_style none +improper_style none + +read_data data.cos.1000SPCE + +variable T equal 300 +variable P equal 1.0 + +velocity all create ${T} 12345 mom yes rot yes dist gaussian + +timestep 1.0 + +# Constraint ################################## +fix com all momentum 100 linear 1 1 1 +fix rigid all shake 1e-4 20 0 b 1 a 1 + +# Viscosity ################################## +variable A equal 0.05e-5 # angstrom/fs^2 + +fix cos all accelerate/cos ${A} +compute cos all viscosity/cos + +variable density equal density +variable lz equal lz +variable vMax equal c_cos[7] # velocity of atoms at z=0 +variable invVis equal v_vMax/${A}/v_density*39.4784/v_lz/v_lz*100 # reciprocal of viscosity 1/Pa/s + +fix npt all npt temp ${T} ${T} 100 iso ${P} ${P} 1000 +fix_modify npt temp cos + +thermo_style custom step cpu temp press pe density v_vMax v_invVis +thermo_modify temp cos +thermo 100 +################################################ + +dump 1 all custom 10000 dump.lammpstrj id mol type element q xu yu zu +dump_modify 1 sort id element O H + +run 2000 diff --git a/examples/VISCOSITY/log.30Apr20.cos.1000SPCE.g++.1 b/examples/VISCOSITY/log.30Apr20.cos.1000SPCE.g++.1 new file mode 100644 index 0000000000..896539a093 --- /dev/null +++ b/examples/VISCOSITY/log.30Apr20.cos.1000SPCE.g++.1 @@ -0,0 +1,160 @@ +LAMMPS (3 Mar 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +# DFF generated Lammps input file + +units real +atom_style full +boundary p p p + +pair_style lj/cut/coul/long 10.0 +pair_modify mix arithmetic +pair_modify tail yes +kspace_style pppm 1.0e-4 +dielectric 1.0 +special_bonds amber +bond_style harmonic +angle_style harmonic +dihedral_style none +improper_style none + +read_data data.1000SPCE.lmp + orthogonal box = (0 0 0) to (31.043 31.043 31.043) + 2 by 2 by 2 MPI processor grid + reading atoms ... + 3000 atoms + scanning bonds ... + 2 = max bonds/atom + scanning angles ... + 1 = max angles/atom + reading bonds ... + 2000 bonds + reading angles ... + 1000 angles + 2 = max # of 1-2 neighbors + 1 = max # of 1-3 neighbors + 1 = max # of 1-4 neighbors + 2 = max # of special neighbors + special bonds CPU = 0.00114917 secs + read_data CPU = 0.00953543 secs + +variable T equal 300 +variable P equal 1.0 + +velocity all create ${T} 12345 mom yes rot yes dist gaussian +velocity all create 300 12345 mom yes rot yes dist gaussian + +timestep 1.0 + +# Constraint ################################## +fix com all momentum 100 linear 1 1 1 +fix rigid all shake 1e-4 20 0 b 1 a 1 + 0 = # of size 2 clusters + 0 = # of size 3 clusters + 0 = # of size 4 clusters + 1000 = # of frozen angles + find clusters CPU = 0.000442737 secs + +# Viscosity ################################## +variable A equal 0.02e-5 # angstrom/fs^2 + +fix cos all accelerate/cos ${A} +fix cos all accelerate/cos 2e-07 +compute cos all viscosity/cos + +variable density equal density +variable lz equal lz +variable vMax equal c_cos[7] # velocity of atoms at z=0 +variable invVis equal v_vMax/${A}/v_density*39.4784/v_lz/v_lz*100 # reciprocal of viscosity 1/Pa/s +variable invVis equal v_vMax/2e-07/v_density*39.4784/v_lz/v_lz*100 + +fix npt all npt temp ${T} ${T} 100 iso ${P} ${P} 1000 +fix npt all npt temp 300 ${T} 100 iso ${P} ${P} 1000 +fix npt all npt temp 300 300 100 iso ${P} ${P} 1000 +fix npt all npt temp 300 300 100 iso 1 ${P} 1000 +fix npt all npt temp 300 300 100 iso 1 1 1000 +fix_modify npt temp cos + +thermo_style custom step cpu temp press pe density v_vMax v_invVis +thermo_modify temp cos +thermo 100 +################################################ + +dump 1 all custom 10000 dump.lammpstrj id mol type element q xu yu zu +dump_modify 1 sort id element O H + +run 2000 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/kspace.cpp:332) + G vector (1/distance) = 0.263539 + grid = 16 16 16 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0325342 + estimated relative force accuracy = 9.79757e-05 + using double precision MKL FFT + 3d grid and FFT values/proc = 3375 512 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 12 + ghost atom cutoff = 12 + binsize = 6, bins = 6 6 6 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut/coul/long, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Per MPI rank memory allocation (min/avg/max) = 10.14 | 10.14 | 10.14 Mbytes +Step CPU Temp Press PotEng Density v_vMax v_invVis + 0 0 450.04468 9838.6886 -7651.736 0.99999331 0.0001293705 2649.9663 + 100 0.33736925 497.65155 2024.4827 -8342.2499 0.98595028 0.00019602427 4034.2452 + 200 0.67116638 446.48518 27.075082 -8908.9684 0.9652009 0.00027615455 5723.7907 + 300 0.99760895 401.79875 -776.99871 -9381.8986 0.95205822 0.00019626685 4086.6103 + 400 1.3239019 369.65373 -510.5336 -9803.6463 0.94817309 0.00022998514 4795.2142 + 500 1.6488092 343.35807 -936.31982 -10146.023 0.94840581 0.0001434423 2990.5423 + 600 1.9826063 319.86131 -1381.3302 -10405.812 0.95459202 6.6411532e-05 1381.5767 + 700 2.3172637 307.74606 -98.775733 -10643.5 0.96669652 0.00010446317 2164.0664 + 800 2.6562841 305.14214 -540.57804 -10865.742 0.97808367 5.4381233e-05 1122.1765 + 900 2.9938415 288.01316 639.00486 -10925.39 0.98686357 0.00010878474 2238.1355 + 1000 3.327893 295.07773 -226.06503 -11033.826 0.99128496 0.00011935058 2451.8608 + 1100 3.6618862 299.21578 306.34231 -11049.152 0.99552203 8.9538943e-05 1836.8166 + 1200 3.9984287 301.82462 85.804646 -11013.564 0.99713434 0.00015912276 3262.51 + 1300 4.3320735 308.6009 268.08897 -11009.836 0.99695358 0.00026212596 5374.72 + 1400 4.668875 298.36903 -258.75495 -10962.299 0.99503447 0.00033087355 6788.7027 + 1500 5.0003694 299.96073 99.512082 -10980.551 0.99315631 0.00033996557 6979.6425 + 1600 5.3367337 304.18018 -500.65441 -11002.054 0.9914558 0.00039075642 8026.9849 + 1700 5.6780828 301.63978 -499.07458 -10992.88 0.99234354 0.00038101175 7824.4738 + 1800 6.0140638 303.25858 640.03432 -11053.335 0.99553958 0.00041336203 8479.7267 + 1900 6.3532521 301.40882 208.28331 -11119.481 0.99534534 0.00032474734 6662.3144 + 2000 6.6938104 298.0462 -236.47954 -11162.212 0.99421846 0.00023869721 4898.8129 +Loop time of 6.69387 on 8 procs for 2000 steps with 3000 atoms + +Performance: 25.815 ns/day, 0.930 hours/ns, 298.781 timesteps/s +99.7% CPU use with 8 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 4.36 | 4.4981 | 4.6781 | 5.1 | 67.20 +Bond | 0.00074545 | 0.00088463 | 0.0012464 | 0.0 | 0.01 +Kspace | 0.86696 | 1.0476 | 1.1863 | 10.5 | 15.65 +Neigh | 0.37733 | 0.37785 | 0.3784 | 0.1 | 5.64 +Comm | 0.19874 | 0.20168 | 0.20729 | 0.6 | 3.01 +Output | 0.0015529 | 0.0015803 | 0.0017546 | 0.2 | 0.02 +Modify | 0.54083 | 0.55143 | 0.55445 | 0.6 | 8.24 +Other | | 0.01483 | | | 0.22 + +Nlocal: 375 ave 385 max 361 min +Histogram: 1 1 0 0 1 0 2 0 1 2 +Nghost: 5772.25 ave 5789 max 5757 min +Histogram: 1 1 2 0 0 0 2 0 0 2 +Neighs: 135285 ave 144189 max 127550 min +Histogram: 1 2 1 1 0 0 0 0 1 2 + +Total # of neighbors = 1082280 +Ave neighs/atom = 360.76 +Ave special neighs/atom = 2 +Neighbor list builds = 101 +Dangerous builds = 1 +Total wall time: 0:00:06 diff --git a/examples/granular/in.pour.drum b/examples/granular/in.pour.drum index c30f30b377..54372cd391 100644 --- a/examples/granular/in.pour.drum +++ b/examples/granular/in.pour.drum @@ -92,6 +92,8 @@ fix 5 all wall/gran/region granular hertz/material 1e5 0.1 0.3 tangential mindl # 'Turn' drum by switching the direction of gravity unfix grav +unfix ins1 +unfix ins2 fix grav all gravity 10 vector 0 -1 0 variable theta equal 2*PI*elapsed/20000.0 diff --git a/examples/kim/in.kim-pm-property b/examples/kim/in.kim-pm-property new file mode 100644 index 0000000000..fea1527820 --- /dev/null +++ b/examples/kim/in.kim-pm-property @@ -0,0 +1,91 @@ +# kim-property example +# +# For detailed information of this example please refer to: +# https://openkim.org/doc/evaluation/tutorial-lammps/ +# +# Description: +# +# This example is designed to calculate the cohesive energy corresponding to +# the equilibrium FCC lattice constant for +# `LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` model for +# argon. The material properties computed in LAMMPS are represented as a +# standard KIM property instance format. (See +# https://openkim.org/doc/schema/properties-framework/ and +# https://lammps.sandia.gov/doc/kim_commands.html for further details). +# Then the created property instance is written to a file named results.edn +# using the `kim_property dump` commands. +# +# Requirement: +# +# This example requires LAMMPS built with the Python 3.6 or later package +# installed. See the `https://lammps.sandia.gov/doc/python.html` doc page for +# more info on building LAMMPS with the version of Python on your system. +# After successfully building LAMMPS with Python, you need to install the +# kim-property Python package, See the +# `https://lammps.sandia.gov/doc/Build_extras.html#kim` doc page for +# further details. +# +# This example requires that the KIM Portable Model (PM) +# `LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` +# is installed. This can be done with the command +# `kim-api-collections-management install user LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004` +# If this command does not work, you may need to setup your PATH to find the utility. +# If you installed the kim-api using the LAMMPS CMake build, you can do the following +# (where the current working directory is assumed to be the LAMMPS build directory) +# source ./kim_build-prefix/bin/kim-api-activate +# If you installed the kim-api using the LAMMPS Make build, you can do the following +# (where the current working directory is assumed to be the LAMMPS src directory) +# source ../lib/kim/installed-kim-api-X.Y.Z/bin/kim-api-activate +# (where you should relplace X.Y.Z with the appropriate kim-api version number). +# +# Or, see https://openkim.org/doc/obtaining-models for alternative options. +# + +# Initialize interatomic potential (KIM model) and units +atom_style atomic + +# Set the OpenKIM model that will be used +kim_init LJ_Shifted_Bernardes_1958MedCutoff_Ar__MO_126566794224_004 metal + +# the equilibrium lattice constant for the fcc structure +variable lattice_constant equal 5.248509056866169 + +# Periodic boundary conditions along all three dimensions +boundary p p p + +# Create an FCC lattice with the lattice spacing +# using a single conventional (orthogonal) unit cell +lattice fcc ${lattice_constant} +region box block 0 1 0 1 0 1 units lattice +create_box 1 box +create_atoms 1 box +mass 1 39.948 + +# Specify the KIM interactions +kim_interactions Ar + +# Compute energy +run 0 + +# Get cohesive energy +variable natoms equal "count(all)" +variable ecohesive equal "-pe/v_natoms" + +# Create a property instance +kim_property create 1 cohesive-potential-energy-cubic-crystal + +# Set all the key-value pairs for this property instance +kim_property modify 1 key short-name source-value 1 fcc & + key species source-value 1 Ar & + key a source-value ${lattice_constant} & + source-unit angstrom & + key basis-atom-coordinates source-value 1 1:3 0.0 0.0 0.0 & + source-value 2 1:3 0.0 0.5 0.5 & + source-value 3 1:3 0.5 0.0 0.5 & + source-value 4 1:3 0.5 0.5 0.0 & + key space-group source-value Fm-3m & + key cohesive-potential-energy source-value ${ecohesive} & + source-unit eV + +# Dump the results in a file +kim_property dump "results.edn" diff --git a/examples/qeq/in.qeq.buck b/examples/qeq/in.qeq.buck index f7ec2bb808..2258a9e2ae 100644 --- a/examples/qeq/in.qeq.buck +++ b/examples/qeq/in.qeq.buck @@ -24,7 +24,7 @@ compute charge2 type2 property/atom q compute q2 type2 reduce ave c_charge2 variable qtot equal count(type1)*c_q1+count(type2)*c_q2 -thermo_style custom step pe c_q1 c_q2 v_qtot spcpu +thermo_style custom step pe c_q1 c_q2 v_qtot thermo 10 timestep 0.0001 diff --git a/examples/streitz/in.streitz.wolf b/examples/streitz/in.streitz.wolf index ff4479448e..cdf7cf33c9 100644 --- a/examples/streitz/in.streitz.wolf +++ b/examples/streitz/in.streitz.wolf @@ -39,7 +39,7 @@ neigh_modify every 10 delay 0 check yes timestep 0.0004 thermo_style custom step temp etotal pe evdwl ecoul elong & - c_q1 c_q2 v_qsum press spcpu + c_q1 c_q2 v_qsum press thermo_modify norm yes thermo 10 diff --git a/lib/atc/Function.cpp b/lib/atc/Function.cpp index 70f8bbfa41..062cbcb9b2 100644 --- a/lib/atc/Function.cpp +++ b/lib/atc/Function.cpp @@ -1,5 +1,11 @@ -#ifndef _WIN32 +#if !defined(_WIN32) + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__NetBSD__) +#include +#else #include +#endif + #endif #include "Function.h" #include "ATC_Error.h" diff --git a/lib/colvars/Makefile.common b/lib/colvars/Makefile.common index 0482cff74a..2f2e2beffb 100644 --- a/lib/colvars/Makefile.common +++ b/lib/colvars/Makefile.common @@ -61,7 +61,7 @@ ifeq ($(COLVARS_LEPTON),no) LEPTON_INCFLAGS = COLVARS_OBJS = $(COLVARS_SRCS:.cpp=.o) else -LEPTON_INCFLAGS = -Ilepton/include -DLEPTON +LEPTON_INCFLAGS = -Ilepton/include -DLEPTON -DLEPTON_USE_STATIC_LIBRARIES COLVARS_OBJS = $(COLVARS_SRCS:.cpp=.o) $(LEPTON_SRCS:.cpp=.o) endif @@ -82,4 +82,20 @@ Makefile.deps: $(COLVARS_SRCS) done include Makefile.deps + +# Exceptions to pattern rule above for Lepton objects + +lepton/src/CompiledExpression.o: lepton/src/CompiledExpression.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< +lepton/src/ExpressionProgram.o: lepton/src/ExpressionProgram.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< +lepton/src/ExpressionTreeNode.o: lepton/src/ExpressionTreeNode.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< +lepton/src/Operation.o: lepton/src/Operation.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< +lepton/src/ParsedExpression.o: lepton/src/ParsedExpression.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< +lepton/src/Parser.o: lepton/src/Parser.cpp + $(CXX) $(CXXFLAGS) -Ilepton/include -DLEPTON_BUILDING_STATIC_LIBRARY -c -o $@ $< + include Makefile.lepton.deps # Hand-generated diff --git a/lib/colvars/lepton/src/MSVC_erfc.h b/lib/colvars/lepton/src/MSVC_erfc.h index c30a8ce542..b1cd87a289 100644 --- a/lib/colvars/lepton/src/MSVC_erfc.h +++ b/lib/colvars/lepton/src/MSVC_erfc.h @@ -8,9 +8,13 @@ * (VC11 has _MSC_VER=1700). */ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) +#if !defined(M_PI) #define M_PI 3.14159265358979323846264338327950288 +#endif +#endif +#if defined(_MSC_VER) #if _MSC_VER <= 1700 // 1700 is VC11, 1800 is VC12 /*************************** * erf.cpp diff --git a/lib/gpu/.gitignore b/lib/gpu/.gitignore index 7b5254e480..d32b287c1e 100644 --- a/lib/gpu/.gitignore +++ b/lib/gpu/.gitignore @@ -2,6 +2,7 @@ /obj_ocl /ocl_get_devices /nvc_get_devices +/hip_get_devices /*.cubin /*_cubin.h /*_cl.h diff --git a/lib/gpu/Makefile.hip b/lib/gpu/Makefile.hip new file mode 100644 index 0000000000..62d34592d1 --- /dev/null +++ b/lib/gpu/Makefile.hip @@ -0,0 +1,148 @@ +# /* ---------------------------------------------------------------------- +# Generic Linux Makefile for HIP +# - export HIP_PLATFORM=hcc (or nvcc) before execution +# - change HIP_ARCH for your GPU +# ------------------------------------------------------------------------- */ + +# this setting should match LAMMPS Makefile +# one of LAMMPS_SMALLBIG (default), LAMMPS_BIGBIG and LAMMPS_SMALLSMALL + +LMP_INC = -DLAMMPS_SMALLBIG + +# precision for GPU calculations +# -D_SINGLE_SINGLE # Single precision for all calculations +# -D_DOUBLE_DOUBLE # Double precision for all calculations +# -D_SINGLE_DOUBLE # Accumulation of forces, etc. in double + +HIP_PRECISION = -D_SINGLE_DOUBLE + +HIP_OPTS = -O3 +HIP_HOST_OPTS = -Wno-deprecated-declarations +HIP_HOST_INCLUDE = + +# use device sort +# requires linking with hipcc and hipCUB + (rocPRIM or CUB for AMD or Nvidia respectively) +HIP_HOST_OPTS += -DUSE_HIP_DEVICE_SORT +# path to cub +HIP_HOST_INCLUDE += -I./ +# path to hipcub +HIP_HOST_INCLUDE += -I$(HIP_PATH)/../include + +# use mpi +HIP_HOST_OPTS += -DMPI_GERYON -DUCL_NO_EXIT +# this settings should match LAMMPS Makefile +MPI_COMP_OPTS = $(shell mpicxx --showme:compile) +MPI_LINK_OPTS = $(shell mpicxx --showme:link) +#MPI_COMP_OPTS += -I/usr/include/mpi -DMPICH_IGNORE_CXX_SEEK -DOMPI_SKIP_MPICXX=1 + +HIP_PATH ?= $(wildcard /opt/rocm/hip) +HIP_PLATFORM=$(shell $(HIP_PATH)/bin/hipconfig --compiler) + +ifeq (hcc,$(HIP_PLATFORM)) + HIP_OPTS += -ffast-math + # possible values: gfx803,gfx900,gfx906 + HIP_ARCH = gfx906 +else ifeq (nvcc,$(HIP_PLATFORM)) + HIP_OPTS += --use_fast_math + HIP_ARCH = -gencode arch=compute_30,code=[sm_30,compute_30] -gencode arch=compute_32,code=[sm_32,compute_32] -gencode arch=compute_35,code=[sm_35,compute_35] \ + -gencode arch=compute_50,code=[sm_50,compute_50] -gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_53,code=[sm_53,compute_53]\ + -gencode arch=compute_60,code=[sm_60,compute_60] -gencode arch=compute_61,code=[sm_61,compute_61] -gencode arch=compute_62,code=[sm_62,compute_62]\ + -gencode arch=compute_70,code=[sm_70,compute_70] -gencode arch=compute_72,code=[sm_72,compute_72] -gencode arch=compute_75,code=[sm_75,compute_75] +else + $(error Specify HIP platform using 'export HIP_PLATFORM=(hcc,nvcc)') +endif + +BIN_DIR = . +OBJ_DIR = ./obj +LIB_DIR = . +AR = ar +BSH = /bin/sh + + +# /* ---------------------------------------------------------------------- +# don't change section below without need +# ------------------------------------------------------------------------- */ + +HIP_OPTS += -DUSE_HIP $(HIP_PRECISION) +HIP_GPU_OPTS += $(HIP_OPTS) -I./ + +ifeq (hcc,$(HIP_PLATFORM)) + HIP_HOST_OPTS += -fPIC + HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --genco + HIP_GPU_OPTS_S = -t="$(HIP_ARCH)" -f=\" + HIP_GPU_OPTS_E = \" + HIP_KERNEL_SUFFIX = .cpp + HIP_LIBS_TARGET = export HCC_AMDGPU_TARGET := $(HIP_ARCH) + export HCC_AMDGPU_TARGET := $(HIP_ARCH) +else ifeq (nvcc,$(HIP_PLATFORM)) + HIP_GPU_CC = $(HIP_PATH)/bin/hipcc --fatbin + HIP_GPU_OPTS += $(HIP_ARCH) + HIP_GPU_SORT_ARCH = $(HIP_ARCH) + # fix nvcc can't handle -pthread flag + MPI_COMP_OPTS := $(subst -pthread,-Xcompiler -pthread,$(MPI_COMP_OPTS)) + MPI_LINK_OPTS := $(subst -pthread,-Xcompiler -pthread,$(MPI_LINK_OPTS)) +endif + +# hipcc is essential for device sort, because of hipcub is header only library and ROCm gpu code generation is deferred to the linking stage +HIP_HOST_CC = $(HIP_PATH)/bin/hipcc +HIP_HOST_OPTS += $(HIP_OPTS) $(MPI_COMP_OPTS) $(LMP_INC) +HIP_HOST_CC_CMD = $(HIP_HOST_CC) $(HIP_HOST_OPTS) $(HIP_HOST_INCLUDE) + +# sources + +ALL_H = $(wildcard ./geryon/ucl*.h) $(wildcard ./geryon/hip*.h) $(wildcard ./lal_*.h) +SRCS := $(wildcard ./lal_*.cpp) +OBJS := $(subst ./,$(OBJ_DIR)/,$(SRCS:%.cpp=%.o)) +CUS := $(wildcard lal_*.cu) +CUHS := $(filter-out pppm_cubin.h, $(CUS:lal_%.cu=%_cubin.h)) pppm_f_cubin.h pppm_d_cubin.h +CUHS := $(addprefix $(OBJ_DIR)/, $(CUHS)) + +all: $(OBJ_DIR) $(CUHS) $(LIB_DIR)/libgpu.a $(BIN_DIR)/hip_get_devices + +$(OBJ_DIR): + mkdir -p $@ + +# GPU kernels compilation + +$(OBJ_DIR)/pppm_f_cubin.h: lal_pppm.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) -Dgrdtyp=float -Dgrdtyp4=float4 $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/pppm_f.cubin $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/pppm_f.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*pppm_f_cubin/pppm_f/g" $@ + @rm $(OBJ_DIR)/temp_pppm_f.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/pppm_f.cubin + +$(OBJ_DIR)/pppm_d_cubin.h: lal_pppm.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) -Dgrdtyp=double -Dgrdtyp4=double4 $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/pppm_d.cubin $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/pppm_d.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*pppm_d_cubin/pppm_d/g" $@ + @rm $(OBJ_DIR)/temp_pppm_d.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/pppm_d.cubin + +$(OBJ_DIR)/%_cubin.h: lal_%.cu $(ALL_H) + @cp $< $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) + $(HIP_GPU_CC) $(HIP_GPU_OPTS_S) $(HIP_GPU_OPTS) $(HIP_GPU_OPTS_E) -o $(OBJ_DIR)/$*.cubin $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) + @xxd -i $(OBJ_DIR)/$*.cubin $@ + @sed -i "s/[a-zA-Z0-9_]*$*_cubin/$*/g" $@ + @rm $(OBJ_DIR)/temp_$*.cu$(HIP_KERNEL_SUFFIX) $(OBJ_DIR)/$*.cubin + +# host sources compilation + +$(OBJ_DIR)/lal_atom.o: lal_atom.cpp $(CUHS) $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ -c $< -I$(OBJ_DIR) $(HIP_GPU_SORT_ARCH) + +$(OBJ_DIR)/lal_%.o: lal_%.cpp $(CUHS) $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ -c $< -I$(OBJ_DIR) + +# libgpu building + +$(LIB_DIR)/libgpu.a: $(OBJS) + $(AR) -crs $@ $(OBJS) + printf "export HIP_PLATFORM := %s\n%s\n" "$(HIP_PLATFORM)" "$(HIP_LIBS_TARGET)" > Makefile.lammps + +# test app building + +$(BIN_DIR)/hip_get_devices: ./geryon/ucl_get_devices.cpp $(ALL_H) + $(HIP_HOST_CC_CMD) -o $@ $< -DUCL_HIP $(MPI_LINK_OPTS) + +clean: + -rm -f $(BIN_DIR)/hip_get_devices $(LIB_DIR)/libgpu.a $(OBJS) $(OBJ_DIR)/temp_* $(CUHS) diff --git a/lib/gpu/README b/lib/gpu/README index 2ef8ce9556..dfa8dcf7ff 100644 --- a/lib/gpu/README +++ b/lib/gpu/README @@ -67,8 +67,8 @@ library requires installing the CUDA GPU driver and CUDA toolkit for your operating system. Installation of the CUDA SDK is not necessary. In addition to the LAMMPS library, the binary nvc_get_devices will also be built. This can be used to query the names and properties of GPU -devices on your system. A Makefile for OpenCL compilation is provided, -but support for OpenCL use is not currently provided by the developers. +devices on your system. A Makefile for OpenCL and ROCm HIP compilation +is provided, but support for it is not currently provided by the developers. Details of the implementation are provided in: ---- @@ -169,6 +169,25 @@ NOTE: The system-specific setting LAMMPS_SMALLBIG (default), LAMMPS_BIGBIG, src/MAKE/Makefile.foo) should be consistent with that specified when building libgpu.a (i.e. by LMP_INC in the lib/gpu/Makefile.bar). + BUILDING FOR HIP FRAMEWORK + -------------------------------- +1. Install the latest ROCm framework (https://github.com/RadeonOpenCompute/ROCm). +2. GPU sorting requires installing hipcub +(https://github.com/ROCmSoftwarePlatform/hipCUB). The HIP CUDA-backend +additionally requires cub (https://nvlabs.github.io/cub). Download and +extract the cub directory to lammps/lib/gpu/ or specify an appropriate +path in lammps/lib/gpu/Makefile.hip. +3. In Makefile.hip it is possible to specify the target platform via +export HIP_PLATFORM=hcc or HIP_PLATFORM=nvcc as well as the target +architecture (gfx803, gfx900, gfx906 etc.) +4. If your MPI implementation does not support `mpicxx --showme` command, +it is required to specify the corresponding MPI compiler and linker flags +in lammps/lib/gpu/Makefile.hip and in lammps/src/MAKE/OPTIONS/Makefile.hip. +5. Building the GPU library (libgpu.a): + cd lammps/lib/gpu; make -f Makefile.hip -j +6. Building the LAMMPS executable (lmp_hip): + cd ../../src; make hip -j + EXAMPLE CONVENTIONAL BUILD PROCESS -------------------------------- diff --git a/lib/gpu/geryon/hip_device.h b/lib/gpu/geryon/hip_device.h new file mode 100644 index 0000000000..93f38d28bb --- /dev/null +++ b/lib/gpu/geryon/hip_device.h @@ -0,0 +1,519 @@ +/* ----------------------------------------------------------------------- + Copyright (2009) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_DEVICE +#define HIP_DEVICE + + +#include +#include +#include +#include +#include +#include "hip_macros.h" +#include "ucl_types.h" + +namespace ucl_hip { + +// -------------------------------------------------------------------------- +// - COMMAND QUEUE STUFF +// -------------------------------------------------------------------------- +typedef hipStream_t command_queue; + +inline void ucl_sync(hipStream_t &stream) { + CU_SAFE_CALL(hipStreamSynchronize(stream)); +} + +struct NVDProperties { + int device_id; + std::string name; + int major; + int minor; + CUDA_INT_TYPE totalGlobalMem; + int multiProcessorCount; + + int maxThreadsPerBlock; + int maxThreadsDim[3]; + int maxGridSize[3]; + int sharedMemPerBlock; + int totalConstantMemory; + int SIMDWidth; + int memPitch; + int regsPerBlock; + int clockRate; + int textureAlign; + + int kernelExecTimeoutEnabled; + int integrated; + int canMapHostMemory; + int concurrentKernels; + int ECCEnabled; + int computeMode; +}; + +/// Class for looking at device properties +/** \note Calls to change the device outside of the class results in incorrect + * behavior + * \note There is no error checking for indexing past the number of devices **/ +class UCL_Device { + public: + /// Collect properties for every GPU on the node + /** \note You must set the active GPU with set() before using the device **/ + inline UCL_Device(); + + inline ~UCL_Device(); + + /// Returns 1 (For compatibility with OpenCL) + inline int num_platforms() { return 1; } + + /// Return a string with name and info of the current platform + inline std::string platform_name() + { return "HIP platform"; } + + /// Delete any contexts/data and set the platform number to be used + inline int set_platform(const int pid); + + /// Return the number of devices that support CUDA + inline int num_devices() { return _properties.size(); } + + /// Set the CUDA device to the specified device number + /** A context and default command queue will be created for the device + * Returns UCL_SUCCESS if successful or UCL_ERROR if the device could not + * be allocated for use. clear() is called to delete any contexts and + * associated data from previous calls to set(). **/ + inline int set(int num); + + /// Delete any context and associated data stored from a call to set() + inline void clear(); + + /// Get the current device number + inline int device_num() { return _device; } + + /// Returns the default stream for the current device + inline command_queue & cq() { return cq(0); } + + /// Returns the stream indexed by i + inline command_queue & cq(const int i) { return _cq[i]; } + + /// Block until all commands in the default stream have completed + inline void sync() { sync(0); } + + /// Block until all commands in the specified stream have completed + inline void sync(const int i) { ucl_sync(cq(i)); } + + /// Get the number of command queues currently available on device + inline int num_queues() + { return _cq.size(); } + + /// Add a stream for device computations + inline void push_command_queue() { + _cq.push_back(hipStream_t()); + CU_SAFE_CALL(hipStreamCreateWithFlags(&_cq.back(),0)); + } + + /// Remove a stream for device computations + /** \note You cannot delete the default stream **/ + inline void pop_command_queue() { + if (_cq.size()<2) return; + CU_SAFE_CALL_NS(hipStreamDestroy(_cq.back())); + _cq.pop_back(); + } + + /// Set the default command queue (by default this is the null stream) + /** \param i index of the command queue (as added by push_command_queue()) + If i is 0, the default command queue is set to the null stream **/ + inline void set_command_queue(const int i) { + if (i==0) _cq[0]=0; + else _cq[0]=_cq[i]; + } + + /// Get the current CUDA device name + inline std::string name() { return name(_device); } + /// Get the CUDA device name + inline std::string name(const int i) + { return std::string(_properties[i].name); } + + /// Get a string telling the type of the current device + inline std::string device_type_name() { return device_type_name(_device); } + /// Get a string telling the type of the device + inline std::string device_type_name(const int i) { return "GPU"; } + + /// Get current device type (UCL_CPU, UCL_GPU, UCL_ACCELERATOR, UCL_DEFAULT) + inline int device_type() { return device_type(_device); } + /// Get device type (UCL_CPU, UCL_GPU, UCL_ACCELERATOR, UCL_DEFAULT) + inline int device_type(const int i) { return UCL_GPU; } + + /// Returns true if host memory is efficiently addressable from device + inline bool shared_memory() { return shared_memory(_device); } + /// Returns true if host memory is efficiently addressable from device + inline bool shared_memory(const int i) { return device_type(i)==UCL_CPU; } + + /// Returns true if double precision is support for the current device + inline bool double_precision() { return double_precision(_device); } + /// Returns true if double precision is support for the device + inline bool double_precision(const int i) {return arch(i)>=1.3;} + + /// Get the number of compute units on the current device + inline unsigned cus() { return cus(_device); } + /// Get the number of compute units + inline unsigned cus(const int i) + { return _properties[i].multiProcessorCount; } + + /// Get the number of cores in the current device + inline unsigned cores() { return cores(_device); } + /// Get the number of cores + inline unsigned cores(const int i) + { if (arch(i)<2.0) return _properties[i].multiProcessorCount*8; + else if (arch(i)<2.1) return _properties[i].multiProcessorCount*32; + else if (arch(i)<3.0) return _properties[i].multiProcessorCount*48; + else return _properties[i].multiProcessorCount*192; } + + /// Get the gigabytes of global memory in the current device + inline double gigabytes() { return gigabytes(_device); } + /// Get the gigabytes of global memory + inline double gigabytes(const int i) + { return static_cast(_properties[i].totalGlobalMem)/1073741824; } + + /// Get the bytes of global memory in the current device + inline size_t bytes() { return bytes(_device); } + /// Get the bytes of global memory + inline size_t bytes(const int i) { return _properties[i].totalGlobalMem; } + + // Get the gigabytes of free memory in the current device + inline double free_gigabytes() { return free_gigabytes(_device); } + // Get the gigabytes of free memory + inline double free_gigabytes(const int i) + { return static_cast(free_bytes(i))/1073741824; } + + // Get the bytes of free memory in the current device + inline size_t free_bytes() { return free_bytes(_device); } + // Get the bytes of free memory + inline size_t free_bytes(const int i) { + CUDA_INT_TYPE dfree, dtotal; + CU_SAFE_CALL_NS(hipMemGetInfo(&dfree, &dtotal)); + return static_cast(dfree); + } + + /// Return the GPGPU compute capability for current device + inline double arch() { return arch(_device); } + /// Return the GPGPU compute capability + inline double arch(const int i) + { return static_cast(_properties[i].minor)/10+_properties[i].major;} + + /// Clock rate in GHz for current device + inline double clock_rate() { return clock_rate(_device); } + /// Clock rate in GHz + inline double clock_rate(const int i) + { return _properties[i].clockRate*1e-6;} + + /// Get the maximum number of threads per block + inline size_t group_size() { return group_size(_device); } + /// Get the maximum number of threads per block + inline size_t group_size(const int i) + { return _properties[i].maxThreadsPerBlock; } + + /// Return the maximum memory pitch in bytes for current device + inline size_t max_pitch() { return max_pitch(_device); } + /// Return the maximum memory pitch in bytes + inline size_t max_pitch(const int i) { return _properties[i].memPitch; } + + /// Returns false if accelerator cannot be shared by multiple processes + /** If it cannot be determined, true is returned **/ + inline bool sharing_supported() { return sharing_supported(_device); } + /// Returns false if accelerator cannot be shared by multiple processes + /** If it cannot be determined, true is returned **/ + inline bool sharing_supported(const int i) + { return (_properties[i].computeMode == hipComputeModeDefault); } + + /// True if splitting device into equal subdevices supported + inline bool fission_equal() + { return fission_equal(_device); } + /// True if splitting device into equal subdevices supported + inline bool fission_equal(const int i) + { return false; } + /// True if splitting device into subdevices by specified counts supported + inline bool fission_by_counts() + { return fission_by_counts(_device); } + /// True if splitting device into subdevices by specified counts supported + inline bool fission_by_counts(const int i) + { return false; } + /// True if splitting device into subdevices by affinity domains supported + inline bool fission_by_affinity() + { return fission_by_affinity(_device); } + /// True if splitting device into subdevices by affinity domains supported + inline bool fission_by_affinity(const int i) + { return false; } + + /// Maximum number of subdevices allowed from device fission + inline int max_sub_devices() + { return max_sub_devices(_device); } + /// Maximum number of subdevices allowed from device fission + inline int max_sub_devices(const int i) + { return 0; } + + /// List all devices along with all properties + inline void print_all(std::ostream &out); + + /// Select the platform that has accelerators (for compatibility with OpenCL) + inline int set_platform_accelerator(int pid=-1) { return UCL_SUCCESS; } + + inline int load_module(const void* program, hipModule_t& module, std::string *log=NULL){ + auto it = _loaded_modules.emplace(program, hipModule_t()); + if(!it.second){ + module = it.first->second; + return UCL_SUCCESS; + } + const unsigned int num_opts=2; + hipJitOption options[num_opts]; + void *values[num_opts]; + + // set up size of compilation log buffer + options[0] = hipJitOptionInfoLogBufferSizeBytes; + values[0] = (void *)(int)10240; + // set up pointer to the compilation log buffer + options[1] = hipJitOptionInfoLogBuffer; + char clog[10240] = { 0 }; + values[1] = clog; + + hipError_t err=hipModuleLoadDataEx(&module,program,num_opts, options,(void **)values); + + if (log!=NULL) + *log=std::string(clog); + + if (err != hipSuccess) { + #ifndef UCL_NO_EXIT + std::cerr << std::endl + << "----------------------------------------------------------\n" + << " UCL Error: Error compiling PTX Program...\n" + << "----------------------------------------------------------\n"; + std::cerr << log << std::endl; + #endif + _loaded_modules.erase(it.first); + return UCL_COMPILE_ERROR; + } + it.first->second = module; + return UCL_SUCCESS; + } + private: + std::unordered_map _loaded_modules; + int _device, _num_devices; + std::vector _properties; + std::vector _cq; + hipDevice_t _cu_device; +}; + +// Grabs the properties for all devices +UCL_Device::UCL_Device() { + CU_SAFE_CALL_NS(hipInit(0)); + CU_SAFE_CALL_NS(hipGetDeviceCount(&_num_devices)); + for (int i=0; i<_num_devices; ++i) { + hipDevice_t dev; + CU_SAFE_CALL_NS(hipDeviceGet(&dev,i)); + int major, minor; + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&minor, hipDeviceAttributeComputeCapabilityMinor, dev)); + if (major==9999) + continue; + + NVDProperties prop; + prop.device_id = i; + prop.major=major; + prop.minor=minor; + + char namecstr[1024]; + CU_SAFE_CALL_NS(hipDeviceGetName(namecstr,1024,dev)); + prop.name=namecstr; + + CU_SAFE_CALL_NS(hipDeviceTotalMem(&prop.totalGlobalMem,dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.multiProcessorCount, hipDeviceAttributeMultiprocessorCount, dev)); + + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsPerBlock, hipDeviceAttributeMaxThreadsPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[0], hipDeviceAttributeMaxBlockDimX, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[1], hipDeviceAttributeMaxBlockDimY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxThreadsDim[2], hipDeviceAttributeMaxBlockDimZ, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[0], hipDeviceAttributeMaxGridDimX, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[1], hipDeviceAttributeMaxGridDimY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.maxGridSize[2], hipDeviceAttributeMaxGridDimZ, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.sharedMemPerBlock, hipDeviceAttributeMaxSharedMemoryPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.totalConstantMemory, hipDeviceAttributeTotalConstantMemory, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.SIMDWidth, hipDeviceAttributeWarpSize, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.memPitch, CU_DEVICE_ATTRIBUTE_MAX_PITCH, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.regsPerBlock, hipDeviceAttributeMaxRegistersPerBlock, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.clockRate, hipDeviceAttributeClockRate, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.textureAlign, CU_DEVICE_ATTRIBUTE_TEXTURE_ALIGNMENT, dev)); + + //#if CUDA_VERSION >= 2020 + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.kernelExecTimeoutEnabled, CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT,dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.integrated, hipDeviceAttributeIntegrated, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.canMapHostMemory, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev)); + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.computeMode, hipDeviceAttributeComputeMode,dev)); + //#endif + //#if CUDA_VERSION >= 3010 + CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.concurrentKernels, hipDeviceAttributeConcurrentKernels, dev)); + //CU_SAFE_CALL_NS(hipDeviceGetAttribute(&prop.ECCEnabled, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, dev)); + //#endif + + _properties.push_back(prop); + } + _device=-1; + _cq.push_back(hipStream_t()); + _cq.back()=0; +} + +UCL_Device::~UCL_Device() { + clear(); +} + +int UCL_Device::set_platform(const int pid) { + clear(); + #ifdef UCL_DEBUG + assert(pid-1) { + for (int i=1; i= 2020 + int driver_version; + hipDriverGetVersion(&driver_version); + out << "Driver Version: " + << driver_version/1000 << "." << driver_version%100 + << std::endl; + //#endif + + if (num_devices() == 0) + out << "There is no device supporting HIP\n"; + for (int i=0; i= 2000 + out << " Number of compute units/multiprocessors: " + << _properties[i].multiProcessorCount << std::endl; + out << " Number of cores: " + << cores(i) << std::endl; + //#endif + out << " Total amount of constant memory: " + << _properties[i].totalConstantMemory << " bytes\n"; + out << " Total amount of local/shared memory per block: " + << _properties[i].sharedMemPerBlock << " bytes\n"; + out << " Total number of registers available per block: " + << _properties[i].regsPerBlock << std::endl; + out << " Warp size: " + << _properties[i].SIMDWidth << std::endl; + out << " Maximum number of threads per block: " + << _properties[i].maxThreadsPerBlock << std::endl; + out << " Maximum group size (# of threads per block) " + << _properties[i].maxThreadsDim[0] << " x " + << _properties[i].maxThreadsDim[1] << " x " + << _properties[i].maxThreadsDim[2] << std::endl; + out << " Maximum item sizes (# threads for each dim) " + << _properties[i].maxGridSize[0] << " x " + << _properties[i].maxGridSize[1] << " x " + << _properties[i].maxGridSize[2] << std::endl; + //out << " Maximum memory pitch: " + // << max_pitch(i) << " bytes\n"; + //out << " Texture alignment: " + // << _properties[i].textureAlign << " bytes\n"; + out << " Clock rate: " + << clock_rate(i) << " GHz\n"; + //#if CUDA_VERSION >= 2020 + //out << " Run time limit on kernels: "; + //if (_properties[i].kernelExecTimeoutEnabled) + // out << "Yes\n"; + //else + // out << "No\n"; + out << " Integrated: "; + if (_properties[i].integrated) + out << "Yes\n"; + else + out << "No\n"; + //out << " Support host page-locked memory mapping: "; + //if (_properties[i].canMapHostMemory) + // out << "Yes\n"; + //else + // out << "No\n"; + out << " Compute mode: "; + if (_properties[i].computeMode == hipComputeModeDefault) + out << "Default\n"; // multiple threads can use device +//#if CUDA_VERSION >= 8000 +// else if (_properties[i].computeMode == hipComputeModeExclusiveProcess) +//#else + else if (_properties[i].computeMode == hipComputeModeExclusive) +//#endif + out << "Exclusive\n"; // only thread can use device + else if (_properties[i].computeMode == hipComputeModeProhibited) + out << "Prohibited\n"; // no thread can use device + //#if CUDART_VERSION >= 4000 + else if (_properties[i].computeMode == hipComputeModeExclusiveProcess) + out << "Exclusive Process\n"; // multiple threads 1 process + //#endif + else + out << "Unknown\n"; + //#endif + //#if CUDA_VERSION >= 3010 + out << " Concurrent kernel execution: "; + if (_properties[i].concurrentKernels) + out << "Yes\n"; + else + out << "No\n"; + //out << " Device has ECC support enabled: "; + //if (_properties[i].ECCEnabled) + // out << "Yes\n"; + //else + // out << "No\n"; + //#endif + } +} + +} + +#endif diff --git a/lib/gpu/geryon/hip_kernel.h b/lib/gpu/geryon/hip_kernel.h new file mode 100644 index 0000000000..654eb44772 --- /dev/null +++ b/lib/gpu/geryon/hip_kernel.h @@ -0,0 +1,298 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_KERNEL +#define HIP_KERNEL + + +#include +#include "hip_device.h" +#include +#include +#include + +namespace ucl_hip { + +class UCL_Texture; +template class UCL_D_Vec; +template class UCL_D_Mat; +template class UCL_Vector; +template class UCL_Matrix; +#define UCL_MAX_KERNEL_ARGS 256 + +/// Class storing 1 or more kernel functions from a single string or file +class UCL_Program { + UCL_Device* _device_ptr; + public: + inline UCL_Program(UCL_Device &device) { _device_ptr = &device; _cq=device.cq(); } + inline UCL_Program(UCL_Device &device, const void *program, + const char *flags="", std::string *log=NULL) { + _device_ptr = &device; _cq=device.cq(); + init(device); + load_string(program,flags,log); + } + + inline ~UCL_Program() {} + + /// Initialize the program with a device + inline void init(UCL_Device &device) { _device_ptr = &device; _cq=device.cq(); } + + /// Clear any data associated with program + /** \note Must call init() after each clear **/ + inline void clear() { } + + /// Load a program from a file and compile with flags + inline int load(const char *filename, const char *flags="", std::string *log=NULL) { + std::ifstream in(filename); + if (!in || in.is_open()==false) { + #ifndef UCL_NO_EXIT + std::cerr << "UCL Error: Could not open kernel file: " + << filename << std::endl; + UCL_GERYON_EXIT; + #endif + return UCL_FILE_NOT_FOUND; + } + + std::string program((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + in.close(); + return load_string(program.c_str(),flags,log); + } + + /// Load a program from a string and compile with flags + inline int load_string(const void *program, const char *flags="", std::string *log=NULL) { + return _device_ptr->load_module(program, _module, log); + } + + friend class UCL_Kernel; + private: + hipModule_t _module; + hipStream_t _cq; + friend class UCL_Texture; +}; + +/// Class for dealing with CUDA Driver kernels +class UCL_Kernel { + public: + UCL_Kernel() : _dimensions(1), _num_args(0) { + _num_blocks[0]=0; + } + + UCL_Kernel(UCL_Program &program, const char *function) : + _dimensions(1), _num_args(0) { + _num_blocks[0]=0; + set_function(program,function); + _cq=program._cq; + } + + ~UCL_Kernel() {} + + /// Clear any function associated with the kernel + inline void clear() { } + + /// Get the kernel function from a program + /** \ret UCL_ERROR_FLAG (UCL_SUCCESS, UCL_FILE_NOT_FOUND, UCL_ERROR) **/ + inline int set_function(UCL_Program &program, const char *function) { + hipError_t err=hipModuleGetFunction(&_kernel,program._module,function); + if (err!=hipSuccess) { + #ifndef UCL_NO_EXIT + std::cerr << "UCL Error: Could not find function: " << function + << " in program.\n"; + UCL_GERYON_EXIT; + #endif + return UCL_FUNCTION_NOT_FOUND; + } + _cq=program._cq; + return UCL_SUCCESS; + } + + /// Set the kernel argument. + /** If not a device pointer, this must be repeated each time the argument + * changes + * \note To set kernel parameter i (i>0), parameter i-1 must be set **/ + template + inline void set_arg(const unsigned index, const dtype * const arg) { + if (index==_num_args) + add_arg(arg); + else if (index<_num_args){ + assert(0==1); // not implemented + } + else + assert(0==1); // Must add kernel parameters in sequential order + } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_D_Vec * const arg) + { set_arg(&arg->begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_D_Mat * const arg) + { set_arg(&arg->begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_Vector * const arg) + { set_arg(&arg->device.begin()); } + + /// Set a geryon container as a kernel argument. + template + inline void set_arg(const UCL_Matrix * const arg) + { set_arg(&arg->device.begin()); } + + /// Add a kernel argument. + inline void add_arg(const hipDeviceptr_t* const arg) { + add_arg((void**)arg); + } + + /// Add a kernel argument. + template + inline void add_arg(const dtype* const arg) { + const auto old_size = _hip_kernel_args.size(); + const auto aligned_size = (old_size+alignof(dtype)-1) & ~(alignof(dtype)-1); + const auto arg_size = sizeof(dtype); + _hip_kernel_args.resize(aligned_size + arg_size); + *((dtype*)(&_hip_kernel_args[aligned_size])) = *arg; + _num_args++; + if (_num_args>UCL_MAX_KERNEL_ARGS) assert(0==1); + } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_D_Vec * const arg) + { add_arg(&arg->begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_D_Mat * const arg) + { add_arg(&arg->begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_Vector * const arg) + { add_arg(&arg->device.begin()); } + + /// Add a geryon container as a kernel argument. + template + inline void add_arg(const UCL_Matrix * const arg) + { add_arg(&arg->device.begin()); } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks, const size_t block_size) { + _dimensions=1; + _num_blocks[0]=num_blocks; + _num_blocks[1]=1; + _num_blocks[2]=1; + + _block_size[0]=block_size; + _block_size[1]=1; + _block_size[2]=1; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue for the kernel is changed to cq **/ + inline void set_size(const size_t num_blocks, const size_t block_size, + command_queue &cq) + { _cq=cq; set_size(num_blocks,block_size); } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y) { + _dimensions=2; + _num_blocks[0]=num_blocks_x; + _num_blocks[1]=num_blocks_y; + _num_blocks[2]=1; + + _block_size[0]=block_size_x; + _block_size[1]=block_size_y; + _block_size[2]=1; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue for the kernel is changed to cq **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y, + command_queue &cq) + {_cq=cq; set_size(num_blocks_x, num_blocks_y, block_size_x, block_size_y);} + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, + const size_t block_size_y, const size_t block_size_z) { + _dimensions=2; + _num_blocks[0]=num_blocks_x; + _num_blocks[1]=num_blocks_y; + _num_blocks[2]=1; + + _block_size[0]=block_size_x; + _block_size[1]=block_size_y; + _block_size[2]=block_size_z; + } + + /// Set the number of thread blocks and the number of threads in each block + /** \note This should be called before any arguments have been added + \note The default command queue is used for the kernel execution **/ + inline void set_size(const size_t num_blocks_x, const size_t num_blocks_y, + const size_t block_size_x, const size_t block_size_y, + const size_t block_size_z, command_queue &cq) { + _cq=cq; + set_size(num_blocks_x, num_blocks_y, block_size_x, block_size_y, + block_size_z); + } + + /// Run the kernel in the default command queue + inline void run() { + size_t args_size = _hip_kernel_args.size(); + void *config[] = { + HIP_LAUNCH_PARAM_BUFFER_POINTER, (void*)_hip_kernel_args.data(), + HIP_LAUNCH_PARAM_BUFFER_SIZE, &args_size, + HIP_LAUNCH_PARAM_END + }; + const auto res = hipModuleLaunchKernel(_kernel,_num_blocks[0],_num_blocks[1], + _num_blocks[2],_block_size[0],_block_size[1], + _block_size[2],0,_cq, NULL, config); + CU_SAFE_CALL(res); +//#endif + } + + /// Clear any arguments associated with the kernel + inline void clear_args() { + _num_args=0; + _hip_kernel_args.clear(); + } + + /// Return the default command queue/stream associated with this data + inline command_queue & cq() { return _cq; } + /// Change the default command queue associated with matrix + inline void cq(command_queue &cq_in) { _cq=cq_in; } + #include "ucl_arg_kludge.h" + + private: + hipFunction_t _kernel; + hipStream_t _cq; + unsigned _dimensions; + unsigned _num_blocks[3]; + unsigned _num_args; + friend class UCL_Texture; + + unsigned _block_size[3]; + std::vector _hip_kernel_args; +}; + +} // namespace + +#endif + diff --git a/lib/gpu/geryon/hip_macros.h b/lib/gpu/geryon/hip_macros.h new file mode 100644 index 0000000000..9c9971b896 --- /dev/null +++ b/lib/gpu/geryon/hip_macros.h @@ -0,0 +1,83 @@ +#ifndef HIP_MACROS_H +#define HIP_MACROS_H + +#include +#include +#include + +//#if CUDA_VERSION >= 3020 +#define CUDA_INT_TYPE size_t +//#else +//#define CUDA_INT_TYPE unsigned +//#endif + +#ifdef MPI_GERYON +#include "mpi.h" +#define NVD_GERYON_EXIT do { \ + int is_final; \ + MPI_Finalized(&is_final); \ + if (!is_final) \ + MPI_Abort(MPI_COMM_WORLD,-1); \ + } while(0) +#else +#define NVD_GERYON_EXIT assert(0==1) +#endif + +#ifndef UCL_GERYON_EXIT +#define UCL_GERYON_EXIT NVD_GERYON_EXIT +#endif + +#ifdef UCL_DEBUG +#define UCL_SYNC_DEBUG +#define UCL_DESTRUCT_CHECK +#endif + +#ifndef UCL_NO_API_CHECK + +#define CU_SAFE_CALL_NS( call ) do { \ + hipError_t err = call; \ + if( hipSuccess != err) { \ + fprintf(stderr, "HIP runtime error %d in call at file '%s' in line %i.\n", \ + err, __FILE__, __LINE__ ); \ + NVD_GERYON_EXIT; \ + } } while (0) + +#ifdef UCL_SYNC_DEBUG + +#define CU_SAFE_CALL( call ) do { \ + CU_SAFE_CALL_NS( call ); \ + hipError_t err=hipCtxSynchronize(); \ + if( hipSuccess != err) { \ + fprintf(stderr, "HIP runtime error %d in file '%s' in line %i.\n", \ + err, __FILE__, __LINE__ ); \ + NVD_GERYON_EXIT; \ + } } while (0) + +#else + +#define CU_SAFE_CALL( call ) CU_SAFE_CALL_NS( call ) + +#endif + +#else // not DEBUG + +// void macros for performance reasons +#define CU_SAFE_CALL_NS( call ) call +#define CU_SAFE_CALL( call) call + +#endif + +#ifdef UCL_DESTRUCT_CHECK + +#define CU_DESTRUCT_CALL( call) CU_SAFE_CALL( call) +#define CU_DESTRUCT_CALL_NS( call) CU_SAFE_CALL_NS( call) + +#else + +#define CU_DESTRUCT_CALL( call) call +#define CU_DESTRUCT_CALL_NS( call) call + +#endif + +#endif + diff --git a/lib/gpu/geryon/hip_mat.h b/lib/gpu/geryon/hip_mat.h new file mode 100644 index 0000000000..d9bbb4e521 --- /dev/null +++ b/lib/gpu/geryon/hip_mat.h @@ -0,0 +1,43 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +/*! \file */ + +#ifndef HIP_MAT_H +#define HIP_MAT_H + + +#include +#include "hip_memory.h" + +/// Namespace for CUDA Driver routines +namespace ucl_hip { + +#define _UCL_MAT_ALLOW +#define _UCL_DEVICE_PTR_MAT +#include "ucl_basemat.h" +#include "ucl_h_vec.h" +#include "ucl_h_mat.h" +#include "ucl_d_vec.h" +#include "ucl_d_mat.h" +#include "ucl_s_obj_help.h" +#include "ucl_vector.h" +#include "ucl_matrix.h" +#undef _UCL_DEVICE_PTR_MAT +#undef _UCL_MAT_ALLOW + +#define UCL_COPY_ALLOW +#include "ucl_copy.h" +#undef UCL_COPY_ALLOW + +#define UCL_PRINT_ALLOW +#include "ucl_print.h" +#undef UCL_PRINT_ALLOW + +} // namespace ucl_cudadr + +#endif diff --git a/lib/gpu/geryon/hip_memory.h b/lib/gpu/geryon/hip_memory.h new file mode 100644 index 0000000000..13f60ad939 --- /dev/null +++ b/lib/gpu/geryon/hip_memory.h @@ -0,0 +1,279 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_MEMORY_H +#define HIP_MEMORY_H + + +#include +#include +#include +#include +#include "hip_macros.h" +#include "hip_device.h" +#include "ucl_types.h" + +namespace ucl_hip { + +// -------------------------------------------------------------------------- +// - API Specific Types +// -------------------------------------------------------------------------- +//typedef dim3 ucl_kernel_dim; + +#ifdef __HIP_PLATFORM_NVCC__ +typedef enum hipArray_Format { + HIP_AD_FORMAT_UNSIGNED_INT8 = 0x01, + HIP_AD_FORMAT_UNSIGNED_INT16 = 0x02, + HIP_AD_FORMAT_UNSIGNED_INT32 = 0x03, + HIP_AD_FORMAT_SIGNED_INT8 = 0x08, + HIP_AD_FORMAT_SIGNED_INT16 = 0x09, + HIP_AD_FORMAT_SIGNED_INT32 = 0x0a, + HIP_AD_FORMAT_HALF = 0x10, + HIP_AD_FORMAT_FLOAT = 0x20 +}hipArray_Format; +#endif + +// -------------------------------------------------------------------------- +// - API SPECIFIC DEVICE POINTERS +// -------------------------------------------------------------------------- +typedef hipDeviceptr_t device_ptr; + +// -------------------------------------------------------------------------- +// - HOST MEMORY ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline int _host_alloc(mat_type &mat, copy_type &cm, const size_t n, + const enum UCL_MEMOPT kind, const enum UCL_MEMOPT kind2){ + hipError_t err=hipSuccess; + if (kind==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (kind==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _host_alloc(mat_type &mat, UCL_Device &dev, const size_t n, + const enum UCL_MEMOPT kind, const enum UCL_MEMOPT kind2){ + hipError_t err=hipSuccess; + if (kind==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (kind==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + mat.cq()=dev.cq(); + return UCL_SUCCESS; +} + +template +inline void _host_free(mat_type &mat) { + if (mat.kind()==UCL_VIEW) + return; + else if (mat.kind()!=UCL_NOT_PINNED) + CU_DESTRUCT_CALL(hipHostFree(mat.begin())); + else + free(mat.begin()); +} + +template +inline int _host_resize(mat_type &mat, const size_t n) { + _host_free(mat); + hipError_t err=hipSuccess; + if (mat.kind()==UCL_NOT_PINNED) + *(mat.host_ptr())=(typename mat_type::data_type*)malloc(n); + else if (mat.kind()==UCL_WRITE_ONLY) + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocWriteCombined); + else + err=hipHostMalloc((void **)mat.host_ptr(),n,hipHostMallocDefault); + if (err!=hipSuccess || *(mat.host_ptr())==NULL) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +// -------------------------------------------------------------------------- +// - DEVICE MEMORY ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline int _device_alloc(mat_type &mat, copy_type &cm, const size_t n, + const enum UCL_MEMOPT kind) { + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, UCL_Device &dev, const size_t n, + const enum UCL_MEMOPT kind) { + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=dev.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, copy_type &cm, const size_t rows, + const size_t cols, size_t &pitch, + const enum UCL_MEMOPT kind) { + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=cm.cq(); + return UCL_SUCCESS; +} + +template +inline int _device_alloc(mat_type &mat, UCL_Device &d, const size_t rows, + const size_t cols, size_t &pitch, + const enum UCL_MEMOPT kind) { + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + mat.cq()=d.cq(); + return UCL_SUCCESS; +} + +template +inline void _device_free(mat_type &mat) { + if (mat.kind()!=UCL_VIEW){ + CU_DESTRUCT_CALL(hipFree((void*)mat.cbegin())); + } +} + +template +inline int _device_resize(mat_type &mat, const size_t n) { + _device_free(mat); + hipError_t err=hipMalloc((void**)&mat.cbegin(),n); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +template +inline int _device_resize(mat_type &mat, const size_t rows, + const size_t cols, size_t &pitch) { + _device_free(mat); + hipError_t err; + size_t upitch; + err=hipMallocPitch((void**)&mat.cbegin(),&upitch, + cols*sizeof(typename mat_type::data_type),rows); + pitch=static_cast(upitch); + if (err!=hipSuccess) + return UCL_MEMORY_ERROR; + return UCL_SUCCESS; +} + +inline void _device_view(hipDeviceptr_t *ptr, hipDeviceptr_t &in) { + *ptr=in; +} + +template +inline void _device_view(hipDeviceptr_t *ptr, numtyp *in) { + *ptr=0; +} + +inline void _device_view(hipDeviceptr_t *ptr, hipDeviceptr_t &in, + const size_t offset, const size_t numsize) { + *ptr=(hipDeviceptr_t)(((char*)in)+offset*numsize); +} + +template +inline void _device_view(hipDeviceptr_t *ptr, numtyp *in, + const size_t offset, const size_t numsize) { + *ptr=0; +} + +// -------------------------------------------------------------------------- +// - DEVICE IMAGE ALLOCATION ROUTINES +// -------------------------------------------------------------------------- +template +inline void _device_image_alloc(mat_type &mat, copy_type &cm, const size_t rows, + const size_t cols) { + assert(0==1); +} + +template +inline void _device_image_alloc(mat_type &mat, UCL_Device &d, const size_t rows, + const size_t cols) { + assert(0==1); +} + +template +inline void _device_image_free(mat_type &mat) { + assert(0==1); +} + +// -------------------------------------------------------------------------- +// - ZERO ROUTINES +// -------------------------------------------------------------------------- +inline void _host_zero(void *ptr, const size_t n) { + memset(ptr,0,n); +} + +template +inline void _device_zero(mat_type &mat, const size_t n, command_queue &cq) { + CU_SAFE_CALL(hipMemsetAsync((void*)mat.cbegin(),0,n,cq)); +} + + +// -------------------------------------------------------------------------- +// - MEMCPY ROUTINES +// -------------------------------------------------------------------------- + + +template +hipMemcpyKind _memcpy_kind(mat1 &dst, const mat2 &src){ + assert(mat1::MEM_TYPE < 2 && mat2::MEM_TYPE < 2); + return (hipMemcpyKind)((1 - mat2::MEM_TYPE)*2 + (1 - mat1::MEM_TYPE)); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const mat2 &src, const size_t n) { + CU_SAFE_CALL(hipMemcpy((void*)dst.begin(), (void*)src.begin(), n, _memcpy_kind(dst, src))); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const mat2 &src, const size_t n, hipStream_t &cq) { + CU_SAFE_CALL(hipMemcpyAsync((void*)dst.begin(), (void*)src.begin(), n, _memcpy_kind(dst, src), cq)); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const size_t dpitch, const mat2 &src, + const size_t spitch, const size_t cols, + const size_t rows) { + CU_SAFE_CALL(hipMemcpy2D((void*)dst.begin(), dpitch, (void*)src.begin(), spitch, cols, rows, _memcpy_kind(dst, src))); +} + +template +inline void ucl_mv_cpy(mat1 &dst, const size_t dpitch, const mat2 &src, + const size_t spitch, const size_t cols, + const size_t rows,hipStream_t &cq) { + CU_SAFE_CALL(hipMemcpy2DAsync((void*)dst.begin(), dpitch, (void*)src.begin(), spitch, cols, rows, _memcpy_kind(dst, src), cq)); +} + +} // namespace ucl_cudart + +#endif + diff --git a/lib/gpu/geryon/hip_texture.h b/lib/gpu/geryon/hip_texture.h new file mode 100644 index 0000000000..e7aa4e1461 --- /dev/null +++ b/lib/gpu/geryon/hip_texture.h @@ -0,0 +1,113 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_TEXTURE +#define HIP_TEXTURE + + +#include +#include "hip_kernel.h" +#include "hip_mat.h" + +namespace ucl_hip { + +#ifdef __HIP_PLATFORM_NVCC__ +inline hipError_t hipModuleGetTexRef(CUtexref* texRef, hipModule_t hmod, const char* name){ + return hipCUResultTohipError(cuModuleGetTexRef(texRef, hmod, name)); +} +inline hipError_t hipTexRefSetFormat(CUtexref tex, hipArray_Format fmt, int NumPackedComponents) { + return hipCUResultTohipError(cuTexRefSetFormat(tex, (CUarray_format)fmt, NumPackedComponents )); +} +inline hipError_t hipTexRefSetAddress(size_t* offset, CUtexref tex, hipDeviceptr_t devPtr, size_t size) { + return hipCUResultTohipError(cuTexRefSetAddress(offset, tex, devPtr, size)); +} +#endif + +/// Class storing a texture reference +class UCL_Texture { + public: + UCL_Texture() {} + ~UCL_Texture() {} + /// Construct with a specified texture reference + inline UCL_Texture(UCL_Program &prog, const char *texture_name) + { get_texture(prog,texture_name); } + /// Set the texture reference for this object + inline void get_texture(UCL_Program &prog, const char *texture_name) + { + #ifdef __HIP_PLATFORM_NVCC__ + CU_SAFE_CALL(hipModuleGetTexRef(&_tex, prog._module, texture_name)); + #else + size_t _global_var_size; + CU_SAFE_CALL(hipModuleGetGlobal(&_device_ptr_to_global_var, &_global_var_size, prog._module, texture_name)); + #endif + } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_D_Vec &vec, const unsigned numel) + { _bind_float(vec,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_D_Mat &vec, const unsigned numel) + { _bind_float(vec,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_Vector &vec, const unsigned numel) + { _bind_float(vec.device,numel); } + + /// Bind a float array where each fetch grabs a vector of length numel + template + inline void bind_float(UCL_Matrix &vec, const unsigned numel) + { _bind_float(vec.device,numel); } + + /// Unbind the texture reference from the memory allocation + inline void unbind() { } + + /// Make a texture reference available to kernel + inline void allow(UCL_Kernel &kernel) { + //#if CUDA_VERSION < 4000 + //CU_SAFE_CALL(cuParamSetTexRef(kernel._kernel, CU_PARAM_TR_DEFAULT, _tex)); + //#endif + } + + private: +#ifdef __HIP_PLATFORM_NVCC__ + CUtexref _tex; +#else + void* _device_ptr_to_global_var; +#endif + friend class UCL_Kernel; + + template + inline void _bind_float(mat_typ &vec, const unsigned numel) { + #ifdef UCL_DEBUG + assert(numel!=0 && numel<5); + #endif + +#ifdef __HIP_PLATFORM_NVCC__ + if (vec.element_size()==sizeof(float)) + CU_SAFE_CALL(hipTexRefSetFormat(_tex, HIP_AD_FORMAT_FLOAT, numel)); + else { + if (numel>2) + CU_SAFE_CALL(hipTexRefSetFormat(_tex, HIP_AD_FORMAT_SIGNED_INT32, numel)); + else + CU_SAFE_CALL(hipTexRefSetFormat(_tex,HIP_AD_FORMAT_SIGNED_INT32,numel*2)); + } + CU_SAFE_CALL(hipTexRefSetAddress(NULL, _tex, vec.cbegin(), vec.numel()*vec.element_size())); +#else + void* data_ptr = (void*)vec.cbegin(); + CU_SAFE_CALL(hipMemcpyHtoD(hipDeviceptr_t(_device_ptr_to_global_var), &data_ptr, sizeof(void*))); +#endif + } +}; + +} // namespace + +#endif + diff --git a/lib/gpu/geryon/hip_timer.h b/lib/gpu/geryon/hip_timer.h new file mode 100644 index 0000000000..3be0b8cfd6 --- /dev/null +++ b/lib/gpu/geryon/hip_timer.h @@ -0,0 +1,107 @@ +/* ----------------------------------------------------------------------- + Copyright (2010) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the Simplified BSD License. + ----------------------------------------------------------------------- */ + +#ifndef HIP_TIMER_H +#define HIP_TIMER_H + + +#include +#include "hip_macros.h" +#include "hip_device.h" + +namespace ucl_hip { + +/// Class for timing CUDA Driver events +class UCL_Timer { + public: + inline UCL_Timer() : _total_time(0.0f), _initialized(false) { } + inline UCL_Timer(UCL_Device &dev) : _total_time(0.0f), _initialized(false) + { init(dev); } + + inline ~UCL_Timer() { clear(); } + + /// Clear any data associated with timer + /** \note init() must be called to reuse timer after a clear() **/ + inline void clear() { + if (_initialized) { + CU_DESTRUCT_CALL(hipEventDestroy(start_event)); + CU_DESTRUCT_CALL(hipEventDestroy(stop_event)); + _initialized=false; + _total_time=0.0; + } + } + + /// Initialize default command queue for timing + inline void init(UCL_Device &dev) { init(dev, dev.cq()); } + + /// Initialize command queue for timing + inline void init(UCL_Device &dev, command_queue &cq) { + clear(); + _cq=cq; + _initialized=true; + CU_SAFE_CALL( hipEventCreateWithFlags(&start_event,0) ); + CU_SAFE_CALL( hipEventCreateWithFlags(&stop_event,0) ); + } + + /// Start timing on command queue + inline void start() { CU_SAFE_CALL(hipEventRecord(start_event,_cq)); } + + /// Stop timing on command queue + inline void stop() { CU_SAFE_CALL(hipEventRecord(stop_event,_cq)); } + + /// Block until the start event has been reached on device + inline void sync_start() + { CU_SAFE_CALL(hipEventSynchronize(start_event)); } + + /// Block until the stop event has been reached on device + inline void sync_stop() + { CU_SAFE_CALL(hipEventSynchronize(stop_event)); } + + /// Set the time elapsed to zero (not the total_time) + inline void zero() { + CU_SAFE_CALL(hipEventRecord(start_event,_cq)); + CU_SAFE_CALL(hipEventRecord(stop_event,_cq)); + } + + /// Set the total time to zero + inline void zero_total() { _total_time=0.0; } + + /// Add time from previous start and stop to total + /** Forces synchronization **/ + inline double add_to_total() + { double t=time(); _total_time+=t; return t/1000.0; } + + /// Add a user specified time to the total (ms) + inline void add_time_to_total(const double t) { _total_time+=t; } + + /// Return the time (ms) of last start to stop - Forces synchronization + inline double time() { + float timer; + CU_SAFE_CALL(hipEventSynchronize(stop_event)); + CU_SAFE_CALL( hipEventElapsedTime(&timer,start_event,stop_event) ); + return timer; + } + + /// Return the time (s) of last start to stop - Forces synchronization + inline double seconds() { return time()/1000.0; } + + /// Return the total time in ms + inline double total_time() { return _total_time; } + + /// Return the total time in seconds + inline double total_seconds() { return _total_time/1000.0; } + + private: + hipEvent_t start_event, stop_event; + hipStream_t _cq; + double _total_time; + bool _initialized; +}; + +} // namespace + +#endif diff --git a/lib/gpu/geryon/ucl_get_devices.cpp b/lib/gpu/geryon/ucl_get_devices.cpp index 1fa758fb46..b8dfc6f7b1 100644 --- a/lib/gpu/geryon/ucl_get_devices.cpp +++ b/lib/gpu/geryon/ucl_get_devices.cpp @@ -36,6 +36,11 @@ using namespace ucl_cudadr; using namespace ucl_cudart; #endif +#ifdef UCL_HIP +#include "hip_device.h" +using namespace ucl_hip; +#endif + int main(int argc, char** argv) { UCL_Device cop; std::cout << "Found " << cop.num_platforms() << " platform(s).\n"; diff --git a/lib/gpu/lal_answer.cpp b/lib/gpu/lal_answer.cpp index aa6d33d334..95d40c0d0a 100644 --- a/lib/gpu/lal_answer.cpp +++ b/lib/gpu/lal_answer.cpp @@ -179,13 +179,15 @@ double AnswerT::energy_virial(double *eatom, double **vatom, if (_eflag) { for (int i=0; i<_inum; i++) evdwl+=engv[i]; - if (_ef_atom) - if (_ilist==NULL) + if (_ef_atom) { + if (_ilist==NULL) { for (int i=0; i<_inum; i++) eatom[i]+=engv[i]; - else + } else { for (int i=0; i<_inum; i++) eatom[_ilist[i]]+=engv[i]; + } + } vstart=_inum; } if (_vflag) { @@ -193,7 +195,7 @@ double AnswerT::energy_virial(double *eatom, double **vatom, for (int j=0; j<6; j++) { for (int i=vstart; i +#include +#endif + namespace LAMMPS_AL { #define AtomT Atom @@ -70,6 +75,26 @@ bool AtomT::alloc(const int nall) { } #endif + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + size_t temp_storage_bytes = 0; + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, sort_out_keys, sort_out_keys, sort_out_values, sort_out_values, _max_atoms)) + return false; + if(sort_out_size < _max_atoms){ + if (sort_out_keys ) hipFree(sort_out_keys); + if (sort_out_values) hipFree(sort_out_values); + hipMalloc(&sort_out_keys , _max_atoms * sizeof(unsigned)); + hipMalloc(&sort_out_values, _max_atoms * sizeof(int )); + sort_out_size = _max_atoms; + } + if(temp_storage_bytes > sort_temp_storage_size){ + if(sort_temp_storage) hipFree(sort_temp_storage); + hipMalloc(&sort_temp_storage, temp_storage_bytes); + sort_temp_storage_size = temp_storage_bytes; + } + } + #endif + // --------------------------- Device allocations int gpu_bytes=0; success=success && (x.alloc(_max_atoms*4,*dev,UCL_WRITE_ONLY, @@ -184,6 +209,27 @@ bool AtomT::add_fields(const bool charge, const bool rot, return false; } #endif + + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + size_t temp_storage_bytes = 0; + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(nullptr, temp_storage_bytes, sort_out_keys, sort_out_keys, sort_out_values, sort_out_values, _max_atoms)) + return false; + if(sort_out_size < _max_atoms){ + if (sort_out_keys ) hipFree(sort_out_keys); + if (sort_out_values) hipFree(sort_out_values); + hipMalloc(&sort_out_keys , _max_atoms * sizeof(unsigned)); + hipMalloc(&sort_out_values, _max_atoms * sizeof(int )); + sort_out_size = _max_atoms; + } + if(temp_storage_bytes > sort_temp_storage_size){ + if(sort_temp_storage) hipFree(sort_temp_storage); + hipMalloc(&sort_temp_storage, temp_storage_bytes); + sort_temp_storage_size = temp_storage_bytes; + } + } + #endif + success=success && (dev_particle_id.alloc(_max_atoms,*dev, UCL_READ_ONLY)==UCL_SUCCESS); gpu_bytes+=dev_particle_id.row_bytes(); @@ -275,6 +321,19 @@ void AtomT::clear_resize() { if (_gpu_nbor==1) cudppDestroyPlan(sort_plan); #endif + #ifdef USE_HIP_DEVICE_SORT + if (_gpu_nbor==1) { + if(sort_out_keys) hipFree(sort_out_keys); + if(sort_out_values) hipFree(sort_out_values); + if(sort_temp_storage) hipFree(sort_temp_storage); + sort_out_keys = nullptr; + sort_out_values = nullptr; + sort_temp_storage = nullptr; + sort_temp_storage_size = 0; + sort_out_size = 0; + } + #endif + if (_gpu_nbor==2) { host_particle_id.clear(); host_cell_id.clear(); @@ -326,6 +385,22 @@ void AtomT::sort_neighbor(const int num_atoms) { UCL_GERYON_EXIT; } #endif + + #ifdef USE_HIP_DEVICE_SORT + if(sort_out_size < num_atoms){ + printf("AtomT::sort_neighbor: invalid temp buffer size\n"); + UCL_GERYON_EXIT; + } + if(hipSuccess != hipcub::DeviceRadixSort::SortPairs(sort_temp_storage, sort_temp_storage_size, (unsigned *)dev_cell_id.begin(), sort_out_keys, (int *)dev_particle_id.begin(), sort_out_values, num_atoms)){ + printf("AtomT::sort_neighbor: DeviceRadixSort error\n"); + UCL_GERYON_EXIT; + } + if(hipSuccess != hipMemcpy((unsigned *)dev_cell_id.begin(), sort_out_keys , num_atoms*sizeof(unsigned), hipMemcpyDeviceToDevice) || + hipSuccess != hipMemcpy((int *) dev_particle_id.begin(), sort_out_values, num_atoms*sizeof(int ), hipMemcpyDeviceToDevice)){ + printf("AtomT::sort_neighbor: copy output error\n"); + UCL_GERYON_EXIT; + } + #endif } #ifdef GPU_CAST diff --git a/lib/gpu/lal_atom.cu b/lib/gpu/lal_atom.cu index 28ff31c566..1418459301 100644 --- a/lib/gpu/lal_atom.cu +++ b/lib/gpu/lal_atom.cu @@ -11,9 +11,9 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_atom.h b/lib/gpu/lal_atom.h index 57880d7ca9..e39740d6c8 100644 --- a/lib/gpu/lal_atom.h +++ b/lib/gpu/lal_atom.h @@ -29,6 +29,11 @@ using namespace ucl_opencl; #include "geryon/nvc_mat.h" #include "geryon/nvc_kernel.h" using namespace ucl_cudart; +#elif defined(USE_HIP) +#include "geryon/hip_timer.h" +#include "geryon/hip_mat.h" +#include "geryon/hip_kernel.h" +using namespace ucl_hip; #else #include "geryon/nvd_timer.h" #include "geryon/nvd_mat.h" @@ -477,6 +482,14 @@ class Atom { CUDPPConfiguration sort_config; CUDPPHandle sort_plan; #endif + + #ifdef USE_HIP_DEVICE_SORT + unsigned* sort_out_keys = nullptr; + int* sort_out_values = nullptr; + void* sort_temp_storage = nullptr; + size_t sort_temp_storage_size = 0; + size_t sort_out_size = 0; + #endif }; } diff --git a/lib/gpu/lal_aux_fun1.h b/lib/gpu/lal_aux_fun1.h index 47a216ff6f..5b7150d950 100644 --- a/lib/gpu/lal_aux_fun1.h +++ b/lib/gpu/lal_aux_fun1.h @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_base_atomic.h b/lib/gpu/lal_base_atomic.h index da17bd928d..c97f42c50e 100644 --- a/lib/gpu/lal_base_atomic.h +++ b/lib/gpu/lal_base_atomic.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_charge.h b/lib/gpu/lal_base_charge.h index 29554ebfd5..b6d3e9e3f8 100644 --- a/lib/gpu/lal_base_charge.h +++ b/lib/gpu/lal_base_charge.h @@ -25,6 +25,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_dipole.h b/lib/gpu/lal_base_dipole.h index eecdcd3aab..856b69b56b 100644 --- a/lib/gpu/lal_base_dipole.h +++ b/lib/gpu/lal_base_dipole.h @@ -23,6 +23,8 @@ #ifdef USE_OPENCL #include "geryon/ocl_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_dpd.h b/lib/gpu/lal_base_dpd.h index b46de2197d..5d1573c1a9 100644 --- a/lib/gpu/lal_base_dpd.h +++ b/lib/gpu/lal_base_dpd.h @@ -23,6 +23,8 @@ #ifdef USE_OPENCL #include "geryon/ocl_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_ellipsoid.h b/lib/gpu/lal_base_ellipsoid.h index d4bdd9e969..dc1e624a2f 100644 --- a/lib/gpu/lal_base_ellipsoid.h +++ b/lib/gpu/lal_base_ellipsoid.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_base_three.h b/lib/gpu/lal_base_three.h index 96ca928edc..38af78b78e 100644 --- a/lib/gpu/lal_base_three.h +++ b/lib/gpu/lal_base_three.h @@ -24,6 +24,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_beck.cu b/lib/gpu/lal_beck.cu index 7d72128b5f..f24132b9a2 100644 --- a/lib/gpu/lal_beck.cu +++ b/lib/gpu/lal_beck.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_born.cu b/lib/gpu/lal_born.cu index 0ca7fea5fe..f9fea6d618 100644 --- a/lib/gpu/lal_born.cu +++ b/lib/gpu/lal_born.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_born_coul_long.cu b/lib/gpu/lal_born_coul_long.cu index 71e5e0ae50..14e644b45a 100644 --- a/lib/gpu/lal_born_coul_long.cu +++ b/lib/gpu/lal_born_coul_long.cu @@ -11,17 +11,17 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_born_coul_long_cs.cu b/lib/gpu/lal_born_coul_long_cs.cu index b3e79d9ec8..6f04fcea94 100644 --- a/lib/gpu/lal_born_coul_long_cs.cu +++ b/lib/gpu/lal_born_coul_long_cs.cu @@ -11,17 +11,18 @@ // // begin : June 2018 // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" + #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else @@ -108,7 +109,7 @@ __kernel void k_born_coul_long_cs(const __global numtyp4 *restrict x_, numtyp rsq = delx*delx+dely*dely+delz*delz; int mtype=itype*lj_types+jtype; - if (rsq pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_born_coul_wolf_cs.cu b/lib/gpu/lal_born_coul_wolf_cs.cu index 847387bfe8..b957b8be69 100644 --- a/lib/gpu/lal_born_coul_wolf_cs.cu +++ b/lib/gpu/lal_born_coul_wolf_cs.cu @@ -11,17 +11,17 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_buck.cu b/lib/gpu/lal_buck.cu index c1e1c7d7e2..0f9044cefc 100644 --- a/lib/gpu/lal_buck.cu +++ b/lib/gpu/lal_buck.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_buck_coul.cu b/lib/gpu/lal_buck_coul.cu index 6f0d414825..163c8e4362 100644 --- a/lib/gpu/lal_buck_coul.cu +++ b/lib/gpu/lal_buck_coul.cu @@ -11,17 +11,17 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_buck_coul_long.cu b/lib/gpu/lal_buck_coul_long.cu index da3237a31f..b1bbf67bc2 100644 --- a/lib/gpu/lal_buck_coul_long.cu +++ b/lib/gpu/lal_buck_coul_long.cu @@ -11,17 +11,17 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_charmm_long.cu b/lib/gpu/lal_charmm_long.cu index 244131f833..4e9802f368 100644 --- a/lib/gpu/lal_charmm_long.cu +++ b/lib/gpu/lal_charmm_long.cu @@ -11,17 +11,17 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_colloid.cu b/lib/gpu/lal_colloid.cu index 28a9809b19..4983142aa0 100644 --- a/lib/gpu/lal_colloid.cu +++ b/lib/gpu/lal_colloid.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_coul.cu b/lib/gpu/lal_coul.cu index 503e674c81..03fc568c77 100644 --- a/lib/gpu/lal_coul.cu +++ b/lib/gpu/lal_coul.cu @@ -11,17 +11,17 @@ // // begin : // email : ndtrung@umich.edu -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_debye.cu b/lib/gpu/lal_coul_debye.cu index 464a1b18de..e7f0b97e23 100644 --- a/lib/gpu/lal_coul_debye.cu +++ b/lib/gpu/lal_coul_debye.cu @@ -11,17 +11,17 @@ // // begin : // email : ndtrung@umich.edu -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_dsf.cu b/lib/gpu/lal_coul_dsf.cu index 82c44cd382..190fb5b7fd 100644 --- a/lib/gpu/lal_coul_dsf.cu +++ b/lib/gpu/lal_coul_dsf.cu @@ -11,17 +11,17 @@ // // begin : 8/15/2012 // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_long.cu b/lib/gpu/lal_coul_long.cu index 365195e00c..7adcdbbabc 100644 --- a/lib/gpu/lal_coul_long.cu +++ b/lib/gpu/lal_coul_long.cu @@ -11,17 +11,17 @@ // // begin : July 2011 // email : a.kohlmeyer@temple.edu -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_coul_long_cs.cu b/lib/gpu/lal_coul_long_cs.cu index 3c34666131..85c9d84bdb 100644 --- a/lib/gpu/lal_coul_long_cs.cu +++ b/lib/gpu/lal_coul_long_cs.cu @@ -11,17 +11,17 @@ // // begin : June 2018 // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_device.cpp b/lib/gpu/lal_device.cpp index 5bd306ea5b..aaf74ed28c 100644 --- a/lib/gpu/lal_device.cpp +++ b/lib/gpu/lal_device.cpp @@ -268,7 +268,7 @@ int DeviceT::init(Answer &ans, const bool charge, gpu_nbor=1; else if (_gpu_mode==Device::GPU_HYB_NEIGH) gpu_nbor=2; - #ifndef USE_CUDPP + #if !defined(USE_CUDPP) && !defined(USE_HIP_DEVICE_SORT) if (gpu_nbor==1) gpu_nbor=2; #endif @@ -341,7 +341,7 @@ int DeviceT::init_nbor(Neighbor *nbor, const int nlocal, gpu_nbor=1; else if (_gpu_mode==Device::GPU_HYB_NEIGH) gpu_nbor=2; - #ifndef USE_CUDPP + #if !defined(USE_CUDPP) && !defined(USE_HIP_DEVICE_SORT) if (gpu_nbor==1) gpu_nbor=2; #endif @@ -712,7 +712,7 @@ int DeviceT::compile_kernels() { gpu_lib_data.update_host(false); _ptx_arch=static_cast(gpu_lib_data[0])/100.0; - #ifndef USE_OPENCL + #if !(defined(USE_OPENCL) || defined(USE_HIP)) if (_ptx_arch>gpu->arch() || floor(_ptx_arch)arch())) return -4; #endif diff --git a/lib/gpu/lal_device.cu b/lib/gpu/lal_device.cu index 37d0758845..afc7a0b988 100644 --- a/lib/gpu/lal_device.cu +++ b/lib/gpu/lal_device.cu @@ -13,7 +13,7 @@ // email : brownw@ornl.gov // *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_dipole_lj.cu b/lib/gpu/lal_dipole_lj.cu index 745bdb7f27..a3ed0d8d40 100644 --- a/lib/gpu/lal_dipole_lj.cu +++ b/lib/gpu/lal_dipole_lj.cu @@ -11,18 +11,18 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dipole_lj_sf.cu b/lib/gpu/lal_dipole_lj_sf.cu index 9847e84823..8032ae82ed 100644 --- a/lib/gpu/lal_dipole_lj_sf.cu +++ b/lib/gpu/lal_dipole_lj_sf.cu @@ -11,19 +11,19 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dipole_long_lj.cu b/lib/gpu/lal_dipole_long_lj.cu index f888dece9b..3aafba43aa 100644 --- a/lib/gpu/lal_dipole_long_lj.cu +++ b/lib/gpu/lal_dipole_long_lj.cu @@ -11,18 +11,18 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( mu_tex,float4); #else -texture pos_tex; -texture q_tex; -texture mu_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture_2d( mu_tex,int4); #endif #else diff --git a/lib/gpu/lal_dpd.cu b/lib/gpu/lal_dpd.cu index 462401ad70..a29e04fc7f 100644 --- a/lib/gpu/lal_dpd.cu +++ b/lib/gpu/lal_dpd.cu @@ -11,16 +11,16 @@ // // begin : Jan 15, 2014 // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture vel_tex; +_texture( pos_tex,float4); +_texture( vel_tex,float4); #else -texture pos_tex; -texture vel_tex; +_texture_2d( pos_tex,int4); +_texture_2d( vel_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_eam.cu b/lib/gpu/lal_eam.cu index 13440b7d45..b22ce7b575 100644 --- a/lib/gpu/lal_eam.cu +++ b/lib/gpu/lal_eam.cu @@ -11,29 +11,29 @@ // // begin : // email : brownw@ornl.gov nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture fp_tex; -texture rhor_sp1_tex; -texture rhor_sp2_tex; -texture frho_sp1_tex; -texture frho_sp2_tex; -texture z2r_sp1_tex; -texture z2r_sp2_tex; +_texture( pos_tex,float4); +_texture( fp_tex,float); +_texture( rhor_sp1_tex,float4); +_texture( rhor_sp2_tex,float4); +_texture( frho_sp1_tex,float4); +_texture( frho_sp2_tex,float4); +_texture( z2r_sp1_tex,float4); +_texture( z2r_sp2_tex,float4); #else -texture pos_tex; -texture fp_tex; -texture rhor_sp1_tex; -texture rhor_sp2_tex; -texture frho_sp1_tex; -texture frho_sp2_tex; -texture z2r_sp1_tex; -texture z2r_sp2_tex; +_texture( pos_tex,int4); +_texture( fp_tex,int2); +_texture( rhor_sp1_tex,int4); +_texture( rhor_sp2_tex,int4); +_texture( frho_sp1_tex,int4); +_texture( frho_sp2_tex,int4); +_texture( z2r_sp1_tex,int4); +_texture( z2r_sp2_tex,int4); #endif #else diff --git a/lib/gpu/lal_ellipsoid_extra.h b/lib/gpu/lal_ellipsoid_extra.h index 71668f5e02..e6122c7404 100644 --- a/lib/gpu/lal_ellipsoid_extra.h +++ b/lib/gpu/lal_ellipsoid_extra.h @@ -18,12 +18,14 @@ enum{SPHERE_SPHERE,SPHERE_ELLIPSE,ELLIPSE_SPHERE,ELLIPSE_ELLIPSE}; -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex, quat_tex; +_texture( pos_tex, float4); +_texture( quat_tex,float4); #else -texture pos_tex, quat_tex; +_texture_2d( pos_tex,int4); +_texture_2d( quat_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_ellipsoid_nbor.cu b/lib/gpu/lal_ellipsoid_nbor.cu index cac77f5dd3..5ad935ba9b 100644 --- a/lib/gpu/lal_ellipsoid_nbor.cu +++ b/lib/gpu/lal_ellipsoid_nbor.cu @@ -9,16 +9,15 @@ // This file is part of the LAMMPS Accelerator Library (LAMMPS_AL) // __________________________________________________________________________ // -// begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ @@ -53,8 +52,8 @@ __kernel void kernel_nbor(const __global numtyp4 *restrict x_, int itype=fast_mul(iw,ntypes); int newj=0; for ( ; nbor pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_gayberne.cu b/lib/gpu/lal_gayberne.cu index cd1ee59fc6..c9d0353ca8 100644 --- a/lib/gpu/lal_gayberne.cu +++ b/lib/gpu/lal_gayberne.cu @@ -11,9 +11,9 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_gayberne_lj.cu b/lib/gpu/lal_gayberne_lj.cu index 7925b72784..fdf40720aa 100644 --- a/lib/gpu/lal_gayberne_lj.cu +++ b/lib/gpu/lal_gayberne_lj.cu @@ -11,9 +11,9 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_lj.cu b/lib/gpu/lal_lj.cu index 5838ac95cf..7297a287e6 100644 --- a/lib/gpu/lal_lj.cu +++ b/lib/gpu/lal_lj.cu @@ -11,14 +11,14 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj96.cu b/lib/gpu/lal_lj96.cu index 8dd63ef920..c602e7555e 100644 --- a/lib/gpu/lal_lj96.cu +++ b/lib/gpu/lal_lj96.cu @@ -11,14 +11,14 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_class2_long.cu b/lib/gpu/lal_lj_class2_long.cu index 41ceca35d7..65f0bf993c 100644 --- a/lib/gpu/lal_lj_class2_long.cu +++ b/lib/gpu/lal_lj_class2_long.cu @@ -11,17 +11,17 @@ // // begin : Mon May 16 2011 // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul.cu b/lib/gpu/lal_lj_coul.cu index 5c7f0da46f..afbb972942 100644 --- a/lib/gpu/lal_lj_coul.cu +++ b/lib/gpu/lal_lj_coul.cu @@ -11,17 +11,17 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_debye.cu b/lib/gpu/lal_lj_coul_debye.cu index 91b105b3da..053fbeccc8 100644 --- a/lib/gpu/lal_lj_coul_debye.cu +++ b/lib/gpu/lal_lj_coul_debye.cu @@ -11,17 +11,17 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_long.cu b/lib/gpu/lal_lj_coul_long.cu index 0e25bb2dbc..ac3479421f 100644 --- a/lib/gpu/lal_lj_coul_long.cu +++ b/lib/gpu/lal_lj_coul_long.cu @@ -11,17 +11,17 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_coul_msm.cu b/lib/gpu/lal_lj_coul_msm.cu index 3f73c6f47d..a3c36eed85 100644 --- a/lib/gpu/lal_lj_coul_msm.cu +++ b/lib/gpu/lal_lj_coul_msm.cu @@ -11,21 +11,21 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; -texture gcons_tex; -texture dgcons_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); +_texture( gcons_tex,float); +_texture( dgcons_tex,float); #else -texture pos_tex; -texture q_tex; -texture gcons_tex; -texture dgcons_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); +_texture( gcons_tex,int2); +_texture( dgcons_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_cubic.cu b/lib/gpu/lal_lj_cubic.cu index 683c6b2aac..f93013fe75 100644 --- a/lib/gpu/lal_lj_cubic.cu +++ b/lib/gpu/lal_lj_cubic.cu @@ -11,14 +11,14 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_dsf.cu b/lib/gpu/lal_lj_dsf.cu index 323576fe77..c1bb197148 100644 --- a/lib/gpu/lal_lj_dsf.cu +++ b/lib/gpu/lal_lj_dsf.cu @@ -11,17 +11,17 @@ // // begin : 7/12/2012 // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_expand.cu b/lib/gpu/lal_lj_expand.cu index 9281ad27bd..46ed9e2a31 100644 --- a/lib/gpu/lal_lj_expand.cu +++ b/lib/gpu/lal_lj_expand.cu @@ -11,15 +11,15 @@ // // begin : // email : ibains@nvidia.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_lj_expand_coul_long.cu b/lib/gpu/lal_lj_expand_coul_long.cu index aa8f02be8c..0f0fe4c2fb 100644 --- a/lib/gpu/lal_lj_expand_coul_long.cu +++ b/lib/gpu/lal_lj_expand_coul_long.cu @@ -11,17 +11,17 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_gromacs.cu b/lib/gpu/lal_lj_gromacs.cu index 93dc3d9456..21381bef30 100644 --- a/lib/gpu/lal_lj_gromacs.cu +++ b/lib/gpu/lal_lj_gromacs.cu @@ -11,15 +11,15 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_lj_sdk.cu b/lib/gpu/lal_lj_sdk.cu index 01b2cdd18d..249b29a4b2 100644 --- a/lib/gpu/lal_lj_sdk.cu +++ b/lib/gpu/lal_lj_sdk.cu @@ -11,14 +11,14 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_lj_sdk_long.cu b/lib/gpu/lal_lj_sdk_long.cu index 5ff64b2254..6dd1829c71 100644 --- a/lib/gpu/lal_lj_sdk_long.cu +++ b/lib/gpu/lal_lj_sdk_long.cu @@ -11,17 +11,17 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_lj_tip4p_long.cpp b/lib/gpu/lal_lj_tip4p_long.cpp index f60f162d7b..5dd082df33 100644 --- a/lib/gpu/lal_lj_tip4p_long.cpp +++ b/lib/gpu/lal_lj_tip4p_long.cpp @@ -23,7 +23,7 @@ const char *lj_tip4p=0; #include "lal_lj_tip4p_long.h" #include -using namespace LAMMPS_AL; +namespace LAMMPS_AL { #define LJTIP4PLongT LJ_TIP4PLong extern Device device; @@ -381,6 +381,5 @@ int** LJTIP4PLongT::compute(const int ago, const int inum_full, } - - template class LJ_TIP4PLong; +} diff --git a/lib/gpu/lal_lj_tip4p_long.cu b/lib/gpu/lal_lj_tip4p_long.cu index c101196433..66c9a6071e 100644 --- a/lib/gpu/lal_lj_tip4p_long.cu +++ b/lib/gpu/lal_lj_tip4p_long.cu @@ -13,7 +13,7 @@ // email : thevsevak@gmail.com // *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifdef LAMMPS_SMALLBIG @@ -27,11 +27,11 @@ #define tagint int #endif #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif #else diff --git a/lib/gpu/lal_mie.cu b/lib/gpu/lal_mie.cu index 33018566eb..36ec8a496b 100644 --- a/lib/gpu/lal_mie.cu +++ b/lib/gpu/lal_mie.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_morse.cu b/lib/gpu/lal_morse.cu index 0a14071d19..d6bab1e131 100644 --- a/lib/gpu/lal_morse.cu +++ b/lib/gpu/lal_morse.cu @@ -11,15 +11,15 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else diff --git a/lib/gpu/lal_neighbor_cpu.cu b/lib/gpu/lal_neighbor_cpu.cu index d005eb9f97..f8b32e1746 100644 --- a/lib/gpu/lal_neighbor_cpu.cu +++ b/lib/gpu/lal_neighbor_cpu.cu @@ -11,9 +11,9 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #endif diff --git a/lib/gpu/lal_neighbor_gpu.cu b/lib/gpu/lal_neighbor_gpu.cu index 83692a24e4..f1da437c86 100644 --- a/lib/gpu/lal_neighbor_gpu.cu +++ b/lib/gpu/lal_neighbor_gpu.cu @@ -12,9 +12,9 @@ // // begin : // email : penwang@nvidia.com, brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifdef LAMMPS_SMALLBIG #define tagint int @@ -27,9 +27,9 @@ #define tagint int #endif #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif __kernel void calc_cell_id(const numtyp4 *restrict pos, diff --git a/lib/gpu/lal_neighbor_shared.h b/lib/gpu/lal_neighbor_shared.h index 834ee8406d..5cfc4e4767 100644 --- a/lib/gpu/lal_neighbor_shared.h +++ b/lib/gpu/lal_neighbor_shared.h @@ -24,6 +24,10 @@ using namespace ucl_opencl; #include "geryon/nvc_kernel.h" #include "geryon/nvc_texture.h" using namespace ucl_cudart; +#elif defined(USE_HIP) +#include "geryon/hip_kernel.h" +#include "geryon/hip_texture.h" +using namespace ucl_hip; #else #include "geryon/nvd_kernel.h" #include "geryon/nvd_texture.h" diff --git a/lib/gpu/lal_pppm.cu b/lib/gpu/lal_pppm.cu index 24636b9a93..6cf8aafbd0 100644 --- a/lib/gpu/lal_pppm.cu +++ b/lib/gpu/lal_pppm.cu @@ -11,17 +11,17 @@ // // begin : // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_preprocessor.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture q_tex; +_texture( pos_tex,float4); +_texture( q_tex,float); #else -texture pos_tex; -texture q_tex; +_texture_2d( pos_tex,int4); +_texture( q_tex,int2); #endif // Allow PPPM to compile without atomics for NVIDIA 1.0 cards, error diff --git a/lib/gpu/lal_pppm.h b/lib/gpu/lal_pppm.h index 65af157513..499aa265d3 100644 --- a/lib/gpu/lal_pppm.h +++ b/lib/gpu/lal_pppm.h @@ -23,6 +23,8 @@ #include "geryon/ocl_texture.h" #elif defined(USE_CUDART) #include "geryon/nvc_texture.h" +#elif defined(USE_HIP) +#include "geryon/hip_texture.h" #else #include "geryon/nvd_texture.h" #endif diff --git a/lib/gpu/lal_precision.h b/lib/gpu/lal_precision.h index d5b1b9b6c0..7f82ba18aa 100644 --- a/lib/gpu/lal_precision.h +++ b/lib/gpu/lal_precision.h @@ -24,9 +24,11 @@ struct _lgpu_int2 { int x; int y; }; +#ifndef USE_HIP #ifndef int2 #define int2 _lgpu_int2 #endif +#endif struct _lgpu_float2 { float x; float y; diff --git a/lib/gpu/lal_preprocessor.h b/lib/gpu/lal_preprocessor.h index 566a451c21..cd95355ee4 100644 --- a/lib/gpu/lal_preprocessor.h +++ b/lib/gpu/lal_preprocessor.h @@ -1,4 +1,4 @@ -// ************************************************************************** +// ************************************************************************** // preprocessor.cu // ------------------- // W. Michael Brown (ORNL) @@ -60,6 +60,150 @@ // //*************************************************************************/ +#define _texture(name, type) texture name +#define _texture_2d(name, type) texture name + +// ------------------------------------------------------------------------- +// HIP DEFINITIONS +// ------------------------------------------------------------------------- + +#ifdef USE_HIP + #include + #ifdef __HIP_PLATFORM_HCC__ + #define mul24(x, y) __mul24(x, y) + #undef _texture + #undef _texture_2d + #define _texture(name, type) __device__ type* name + #define _texture_2d(name, type) __device__ type* name + #endif + #define GLOBAL_ID_X threadIdx.x+mul24(blockIdx.x,blockDim.x) + #define GLOBAL_ID_Y threadIdx.y+mul24(blockIdx.y,blockDim.y) + #define GLOBAL_SIZE_X mul24(gridDim.x,blockDim.x); + #define GLOBAL_SIZE_Y mul24(gridDim.y,blockDim.y); + #define THREAD_ID_X threadIdx.x + #define THREAD_ID_Y threadIdx.y + #define BLOCK_ID_X blockIdx.x + #define BLOCK_ID_Y blockIdx.y + #define BLOCK_SIZE_X blockDim.x + #define BLOCK_SIZE_Y blockDim.y + #define __kernel extern "C" __global__ + #ifdef __local + #undef __local + #endif + #define __local __shared__ + #define __global + #define restrict __restrict__ + #define atom_add atomicAdd + #define ucl_inline static __inline__ __device__ + + #define THREADS_PER_ATOM 4 + #define THREADS_PER_CHARGE 8 + #define BLOCK_NBOR_BUILD 128 + #define BLOCK_PAIR 256 + #define BLOCK_BIO_PAIR 256 + #define BLOCK_ELLIPSE 128 + #define MAX_SHARED_TYPES 11 + + #ifdef _SINGLE_SINGLE + ucl_inline double shfl_xor(double var, int laneMask, int width) { + #ifdef __HIP_PLATFORM_HCC__ + return __shfl_xor(var, laneMask, width); + #else + return __shfl_xor_sync(0xffffffff, var, laneMask, width); + #endif + } + #else + ucl_inline double shfl_xor(double var, int laneMask, int width) { + int2 tmp; + tmp.x = __double2hiint(var); + tmp.y = __double2loint(var); + #ifdef __HIP_PLATFORM_HCC__ + tmp.x = __shfl_xor(tmp.x,laneMask,width); + tmp.y = __shfl_xor(tmp.y,laneMask,width); + #else + tmp.x = __shfl_xor_sync(0xffffffff, tmp.x,laneMask,width); + tmp.y = __shfl_xor_sync(0xffffffff, tmp.y,laneMask,width); + #endif + return __hiloint2double(tmp.x,tmp.y); + } + #endif + + #ifdef __HIP_PLATFORM_HCC__ + #define ARCH 600 + #define WARP_SIZE 64 + #endif + + #ifdef __HIP_PLATFORM_NVCC__ + #define ARCH __CUDA_ARCH__ + #define WARP_SIZE 32 + #endif + + #define fast_mul(X,Y) (X)*(Y) + + #define MEM_THREADS WARP_SIZE + #define PPPM_BLOCK_1D 64 + #define BLOCK_CELL_2D 8 + #define BLOCK_CELL_ID 128 + #define MAX_BIO_SHARED_TYPES 128 + + #ifdef __HIP_PLATFORM_NVCC__ + #ifdef _DOUBLE_DOUBLE + #define fetch4(ans,i,pos_tex) { \ + int4 xy = tex1Dfetch(pos_tex,i*2); \ + int4 zt = tex1Dfetch(pos_tex,i*2+1); \ + ans.x=__hiloint2double(xy.y, xy.x); \ + ans.y=__hiloint2double(xy.w, xy.z); \ + ans.z=__hiloint2double(zt.y, zt.x); \ + ans.w=__hiloint2double(zt.w, zt.z); \ + } + #define fetch(ans,i,q_tex) { \ + int2 qt = tex1Dfetch(q_tex,i); \ + ans=__hiloint2double(qt.y, qt.x); \ + } + #else + #define fetch4(ans,i,pos_tex) ans=tex1Dfetch(pos_tex, i); + #define fetch(ans,i,q_tex) ans=tex1Dfetch(q_tex,i); + #endif + #else + #ifdef _DOUBLE_DOUBLE + #define fetch4(ans,i,pos_tex) (ans=*(((double4*)pos_tex) + i)) + #define fetch(ans,i,q_tex) (ans=*(((double *) q_tex) + i)) + #else + #define fetch4(ans,i,pos_tex) (ans=*(((float4*)pos_tex) + i)) + #define fetch(ans,i,q_tex) (ans=*(((float *) q_tex) + i)) + #endif + #endif + + #ifdef _DOUBLE_DOUBLE + #define ucl_exp exp + #define ucl_powr pow + #define ucl_atan atan + #define ucl_cbrt cbrt + #define ucl_ceil ceil + #define ucl_abs fabs + #define ucl_rsqrt rsqrt + #define ucl_sqrt sqrt + #define ucl_recip(x) ((numtyp)1.0/(x)) + + #else + #define ucl_atan atanf + #define ucl_cbrt cbrtf + #define ucl_ceil ceilf + #define ucl_abs fabsf + #define ucl_recip(x) ((numtyp)1.0/(x)) + #define ucl_rsqrt rsqrtf + #define ucl_sqrt sqrtf + + #ifdef NO_HARDWARE_TRANSCENDENTALS + #define ucl_exp expf + #define ucl_powr powf + #else + #define ucl_exp __expf + #define ucl_powr __powf + #endif + #endif +#endif + // ------------------------------------------------------------------------- // CUDA DEFINITIONS // ------------------------------------------------------------------------- diff --git a/lib/gpu/lal_re_squared.cu b/lib/gpu/lal_re_squared.cu index e238734074..8852a46913 100644 --- a/lib/gpu/lal_re_squared.cu +++ b/lib/gpu/lal_re_squared.cu @@ -11,9 +11,9 @@ // // begin : Fri May 06 2011 // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_re_squared_lj.cu b/lib/gpu/lal_re_squared_lj.cu index d69dae2461..112a4db8d9 100644 --- a/lib/gpu/lal_re_squared_lj.cu +++ b/lib/gpu/lal_re_squared_lj.cu @@ -11,9 +11,9 @@ // // begin : Fri May 06 2011 // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_ellipsoid_extra.h" #endif diff --git a/lib/gpu/lal_soft.cu b/lib/gpu/lal_soft.cu index 831b986725..5df34e7b1d 100644 --- a/lib/gpu/lal_soft.cu +++ b/lib/gpu/lal_soft.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_sw.cu b/lib/gpu/lal_sw.cu index 3b6de5a683..6076c55283 100644 --- a/lib/gpu/lal_sw.cu +++ b/lib/gpu/lal_sw.cu @@ -11,21 +11,21 @@ // // begin : Tue March 26, 2013 // email : brownw@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture sw1_tex; -texture sw2_tex; -texture sw3_tex; +_texture( pos_tex,float4); +_texture( sw1_tex,float4); +_texture( sw2_tex,float4); +_texture( sw3_tex,float4); #else -texture pos_tex; -texture sw1_tex; -texture sw2_tex; -texture sw3_tex; +_texture_2d( pos_tex,int4); +_texture( sw1_tex,int4); +_texture( sw2_tex,int4); +_texture( sw3_tex,int4); #endif #else diff --git a/lib/gpu/lal_table.cu b/lib/gpu/lal_table.cu index 971b56d96e..0cf0de2af0 100644 --- a/lib/gpu/lal_table.cu +++ b/lib/gpu/lal_table.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_tersoff.cu b/lib/gpu/lal_tersoff.cu index 2e29ca721b..2e68215ff8 100644 --- a/lib/gpu/lal_tersoff.cu +++ b/lib/gpu/lal_tersoff.cu @@ -11,25 +11,25 @@ // // begin : Thu April 17, 2014 // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_extra.h b/lib/gpu/lal_tersoff_extra.h index 47d16678f0..7ee29751b7 100644 --- a/lib/gpu/lal_tersoff_extra.h +++ b/lib/gpu/lal_tersoff_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_EXTRA_H #define LAL_TERSOFF_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_tersoff_mod.cu b/lib/gpu/lal_tersoff_mod.cu index c85f5e08ca..c26c17969f 100644 --- a/lib/gpu/lal_tersoff_mod.cu +++ b/lib/gpu/lal_tersoff_mod.cu @@ -11,25 +11,25 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_mod_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_mod_extra.h b/lib/gpu/lal_tersoff_mod_extra.h index a130d98488..fb658cb0da 100644 --- a/lib/gpu/lal_tersoff_mod_extra.h +++ b/lib/gpu/lal_tersoff_mod_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_MOD_EXTRA_H #define LAL_TERSOFF_MOD_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_tersoff_zbl.cu b/lib/gpu/lal_tersoff_zbl.cu index b574a529c0..b97f9247d0 100644 --- a/lib/gpu/lal_tersoff_zbl.cu +++ b/lib/gpu/lal_tersoff_zbl.cu @@ -11,27 +11,27 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_tersoff_zbl_extra.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; -texture ts6_tex; +_texture( pos_tex,float4); +_texture( ts1_tex,float4); +_texture( ts2_tex,float4); +_texture( ts3_tex,float4); +_texture( ts4_tex,float4); +_texture( ts5_tex,float4); +_texture( ts6_tex,float4); #else -texture pos_tex; -texture ts1_tex; -texture ts2_tex; -texture ts3_tex; -texture ts4_tex; -texture ts5_tex; -texture ts6_tex; +_texture_2d( pos_tex,int4); +_texture( ts1_tex,int4); +_texture( ts2_tex,int4); +_texture( ts3_tex,int4); +_texture( ts4_tex,int4); +_texture( ts5_tex,int4); +_texture( ts6_tex,int4); #endif #else diff --git a/lib/gpu/lal_tersoff_zbl_extra.h b/lib/gpu/lal_tersoff_zbl_extra.h index 32c05a3716..9e5bcb10b4 100644 --- a/lib/gpu/lal_tersoff_zbl_extra.h +++ b/lib/gpu/lal_tersoff_zbl_extra.h @@ -16,7 +16,7 @@ #ifndef LAL_TERSOFF_ZBL_EXTRA_H #define LAL_TERSOFF_ZBL_EXTRA_H -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #else #endif diff --git a/lib/gpu/lal_ufm.cu b/lib/gpu/lal_ufm.cu index 51c4df3b5b..03d1e85bdf 100644 --- a/lib/gpu/lal_ufm.cu +++ b/lib/gpu/lal_ufm.cu @@ -10,17 +10,17 @@ This file is part of the LAMMPS Accelerator Library (LAMMPS_AL) __________________________________________________________________________ - begin : + begin : email : pl.rodolfo@gmail.com dekoning@ifi.unicamp.br ***************************************************************************/ -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ @@ -31,10 +31,10 @@ __kernel void k_ufm(const __global numtyp4 *restrict x_, const __global numtyp4 *restrict uf3, const int lj_types, const __global numtyp *restrict sp_lj, - const __global int * dev_nbor, - const __global int * dev_packed, - __global acctyp4 *restrict ans, - __global acctyp *restrict engv, + const __global int * dev_nbor, + const __global int * dev_packed, + __global acctyp4 *restrict ans, + __global acctyp *restrict engv, const int eflag, const int vflag, const int inum, const int nbor_pitch, const int t_per_atom) { int tid, ii, offset; @@ -46,19 +46,19 @@ __kernel void k_ufm(const __global numtyp4 *restrict x_, acctyp virial[6]; for (int i=0; i<6; i++) virial[i]=(acctyp)0; - + if (ii0) uf3[tid]=uf3_in[tid]; } - + acctyp energy=(acctyp)0; acctyp4 f; f.x=(acctyp)0; f.y=(acctyp)0; f.z=(acctyp)0; @@ -132,7 +132,7 @@ __kernel void k_ufm_fast(const __global numtyp4 *restrict x_, virial[i]=(acctyp)0; __syncthreads(); - + if (ii pos_tex; -texture param1_tex; -texture param2_tex; -texture param3_tex; -texture param4_tex; -texture param5_tex; +_texture( pos_tex,float4); +_texture( param1_tex,float4); +_texture( param2_tex,float4); +_texture( param3_tex,float4); +_texture( param4_tex,float4); +_texture( param5_tex,float4); #else -texture pos_tex; -texture param1_tex; -texture param2_tex; -texture param3_tex; -texture param4_tex; -texture param5_tex; +_texture_2d( pos_tex,int4); +_texture( param1_tex,int4); +_texture( param2_tex,int4); +_texture( param3_tex,int4); +_texture( param4_tex,int4); +_texture( param5_tex,int4); #endif #else @@ -290,7 +290,7 @@ __kernel void k_vashishta(const __global numtyp4 *restrict x_, if (eflag>0) energy += (param3_bigh*reta+vc2-vc3-param3_bigw*r6inv-r*param3_dvrc+param3_c0); - + if (vflag>0) { virial[0] += delx*delx*force; virial[1] += dely*dely*force; @@ -471,13 +471,13 @@ __kernel void k_vashishta_three_center(const __global numtyp4 *restrict x_, numtyp rsq1 = delr1x*delr1x+delr1y*delr1y+delr1z*delr1z; int ijparam=elem2param[itype*nelements*nelements+jtype*nelements+jtype]; - + numtyp4 param4_ijparam; fetch4(param4_ijparam,ijparam,param4_tex); param_r0sq_ij=param4_ijparam.x; if (rsq1 > param_r0sq_ij) continue; // still keep this for neigh no and tpa > 1 param_gamma_ij=param4_ijparam.y; param_r0_ij=param4_ijparam.w; - + int nbor_k,k_end; if (dev_packed==dev_nbor) { nbor_k=nborj_start-offset_j+offset_k; @@ -619,7 +619,7 @@ __kernel void k_vashishta_three_end(const __global numtyp4 *restrict x_, param_gamma_ij=param4_ijparam.y; param_r0_ij = param4_ijparam.w; - + int nbor_k,numk; if (dev_nbor==dev_packed) { if (gpu_nbor) nbor_k=j+nbor_pitch; @@ -665,14 +665,14 @@ __kernel void k_vashishta_three_end(const __global numtyp4 *restrict x_, if (rsq2 < param_r0sq_ik) { param_gamma_ik=param4_ikparam.y; param_r0_ik=param4_ikparam.w; - + int ijkparam=elem2param[jtype*nelements*nelements+itype*nelements+ktype]; //jik numtyp4 param5_ijkparam; fetch4(param5_ijkparam,ijkparam,param5_tex); param_bigc_ijk=param5_ijkparam.x; param_costheta_ijk=param5_ijkparam.y; param_bigb_ijk=param5_ijkparam.z; param_big2b_ijk=param5_ijkparam.w; - + numtyp fjx, fjy, fjz; //if (evatom==0) { threebody_half(delr1x,delr1y,delr1z); @@ -774,7 +774,7 @@ __kernel void k_vashishta_three_end_vatom(const __global numtyp4 *restrict x_, param_gamma_ij=param4_ijparam.y; param_r0_ij=param4_ijparam.w; - + int nbor_k,numk; if (dev_nbor==dev_packed) { if (gpu_nbor) nbor_k=j+nbor_pitch; @@ -827,7 +827,7 @@ __kernel void k_vashishta_three_end_vatom(const __global numtyp4 *restrict x_, param_costheta_ijk=param5_ijkparam.y; param_bigb_ijk=param5_ijkparam.z; param_big2b_ijk=param5_ijkparam.w; - + numtyp fjx, fjy, fjz, fkx, fky, fkz; threebody(delr1x,delr1y,delr1z,eflag,energy); diff --git a/lib/gpu/lal_yukawa.cu b/lib/gpu/lal_yukawa.cu index a8d637ec97..62bc013dc6 100644 --- a/lib/gpu/lal_yukawa.cu +++ b/lib/gpu/lal_yukawa.cu @@ -11,14 +11,14 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/gpu/lal_yukawa_colloid.cu b/lib/gpu/lal_yukawa_colloid.cu index a3cbbbc11c..30b458fec7 100644 --- a/lib/gpu/lal_yukawa_colloid.cu +++ b/lib/gpu/lal_yukawa_colloid.cu @@ -11,17 +11,17 @@ // // begin : // email : nguyentd@ornl.gov -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; -texture rad_tex; +_texture( pos_tex,float4); +_texture( rad_tex,float); #else -texture pos_tex; -texture rad_tex; +_texture_2d( pos_tex,int4); +_texture( rad_tex,int2); #endif #else diff --git a/lib/gpu/lal_zbl.cu b/lib/gpu/lal_zbl.cu index 33c850e134..2539c0ddd7 100644 --- a/lib/gpu/lal_zbl.cu +++ b/lib/gpu/lal_zbl.cu @@ -11,14 +11,14 @@ // // begin : // email : ndactrung@gmail.com -// ***************************************************************************/ +// *************************************************************************** -#ifdef NV_KERNEL +#if defined(NV_KERNEL) || defined(USE_HIP) #include "lal_aux_fun1.h" #ifndef _DOUBLE_DOUBLE -texture pos_tex; +_texture( pos_tex,float4); #else -texture pos_tex; +_texture_2d( pos_tex,int4); #endif #else #define pos_tex x_ diff --git a/lib/kokkos/BUILD.md b/lib/kokkos/BUILD.md new file mode 100644 index 0000000000..63dbf7fdb2 --- /dev/null +++ b/lib/kokkos/BUILD.md @@ -0,0 +1,310 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Installing and Using Kokkos + +## Kokkos Philosophy +Kokkos provides a modern CMake style build system. +As C++ continues to develop for C++20 and beyond, CMake is likely to provide the most robust support +for C++. Applications heavily leveraging Kokkos are strongly encouraged to use a CMake build system. + +You can either use Kokkos as an installed package (encouraged) or use Kokkos in-tree in your project. +Modern CMake is exceedingly simple at a high-level (with the devil in the details). +Once Kokkos is installed In your `CMakeLists.txt` simply use: +```` +find_package(Kokkos REQUIRED) +```` +Then for every executable or library in your project: +```` +target_link_libraries(myTarget Kokkos::kokkos) +```` +That's it! There is no checking Kokkos preprocessor, compiler, or linker flags. +Kokkos propagates all the necessary flags to your project. +This means not only is linking to Kokkos easy, but Kokkos itself can actually configure compiler and linker flags for *your* +project. If building in-tree, there is no `find_package` and you link with `target_link_libraries(kokkos)`. + + +## Configuring CMake +A very basic installation is done with: +```` +cmake ${srcdir} \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_INSTALL_PREFIX=${my_install_folder} +```` +which builds and installed a default Kokkos when you run `make install`. +There are numerous device backends, options, and architecture-specific optimizations that can be configured, e.g. +```` +cmake ${srcdir} \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_INSTALL_PREFIX=${my_install_folder} \ + -DKokkos_ENABLE_OPENMP=On +```` +which activates the OpenMP backend. All of the options controlling device backends, options, architectures, and third-party libraries (TPLs) are given below. + +## Platform-specific Problems + +### Cray + +* The Cray compiler wrappers do static linking by default. This seems to break the Kokkos build. You will likely need to set the environment variable `CRAYPE_LINK_TYPE=dynamic` in order to link correctly. Kokkos warns during configure if this is missing. +* The Cray compiler identifies to CMake as Clang, but it sometimes has its own flags that differ from Clang. We try to include all exceptions, but flag errors may occur in which a Clang-specific flag is passed that the Cray compiler does not recognize. + +## Spack +An alternative to manually building with the CMake is to use the Spack package manager. +To do so, download the `kokkos-spack` git repo and add to the package list: +```` +spack repo add $path-to-kokkos-spack +```` +A basic installation would be done as: +```` +spack install kokkos +```` +Spack allows options and and compilers to be tuned in the install command. +```` +spack install kokkos@3.0 %gcc@7.3.0 +openmp +```` +This example illustrates the three most common parameters to Spack: +* Variants: specified with, e.g. `+openmp`, this activates (or deactivates with, e.g. `~openmp`) certain options. +* Version: immediately following `kokkos` the `@version` can specify a particular Kokkos to build +* Compiler: a default compiler will be chosen if not specified, but an exact compiler version can be given with the `%`option. + +For a complete list of Kokkos options, run: +```` +spack info kokkos +```` +More details can be found in the kokkos-spack repository [README](https://github.com/kokkos/kokkos-spack/blob/master/README.md). + +#### Spack Development +Spack currently installs packages to a location determined by a unique hash. This hash name is not really "human readable". +Generally, Spack usage should never really require you to reference the computer-generated unique install folder. +If you must know, you can locate Spack Kokkos installations with: +```` +spack find -p kokkos ... +```` +where `...` is the unique spec identifying the particular Kokkos configuration and version. + +A better way to use Spack for doing Kokkos development is the dev-build feature of Spack. +For dev-build details, consult the kokkos-spack repository [README](https://github.com/kokkos/kokkos-spack/blob/master/README.md). + +# Kokkos Keyword Listing + +## Device Backends +Device backends can be enabled by specifying `-DKokkos_ENABLE_X`. + +* Kokkos_ENABLE_CUDA + * Whether to build CUDA backend + * BOOL Default: OFF +* Kokkos_ENABLE_HPX + * Whether to build HPX backend (experimental) + * BOOL Default: OFF +* Kokkos_ENABLE_OPENMP + * Whether to build OpenMP backend + * BOOL Default: OFF +* Kokkos_ENABLE_PTHREAD + * Whether to build Pthread backend + * BOOL Default: OFF +* Kokkos_ENABLE_SERIAL + * Whether to build serial backend + * BOOL Default: ON + +## Enable Options +Options can be enabled by specifying `-DKokkos_ENABLE_X`. + +* Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION + * Whether to aggressively vectorize loops + * BOOL Default: OFF +* Kokkos_ENABLE_COMPILER_WARNINGS + * Whether to print all compiler warnings + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_CONSTEXPR + * Whether to activate experimental relaxed constexpr functions + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_LAMBDA + * Whether to activate experimental lambda features + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_LDG_INTRINSIC + * Whether to use CUDA LDG intrinsics + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE + * Whether to enable relocatable device code (RDC) for CUDA + * BOOL Default: OFF +* Kokkos_ENABLE_CUDA_UVM + * Whether to use unified memory (UM) by default for CUDA + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG + * Whether to activate extra debug features - may increase compile times + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG_BOUNDS_CHECK + * Whether to use bounds checking - will increase runtime + * BOOL Default: OFF +* Kokkos_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + * Debug check on dual views + * BOOL Default: OFF +* Kokkos_ENABLE_DEPRECATED_CODE + * Whether to enable deprecated code + * BOOL Default: OFF +* Kokkos_ENABLE_EXAMPLES + * Whether to enable building examples + * BOOL Default: OFF +* Kokkos_ENABLE_HPX_ASYNC_DISPATCH + * Whether HPX supports asynchronous dispatch + * BOOL Default: OFF +* Kokkos_ENABLE_LARGE_MEM_TESTS + * Whether to perform extra large memory tests + * BOOL_Default: OFF +* Kokkos_ENABLE_PROFILING + * Whether to create bindings for profiling tools + * BOOL Default: ON +* Kokkos_ENABLE_PROFILING_LOAD_PRINT + * Whether to print information about which profiling tools gotloaded + * BOOL Default: OFF +* Kokkos_ENABLE_TESTS + * Whether to build serial backend + * BOOL Default: OFF + +## Other Options +* Kokkos_CXX_STANDARD + * The C++ standard for Kokkos to use: c++11, c++14, c++17, or c++20. This should be given in CMake style as 11, 14, 17, or 20. + * STRING Default: 11 + +## Third-party Libraries (TPLs) +The following options control enabling TPLs: +* Kokkos_ENABLE_HPX + * Whether to enable the HPX library + * BOOL Default: OFF +* Kokkos_ENABLE_HWLOC + * Whether to enable the HWLOC library + * BOOL Default: Off +* Kokkos_ENABLE_LIBNUMA + * Whether to enable the LIBNUMA library + * BOOL Default: Off +* Kokkos_ENABLE_MEMKIND + * Whether to enable the MEMKIND library + * BOOL Default: Off +* Kokkos_ENABLE_LIBDL + * Whether to enable the LIBDL library + * BOOL Default: On +* Kokkos_ENABLE_LIBRT + * Whether to enable the LIBRT library + * BOOL Default: Off + +The following options control finding and configuring non-CMake TPLs: +* Kokkos_CUDA_DIR or CUDA_ROOT + * Location of CUDA install prefix for libraries + * PATH Default: +* Kokkos_HWLOC_DIR or HWLOC_ROOT + * Location of HWLOC install prefix + * PATH Default: +* Kokkos_LIBNUMA_DIR or LIBNUMA_ROOT + * Location of LIBNUMA install prefix + * PATH Default: +* Kokkos_MEMKIND_DIR or MEMKIND_ROOT + * Location of MEMKIND install prefix + * PATH Default: +* Kokkos_LIBDL_DIR or LIBDL_ROOT + * Location of LIBDL install prefix + * PATH Default: +* Kokkos_LIBRT_DIR or LIBRT_ROOT + * Location of LIBRT install prefix + * PATH Default: + +The following options control `find_package` paths for CMake-based TPLs: +* HPX_DIR or HPX_ROOT + * Location of HPX prefix (ROOT) or CMake config file (DIR) + * PATH Default: + +## Architecture Keywords +Architecture-specific optimizations can be enabled by specifying `-DKokkos_ARCH_X`. + +* Kokkos_ARCH_AMDAVX + * Whether to optimize for the AMDAVX architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV80 + * Whether to optimize for the ARMV80 architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV81 + * Whether to optimize for the ARMV81 architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV8_THUNDERX + * Whether to optimize for the ARMV8_THUNDERX architecture + * BOOL Default: OFF +* Kokkos_ARCH_ARMV8_TX2 + * Whether to optimize for the ARMV8_TX2 architecture + * BOOL Default: OFF +* Kokkos_ARCH_BDW + * Whether to optimize for the BDW architecture + * BOOL Default: OFF +* Kokkos_ARCH_BGQ + * Whether to optimize for the BGQ architecture + * BOOL Default: OFF +* Kokkos_ARCH_EPYC + * Whether to optimize for the EPYC architecture + * BOOL Default: OFF +* Kokkos_ARCH_HSW + * Whether to optimize for the HSW architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER30 + * Whether to optimize for the KEPLER30 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER32 + * Whether to optimize for the KEPLER32 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER35 + * Whether to optimize for the KEPLER35 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KEPLER37 + * Whether to optimize for the KEPLER37 architecture + * BOOL Default: OFF +* Kokkos_ARCH_KNC + * Whether to optimize for the KNC architecture + * BOOL Default: OFF +* Kokkos_ARCH_KNL + * Whether to optimize for the KNL architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL50 + * Whether to optimize for the MAXWELL50 architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL52 + * Whether to optimize for the MAXWELL52 architecture + * BOOL Default: OFF +* Kokkos_ARCH_MAXWELL53 + * Whether to optimize for the MAXWELL53 architecture + * BOOL Default: OFF +* Kokkos_ARCH_PASCAL60 + * Whether to optimize for the PASCAL60 architecture + * BOOL Default: OFF +* Kokkos_ARCH_PASCAL61 + * Whether to optimize for the PASCAL61 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER7 + * Whether to optimize for the POWER7 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER8 + * Whether to optimize for the POWER8 architecture + * BOOL Default: OFF +* Kokkos_ARCH_POWER9 + * Whether to optimize for the POWER9 architecture + * BOOL Default: OFF +* Kokkos_ARCH_SKX + * Whether to optimize for the SKX architecture + * BOOL Default: OFF +* Kokkos_ARCH_SNB + * Whether to optimize for the SNB architecture + * BOOL Default: OFF +* Kokkos_ARCH_TURING75 + * Whether to optimize for the TURING75 architecture + * BOOL Default: OFF +* Kokkos_ARCH_VOLTA70 + * Whether to optimize for the VOLTA70 architecture + * BOOL Default: OFF +* Kokkos_ARCH_VOLTA72 + * Whether to optimize for the VOLTA72 architecture + * BOOL Default: OFF +* Kokkos_ARCH_WSM + * Whether to optimize for the WSM architecture + * BOOL Default: OFF + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/devel/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. diff --git a/lib/kokkos/CHANGELOG.md b/lib/kokkos/CHANGELOG.md index 8d196e2c35..9595b03ff9 100644 --- a/lib/kokkos/CHANGELOG.md +++ b/lib/kokkos/CHANGELOG.md @@ -1,5 +1,112 @@ # Change Log +## [3.1.1](https://github.com/kokkos/kokkos/tree/3.1.1) (2020-04-14) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.1.00...3.1.1) + +**Fixed bugs:** + +- Fix complex_double misalignment in reduce, clang+CUDA [\#2989](https://github.com/kokkos/kokkos/issues/2989) +- Fix compilation fails when profiling disabled and CUDA enabled [\#3001](https://github.com/kokkos/kokkos/issues/3001) +- Fix cuda reduction of non-trivial scalars of size 4 [\#2990](https://github.com/kokkos/kokkos/issues/2990) +- Configure and install version file when building in Trilinos [\#2957](https://github.com/kokkos/kokkos/pull/2957) +- Fix OpenMPTarget build missing include and namespace [\#3000](https://github.com/kokkos/kokkos/issues/3000) +- fix typo in KOKKOS_SET_EXE_PROPERTY() [\#2959](https://github.com/kokkos/kokkos/issues/2959) +- Fix non-zero span subviews of zero sized subviews [\#2979](https://github.com/kokkos/kokkos/issues/2979) + +## [3.1.00](https://github.com/kokkos/kokkos/tree/3.1.00) (2020-04-14) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.0.00...3.1.00) + +**Features:** + +- HIP Support for AMD +- OpenMPTarget Support with clang +- Windows VS19 (Serial) Support [\#1533](https://github.com/kokkos/kokkos/issues/1533) + +**Implemented enhancements:** + +- generate\_makefile.bash should allow tests to be disabled [\#2886](https://github.com/kokkos/kokkos/issues/2886) +- clang/7+cuda/9 build -Werror-unused parameter error in nightly test [\#2884](https://github.com/kokkos/kokkos/issues/2884) +- ScatterView memory space is not user settable [\#2826](https://github.com/kokkos/kokkos/issues/2826) +- clang/8+cuda/10.0 build error with c++17 [\#2809](https://github.com/kokkos/kokkos/issues/2809) +- warnings.... [\#2805](https://github.com/kokkos/kokkos/issues/2805) +- Kokkos version in cpp define [\#2787](https://github.com/kokkos/kokkos/issues/2787) +- Remove Defunct QThreads Backend [\#2751](https://github.com/kokkos/kokkos/issues/2751) +- Improve Kokkos::fence behavior with multiple execution spaces [\#2659](https://github.com/kokkos/kokkos/issues/2659) +- polylithic\(?\) initialization of Kokkos [\#2658](https://github.com/kokkos/kokkos/issues/2658) +- Unnecessary\(?\) check for host execution space initialization from Cuda initialization [\#2652](https://github.com/kokkos/kokkos/issues/2652) +- Kokkos error reporting failures with CUDA GPUs in exclusive mode [\#2471](https://github.com/kokkos/kokkos/issues/2471) +- atomicMax equivalent \(and other atomics\) [\#2401](https://github.com/kokkos/kokkos/issues/2401) +- Fix alignment for Kokkos::complex [\#2255](https://github.com/kokkos/kokkos/issues/2255) +- Warnings with Cuda 10.1 [\#2206](https://github.com/kokkos/kokkos/issues/2206) +- dual view with Kokkos::ViewAllocateWithoutInitializing [\#2188](https://github.com/kokkos/kokkos/issues/2188) +- Check error code from cudaOccupancyMaxActiveBlocksPerMultiprocessor [\#2172](https://github.com/kokkos/kokkos/issues/2172) +- Add non-member Kokkos::resize/realloc for DualView [\#2170](https://github.com/kokkos/kokkos/issues/2170) +- Construct DualView without initialization [\#2046](https://github.com/kokkos/kokkos/issues/2046) +- Expose is\_assignable to determine if one view can be assigned to another [\#1936](https://github.com/kokkos/kokkos/issues/1936) +- profiling label [\#1935](https://github.com/kokkos/kokkos/issues/1935) +- team\_broadcast of bool failed on CUDA backend [\#1908](https://github.com/kokkos/kokkos/issues/1908) +- View static\_extent [\#660](https://github.com/kokkos/kokkos/issues/660) +- Misleading Kokkos::Cuda::initialize ERROR message when compiled for wrong GPU architecture [\#1944](https://github.com/kokkos/kokkos/issues/1944) +- Cryptic Error When Malloc Fails [\#2164](https://github.com/kokkos/kokkos/issues/2164) +- Drop support for intermediate standards in CMake [\#2336](https://github.com/kokkos/kokkos/issues/2336) + +**Fixed bugs:** + +- DualView sync\_device with length zero creates cuda errors [\#2946](https://github.com/kokkos/kokkos/issues/2946) +- building with nvcc and clang \(or clang based XL\) as host compiler: "Kokkos::atomic\_fetch\_min\(volatile int \*, int\)" has already been defined [\#2903](https://github.com/kokkos/kokkos/issues/2903) +- Cuda 9.1,10.1 debug builds failing due to -Werror=unused-parameter [\#2880](https://github.com/kokkos/kokkos/issues/2880) +- clang -Werror: Kokkos\_FixedBufferMemoryPool.hpp:140:28: error: unused parameter 'alloc\_size' [\#2869](https://github.com/kokkos/kokkos/issues/2869) +- intel/16.0.1, intel/17.0.1 nightly build failures with debugging enabled [\#2867](https://github.com/kokkos/kokkos/issues/2867) +- intel/16.0.1 debug build errors [\#2863](https://github.com/kokkos/kokkos/issues/2863) +- xl/16.1.1 with cpp14, openmp build, nightly test failures [\#2856](https://github.com/kokkos/kokkos/issues/2856) +- Intel nightly test failures: team\_vector [\#2852](https://github.com/kokkos/kokkos/issues/2852) +- Kokkos Views with intmax/2\\> for complex\ uses std::ostream, not std::istream [\#2313](https://github.com/kokkos/kokkos/issues/2313) +- Macros: Restrict not honored for non-intel compilers [\#1922](https://github.com/kokkos/kokkos/issues/1922) + + ## [2.9.00](https://github.com/kokkos/kokkos/tree/2.9.00) (2019-06-24) [Full Changelog](https://github.com/kokkos/kokkos/compare/2.8.00...2.9.00) @@ -174,7 +281,7 @@ ## [2.6.00](https://github.com/kokkos/kokkos/tree/2.6.00) (2018-03-07) [Full Changelog](https://github.com/kokkos/kokkos/compare/2.5.00...2.6.00) -**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.6** +**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.6** **Implemented enhancements:** @@ -218,7 +325,7 @@ ## [2.5.00](https://github.com/kokkos/kokkos/tree/2.5.00) (2017-12-15) [Full Changelog](https://github.com/kokkos/kokkos/compare/2.04.11...2.5.00) -**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.5** +**Part of the Kokkos C++ Performance Portability Programming EcoSystem 2.5** **Implemented enhancements:** diff --git a/lib/kokkos/CMakeLists.txt b/lib/kokkos/CMakeLists.txt index 236f523aec..0e2aaa1897 100644 --- a/lib/kokkos/CMakeLists.txt +++ b/lib/kokkos/CMakeLists.txt @@ -1,128 +1,237 @@ -# Is this a build as part of Trilinos? +# We want to determine if options are given with the wrong case +# In order to detect which arguments are given to compare against +# the list of valid arguments, at the beginning here we need to +# form a list of all the given variables. If it begins with any +# case of KoKkOS, we add it to the list. + + +GET_CMAKE_PROPERTY(_variableNames VARIABLES) +SET(KOKKOS_GIVEN_VARIABLES) +FOREACH (var ${_variableNames}) + STRING(TOUPPER ${var} UC_VAR) + STRING(FIND ${UC_VAR} KOKKOS IDX) + IF (${IDX} EQUAL 0) + LIST(APPEND KOKKOS_GIVEN_VARIABLES ${var}) + ENDIF() +ENDFOREACH() + +# Basic initialization (Used in KOKKOS_SETTINGS) +SET(Kokkos_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET(KOKKOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +SET(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) +SET(KOKKOS_PATH ${Kokkos_SOURCE_DIR}) +SET(KOKKOS_TOP_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# Needed to simplify syntax of if statements +CMAKE_POLICY(SET CMP0054 NEW) +# Needed to make IN_LIST a valid operator +CMAKE_POLICY(SET CMP0057 NEW) + +# Is this a build as part of Trilinos? IF(COMMAND TRIBITS_PACKAGE_DECL) - SET(KOKKOS_HAS_TRILINOS ON CACHE BOOL "") + SET(KOKKOS_HAS_TRILINOS ON) ELSE() - SET(KOKKOS_HAS_TRILINOS OFF CACHE BOOL "") + SET(KOKKOS_HAS_TRILINOS OFF) ENDIF() -IF(NOT KOKKOS_HAS_TRILINOS) - cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_pick_cxx_std.cmake) - # Define Project Name if this is a standalone build - IF(NOT DEFINED ${PROJECT_NAME}) - project(Kokkos CXX) +SET(KOKKOS_ENABLED_OPTIONS) #exported in config file +SET(KOKKOS_ENABLED_DEVICES) #exported in config file +SET(KOKKOS_ENABLED_TPLS) #exported in config file +SET(KOKKOS_ENABLED_ARCH_LIST) #exported in config file + +#These are helper flags used for sanity checks during config +#Certain features should depend on other features being configured first +SET(KOKKOS_CFG_DAG_NONE On) #sentinel to indicate no dependencies +SET(KOKKOS_CFG_DAG_DEVICES_DONE Off) +SET(KOKKOS_CFG_DAG_OPTIONS_DONE Off) +SET(KOKKOS_CFG_DAG_ARCH_DONE Off) +SET(KOKKOS_CFG_DAG_CXX_STD_DONE Off) +SET(KOKKOS_CFG_DAG_COMPILER_ID_DONE Off) +FUNCTION(KOKKOS_CFG_DEPENDS SUCCESSOR PRECURSOR) + SET(PRE_FLAG KOKKOS_CFG_DAG_${PRECURSOR}) + SET(POST_FLAG KOKKOS_CFG_DAG_${SUCCESSOR}) + IF (NOT ${PRE_FLAG}) + MESSAGE(FATAL_ERROR "Bad CMake refactor: feature ${SUCCESSOR} cannot be configured until ${PRECURSOR} is configured") ENDIF() + GLOBAL_SET(${POST_FLAG} On) +ENDFUNCTION() - # Basic initialization (Used in KOKKOS_SETTINGS) - set(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) - set(KOKKOS_PATH ${KOKKOS_SRC_PATH}) - #------------ COMPILER AND FEATURE CHECKS ------------------------------------ - include(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) - set_kokkos_cxx_compiler() - set_kokkos_cxx_standard() +LIST(APPEND CMAKE_MODULE_PATH cmake/Modules) - #------------ GET OPTIONS AND KOKKOS_SETTINGS -------------------------------- - # Add Kokkos' modules to CMake's module path. - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") +IF(NOT KOKKOS_HAS_TRILINOS) + cmake_minimum_required(VERSION 3.10 FATAL_ERROR) + set(CMAKE_DISABLE_SOURCE_CHANGES ON) + set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) + IF (Spack_WORKAROUND) + #if we are explicitly using Spack for development, + #nuke the Spack compiler + SET(SPACK_CXX $ENV{SPACK_CXX}) + IF(SPACK_CXX) + SET(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) + SET(ENV{CXX} ${SPACK_CXX}) + ENDIF() + ENDif() + IF(NOT DEFINED ${PROJECT_NAME}) + # WORKAROUND FOR HIPCC + IF(Kokkos_ENABLE_HIP) + SET(KOKKOS_INTERNAL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --amdgpu-target=gfx906") + ENDIF() + PROJECT(Kokkos CXX) + IF(Kokkos_ENABLE_HIP) + SET(CMAKE_CXX_FLAGS ${KOKKOS_INTERNAL_CMAKE_CXX_FLAGS}) + ENDIF() + ENDIF() +ENDIF() - set(KOKKOS_CMAKE_VERBOSE True) - include(${KOKKOS_SRC_PATH}/cmake/kokkos_options.cmake) +IF (NOT CMAKE_SIZEOF_VOID_P) + STRING(FIND ${CMAKE_CXX_COMPILER} nvcc_wrapper FIND_IDX) + IF (NOT FIND_IDX STREQUAL -1) + MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is CUDA linkage using nvcc_wrapper. Please ensure your CUDA environment is correctly configured.") + ELSE() + MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is linkage errors during CMake compiler validation. Please consult the CMake error log shown below for the exact error during compiler validation") + ENDIF() +ELSEIF (NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + MESSAGE(FATAL_ERROR "Kokkos assumes a 64-bit build; i.e., 8-byte pointers, but found ${CMAKE_SIZEOF_VOID_P}-byte pointers instead") +ENDIF() - include(${KOKKOS_SRC_PATH}/cmake/kokkos_settings.cmake) - #------------ GENERATE HEADER AND SOURCE FILES ------------------------------- - execute_process( - COMMAND ${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} PREFIX=${CMAKE_INSTALL_PREFIX} generate_build_settings - WORKING_DIRECTORY "${Kokkos_BINARY_DIR}" - OUTPUT_FILE ${Kokkos_BINARY_DIR}/core_src_make.out - RESULT_VARIABLE GEN_SETTINGS_RESULT - ) - if (GEN_SETTINGS_RESULT) - message(FATAL_ERROR "Kokkos settings generation failed:\n" - "${KOKKOS_SETTINGS} make -f ${KOKKOS_SRC_PATH}/cmake/Makefile.generate_cmake_settings CXX=${CMAKE_CXX_COMPILER} generate_build_settings") - endif() - include(${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake) - install(FILES ${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake DESTINATION lib/cmake/Kokkos) - install(FILES ${Kokkos_BINARY_DIR}/kokkos_generated_settings.cmake DESTINATION ${CMAKE_INSTALL_PREFIX}) - string(REPLACE " " ";" KOKKOS_TPL_INCLUDE_DIRS "${KOKKOS_GMAKE_TPL_INCLUDE_DIRS}") - string(REPLACE " " ";" KOKKOS_TPL_LIBRARY_DIRS "${KOKKOS_GMAKE_TPL_LIBRARY_DIRS}") - string(REPLACE " " ";" KOKKOS_TPL_LIBRARY_NAMES "${KOKKOS_GMAKE_TPL_LIBRARY_NAMES}") - list(REMOVE_ITEM KOKKOS_TPL_INCLUDE_DIRS "") - list(REMOVE_ITEM KOKKOS_TPL_LIBRARY_DIRS "") - list(REMOVE_ITEM KOKKOS_TPL_LIBRARY_NAMES "") - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) +set(Kokkos_VERSION_MAJOR 3) +set(Kokkos_VERSION_MINOR 1) +set(Kokkos_VERSION_PATCH 1) +set(Kokkos_VERSION "${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR}.${Kokkos_VERSION_PATCH}") +math(EXPR KOKKOS_VERSION "${Kokkos_VERSION_MAJOR} * 10000 + ${Kokkos_VERSION_MINOR} * 100 + ${Kokkos_VERSION_PATCH}") - #------------ NOW BUILD ------------------------------------------------------ - include(${KOKKOS_SRC_PATH}/cmake/kokkos_build.cmake) +IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") + MESSAGE(STATUS "Setting policy CMP0074 to use _ROOT variables") + CMAKE_POLICY(SET CMP0074 NEW) +ENDIF() - #------------ Add in Fake Tribits Handling to allow unit test builds- -------- +# Load either the real TriBITS or a TriBITS wrapper +# for certain utility functions that are universal (like GLOBAL_SET) +INCLUDE(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) - include(${KOKKOS_SRC_PATH}/cmake/tribits.cmake) +IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + #If we are building CUDA, we have tricked CMake because we declare a CXX project + #If the default C++ standard for a given compiler matches the requested + #standard, then CMake just omits the -std flag in later versions of CMake + #This breaks CUDA compilation (CUDA compiler can have a different default + #-std then the underlying host compiler by itself). Setting this variable + #forces CMake to always add the -std flag even if it thinks it doesn't need it + GLOBAL_SET(CMAKE_CXX_STANDARD_DEFAULT 98) +ENDIF() - TRIBITS_PACKAGE_DECL(Kokkos) +# These are the variables we will append to as we go +# I really wish these were regular variables +# but scoping issues can make it difficult +GLOBAL_SET(KOKKOS_COMPILE_OPTIONS) +GLOBAL_SET(KOKKOS_LINK_OPTIONS) +GLOBAL_SET(KOKKOS_CUDA_OPTIONS) +GLOBAL_SET(KOKKOS_CUDAFE_OPTIONS) +GLOBAL_SET(KOKKOS_XCOMPILER_OPTIONS) +# We need to append text here for making sure TPLs +# we import are available for an installed Kokkos +GLOBAL_SET(KOKKOS_TPL_EXPORTS) - ADD_SUBDIRECTORY(core) - ADD_SUBDIRECTORY(containers) - ADD_SUBDIRECTORY(algorithms) +# Include a set of Kokkos-specific wrapper functions that +# will either call raw CMake or TriBITS +# These are functions like KOKKOS_INCLUDE_DIRECTORIES +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tribits.cmake) + +# Check the environment and set certain variables +# to allow platform-specific checks +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_check_env.cmake) +# The build environment setup goes in the following steps +# 1) Check all the enable options. This includes checking Kokkos_DEVICES +# 2) Check the compiler ID (type and version) +# 3) Check the CXX standard and select important CXX flags +# 4) Check for any third-party libraries (TPLs) like hwloc +# 5) Check if optimizing for a particular architecture and add arch-specific flags +KOKKOS_SETUP_BUILD_ENVIRONMENT() + +# Finish off the build +# 6) Recurse into subdirectories and configure individual libraries +# 7) Export and install targets + +OPTION(BUILD_SHARED_LIBS "Build shared libraries" OFF) +# Workaround for building position independent code. +IF(BUILD_SHARED_LIBS) + SET(CMAKE_POSITION_INDEPENDENT_CODE ON) +ENDIF() + +SET(KOKKOS_EXT_LIBRARIES Kokkos::kokkos Kokkos::kokkoscore Kokkos::kokkoscontainers Kokkos::kokkosalgorithms) +SET(KOKKOS_INT_LIBRARIES kokkos kokkoscore kokkoscontainers kokkosalgorithms) +SET_PROPERTY(GLOBAL PROPERTY KOKKOS_INT_LIBRARIES ${KOKKOS_INT_LIBRARIES}) + +GET_DIRECTORY_PROPERTY(HAS_PARENT PARENT_DIRECTORY) +IF (KOKKOS_HAS_TRILINOS) + SET(TRILINOS_INCDIR ${CMAKE_INSTALL_PREFIX}/${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}) + SET(KOKKOS_HEADER_DIR ${TRILINOS_INCDIR}) + SET(KOKKOS_IS_SUBDIRECTORY TRUE) +ELSEIF(HAS_PARENT) + SET(KOKKOS_HEADER_DIR "include/kokkos") + SET(KOKKOS_IS_SUBDIRECTORY TRUE) ELSE() + SET(KOKKOS_HEADER_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + SET(KOKKOS_IS_SUBDIRECTORY FALSE) +ENDIF() + + + #------------------------------------------------------------------------------ # # A) Forward declare the package so that certain options are also defined for # subpackages -# -TRIBITS_PACKAGE_DECL(Kokkos) # ENABLE_SHADOWING_WARNINGS) +## This restores the old behavior of ProjectCompilerPostConfig.cmake +# It sets the CMAKE_CXX_FLAGS globally to those used by Kokkos +# We must do this before KOKKOS_PACKAGE_DECL +IF (KOKKOS_HAS_TRILINOS) + # Overwrite the old flags at the top-level + # Because Tribits doesn't use lists, it uses spaces for the list of CXX flags + # we have to match the annoying behavior + STRING(REPLACE ";" " " KOKKOSCORE_COMPILE_OPTIONS "${KOKKOS_COMPILE_OPTIONS}") + LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS ${KOKKOS_COMPILE_OPTIONS}) + LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS ${KOKKOS_CUDA_OPTIONS}) + FOREACH(XCOMP_FLAG ${KOKKOS_XCOMPILER_OPTIONS}) + SET(KOKKOSCORE_XCOMPILER_OPTIONS "${KOKKOSCORE_XCOMPILER_OPTIONS} -Xcompiler ${XCOMP_FLAG}") + LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS -Xcompiler ${XCOMP_FLAG}) + ENDFOREACH() + SET(KOKKOSCORE_CXX_FLAGS "${KOKKOSCORE_COMPILE_OPTIONS} ${CMAKE_CXX${KOKKOS_CXX_STANDARD}_STANDARD_COMPILE_OPTION} ${KOKKOSCORE_XCOMPILER_OPTIONS}") + IF (KOKKOS_ENABLE_CUDA) + STRING(REPLACE ";" " " KOKKOSCORE_CUDA_OPTIONS "${KOKKOS_CUDA_OPTIONS}") + FOREACH(CUDAFE_FLAG ${KOKKOS_CUDAFE_OPTIONS}) + SET(KOKKOSCORE_CUDAFE_OPTIONS "${KOKKOSCORE_CUDAFE_OPTIONS} -Xcudafe ${CUDAFE_FLAG}") + LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS -Xcudafe ${CUDAFE_FLAG}) + ENDFOREACH() + SET(KOKKOSCORE_CXX_FLAGS "${KOKKOSCORE_CXX_FLAGS} ${KOKKOSCORE_CUDA_OPTIONS} ${KOKKOSCORE_CUDAFE_OPTIONS}") + ENDIF() + # Both parent scope and this package + # In ProjectCompilerPostConfig.cmake, we capture the "global" flags Trilinos wants in + # TRILINOS_TOPLEVEL_CXX_FLAGS + SET(CMAKE_CXX_FLAGS "${TRILINOS_TOPLEVEL_CXX_FLAGS} ${KOKKOSCORE_CXX_FLAGS}" PARENT_SCOPE) + SET(CMAKE_CXX_FLAGS "${TRILINOS_TOPLEVEL_CXX_FLAGS} ${KOKKOSCORE_CXX_FLAGS}") + #CMAKE_CXX_FLAGS will get added to Kokkos and Kokkos dependencies automatically here + #These flags get set up in KOKKOS_PACKAGE_DECL, which means they + #must be configured before KOKKOS_PACKAGE_DECL + SET(KOKKOS_ALL_COMPILE_OPTIONS + $<$:${KOKKOS_ALL_COMPILE_OPTIONS}>) +ENDIF() + +KOKKOS_PACKAGE_DECL() #------------------------------------------------------------------------------ # -# B) Install Kokkos' build files +# D) Process the subpackages (subdirectories) for Kokkos # -# If using the Makefile-generated files, then need to set things up. -# Here, assume that TriBITS has been run from ProjectCompilerPostConfig.cmake -# and already generated KokkosCore_config.h and kokkos_generated_settings.cmake -# in the previously define Kokkos_GEN_DIR -# We need to copy them over to the correct place and source the cmake file - -if(NOT KOKKOS_LEGACY_TRIBITS) - set(Kokkos_GEN_DIR ${CMAKE_BINARY_DIR}) - file(COPY "${Kokkos_GEN_DIR}/KokkosCore_config.h" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" USE_SOURCE_PERMISSIONS) - install(FILES "${Kokkos_GEN_DIR}/KokkosCore_config.h" - DESTINATION include) - file(COPY "${Kokkos_GEN_DIR}/kokkos_generated_settings.cmake" - DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" USE_SOURCE_PERMISSIONS) - - include(${CMAKE_CURRENT_BINARY_DIR}/kokkos_generated_settings.cmake) - # Sources come from makefile-generated kokkos_generated_settings.cmake file - # Enable using the individual sources if needed - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) -endif () - - -#------------------------------------------------------------------------------ -# -# C) Install Kokkos' executable scripts -# - -# nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. -# Kokkos needs nvcc_wrapper in order to build. Other libraries and -# executables also need nvcc_wrapper. Thus, we need to install it. -# If the argument of DESTINATION is a relative path, CMake computes it -# as relative to ${CMAKE_INSTALL_PATH}. - -INSTALL(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper DESTINATION bin) - - -#------------------------------------------------------------------------------ -# -# D) Process the subpackages for Kokkos -# - -TRIBITS_PROCESS_SUBPACKAGES() +KOKKOS_PROCESS_SUBPACKAGES() #------------------------------------------------------------------------------ @@ -130,10 +239,39 @@ TRIBITS_PROCESS_SUBPACKAGES() # E) If Kokkos itself is enabled, process the Kokkos package # -TRIBITS_PACKAGE_DEF() +KOKKOS_PACKAGE_DEF() +KOKKOS_EXCLUDE_AUTOTOOLS_FILES() +KOKKOS_PACKAGE_POSTPROCESS() -TRIBITS_EXCLUDE_AUTOTOOLS_FILES() - -TRIBITS_PACKAGE_POSTPROCESS() +#We are ready to configure the header +CONFIGURE_FILE(cmake/KokkosCore_config.h.in KokkosCore_config.h @ONLY) +IF (NOT KOKKOS_HAS_TRILINOS) + ADD_LIBRARY(kokkos INTERFACE) + #Make sure in-tree projects can reference this as Kokkos:: + #to match the installed target names + ADD_LIBRARY(Kokkos::kokkos ALIAS kokkos) + TARGET_LINK_LIBRARIES(kokkos INTERFACE kokkoscore kokkoscontainers kokkosalgorithms) + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(kokkos) +ENDIF() +INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) + +# nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. +# Kokkos needs nvcc_wrapper in order to build. Other libraries and +# executables also need nvcc_wrapper. Thus, we need to install it. +# If the argument of DESTINATION is a relative path, CMake computes it +# as relative to ${CMAKE_INSTALL_PATH}. +INSTALL(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper DESTINATION ${CMAKE_INSTALL_BINDIR}) +INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + + +# Finally - if we are a subproject - make sure the enabled devices are visible +IF (HAS_PARENT) + FOREACH(DEV Kokkos_ENABLED_DEVICES) + #I would much rather not make these cache variables or global properties, but I can't + #make any guarantees on whether PARENT_SCOPE is good enough to make + #these variables visible where I need them + SET(Kokkos_ENABLE_${DEV} ON PARENT_SCOPE) + SET_PROPERTY(GLOBAL PROPERTY Kokkos_ENABLE_${DEV} ON) + ENDFOREACH() ENDIF() diff --git a/lib/kokkos/CONTRIBUTING.md b/lib/kokkos/CONTRIBUTING.md new file mode 100644 index 0000000000..b4f3057cef --- /dev/null +++ b/lib/kokkos/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing to Kokkos + +## Pull Requests +We actively welcome pull requests. +1. Fork the repo and create your branch from `develop`. +2. If you've added code that should be tested, add tests. +3. If you've changed APIs, update the documentation. +4. Ensure the test suite passes. + +## Issues +We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. + +## License +By contributing to Kokkos, you agree that your contributions will be licensed under the LICENSE file in the root directory of this source tree. diff --git a/lib/kokkos/Copyright.txt b/lib/kokkos/Copyright.txt index 50b76995af..5e2f8d8647 100644 --- a/lib/kokkos/Copyright.txt +++ b/lib/kokkos/Copyright.txt @@ -1,12 +1,13 @@ //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -35,6 +36,6 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER diff --git a/lib/kokkos/LICENSE b/lib/kokkos/LICENSE index c68a8a2a9f..c6f17087d5 100644 --- a/lib/kokkos/LICENSE +++ b/lib/kokkos/LICENSE @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Kokkos is licensed under 3-clause BSD terms of use: @@ -24,10 +25,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR diff --git a/lib/kokkos/Makefile.kokkos b/lib/kokkos/Makefile.kokkos index e9ad57f0ae..dc53de0a12 100644 --- a/lib/kokkos/Makefile.kokkos +++ b/lib/kokkos/Makefile.kokkos @@ -6,15 +6,20 @@ ifndef KOKKOS_PATH endif CXXFLAGS=$(CCFLAGS) -# Options: Cuda,ROCm,OpenMP,Pthreads,Qthreads,Serial +KOKKOS_VERSION_MAJOR = 3 +KOKKOS_VERSION_MINOR = 1 +KOKKOS_VERSION_PATCH = 1 +KOKKOS_VERSION = $(shell echo $(KOKKOS_VERSION_MAJOR)*10000+$(KOKKOS_VERSION_MINOR)*100+$(KOKKOS_VERSION_PATCH) | bc) + +# Options: Cuda,HIP,ROCm,OpenMP,Pthread,Serial KOKKOS_DEVICES ?= "OpenMP" -#KOKKOS_DEVICES ?= "Pthreads" +#KOKKOS_DEVICES ?= "Pthread" # Options: # Intel: KNC,KNL,SNB,HSW,BDW,SKX # NVIDIA: Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72,Turing75 # ARM: ARMv80,ARMv81,ARMv8-ThunderX,ARMv8-TX2 # IBM: BGQ,Power7,Power8,Power9 -# AMD-GPUS: Kaveri,Carrizo,Fiji,Vega +# AMD-GPUS: Vega900,Vega906 # AMD-CPUS: AMDAVX,Ryzen,EPYC KOKKOS_ARCH ?= "" # Options: yes,no @@ -23,16 +28,21 @@ KOKKOS_DEBUG ?= "no" KOKKOS_USE_TPLS ?= "" # Options: c++11,c++14,c++1y,c++17,c++1z,c++2a KOKKOS_CXX_STANDARD ?= "c++11" -# Options: aggressive_vectorization,disable_profiling,enable_deprecated_code,disable_deprecated_code,enable_large_mem_tests +# Options: aggressive_vectorization,disable_profiling,enable_deprecated_code,disable_deprecated_code,enable_large_mem_tests,disable_complex_align KOKKOS_OPTIONS ?= "" # Option for setting ETI path KOKKOS_ETI_PATH ?= ${KOKKOS_PATH}/core/src/eti KOKKOS_CMAKE ?= "no" +KOKKOS_TRIBITS ?= "no" +KOKKOS_STANDALONE_CMAKE ?= "no" # Default settings specific options. -# Options: force_uvm,use_ldg,rdc,enable_lambda +# Options: force_uvm,use_ldg,rdc,enable_lambda,enable_constexpr KOKKOS_CUDA_OPTIONS ?= "enable_lambda" +# Options: rdc +KOKKOS_HIP_OPTIONS ?= "" + # Default settings specific options. # Options: enable_async_dispatch KOKKOS_HPX_OPTIONS ?= "" @@ -47,7 +57,8 @@ kokkos_has_string=$(if $(findstring $2,$1),1,0) # Will return a 1 if /path/to/file exists kokkos_path_exists=$(if $(wildcard $1),1,0) -# Check for general settings. +# Check for general settings + KOKKOS_INTERNAL_ENABLE_DEBUG := $(call kokkos_has_string,$(KOKKOS_DEBUG),yes) KOKKOS_INTERNAL_ENABLE_CXX11 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++11) KOKKOS_INTERNAL_ENABLE_CXX14 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++14) @@ -67,6 +78,7 @@ KOKKOS_INTERNAL_OPT_RANGE_AGGRESSIVE_VECTORIZATION := $(call kokkos_has_string,$ KOKKOS_INTERNAL_DISABLE_PROFILING := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_profiling) KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_deprecated_code) KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_deprecated_code) +KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_complex_align) KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_dualview_modify_check) KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_profile_load_print) KOKKOS_INTERNAL_ENABLE_LARGE_MEM_TESTS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_large_mem_tests) @@ -74,32 +86,54 @@ KOKKOS_INTERNAL_CUDA_USE_LDG := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS), KOKKOS_INTERNAL_CUDA_USE_UVM := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),force_uvm) KOKKOS_INTERNAL_CUDA_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),rdc) KOKKOS_INTERNAL_CUDA_USE_LAMBDA := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_lambda) +KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_constexpr) KOKKOS_INTERNAL_HPX_ENABLE_ASYNC_DISPATCH := $(call kokkos_has_string,$(KOKKOS_HPX_OPTIONS),enable_async_dispatch) KOKKOS_INTERNAL_ENABLE_ETI := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_eti) +KOKKOS_INTERNAL_HIP_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_HIP_OPTIONS),rdc) # Check for Kokkos Host Execution Spaces one of which must be on. KOKKOS_INTERNAL_USE_OPENMP := $(call kokkos_has_string,$(subst OpenMPTarget,,$(KOKKOS_DEVICES)),OpenMP) KOKKOS_INTERNAL_USE_PTHREADS := $(call kokkos_has_string,$(KOKKOS_DEVICES),Pthread) -KOKKOS_INTERNAL_USE_QTHREADS := $(call kokkos_has_string,$(KOKKOS_DEVICES),Qthreads) KOKKOS_INTERNAL_USE_HPX := $(call kokkos_has_string,$(KOKKOS_DEVICES),HPX) KOKKOS_INTERNAL_USE_SERIAL := $(call kokkos_has_string,$(KOKKOS_DEVICES),Serial) ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 0) ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 0) - ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 0) - ifeq ($(KOKKOS_INTERNAL_USE_HPX), 0) - KOKKOS_INTERNAL_USE_SERIAL := 1 - endif + ifeq ($(KOKKOS_INTERNAL_USE_HPX), 0) + KOKKOS_INTERNAL_USE_SERIAL := 1 endif endif endif # Check for other Execution Spaces. KOKKOS_INTERNAL_USE_CUDA := $(call kokkos_has_string,$(KOKKOS_DEVICES),Cuda) -KOKKOS_INTERNAL_USE_ROCM := $(call kokkos_has_string,$(KOKKOS_DEVICES),ROCm) +KOKKOS_INTERNAL_USE_HIP := $(call kokkos_has_string,$(KOKKOS_DEVICES),HIP) KOKKOS_INTERNAL_USE_OPENMPTARGET := $(call kokkos_has_string,$(KOKKOS_DEVICES),OpenMPTarget) +KOKKOS_DEVICELIST = +ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1) + KOKKOS_DEVICELIST += Serial +endif +ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) + KOKKOS_DEVICELIST += OpenMP +endif +ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1) + KOKKOS_DEVICELIST += Threads +endif +ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) + KOKKOS_DEVICELIST += HPX +endif +ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) + KOKKOS_DEVICELIST += Cuda +endif +ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) + KOKKOS_DEVICELIST += HIP +endif +ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) + KOKKOS_DEVICELIST += OPENMPTARGET +endif + ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc) ifeq ($(origin CUDA_PATH), undefined) @@ -123,10 +157,11 @@ KOKKOS_INTERNAL_COMPILER_INTEL := $(call kokkos_has_string,$(KOKKOS_CXX_VE KOKKOS_INTERNAL_COMPILER_PGI := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),PGI) KOKKOS_INTERNAL_COMPILER_XL := $(strip $(shell $(CXX) -qversion 2>&1 | grep XL | wc -l)) KOKKOS_INTERNAL_COMPILER_CRAY := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep "CC-" | wc -l)) -KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l)) +KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l)>0" | bc)) KOKKOS_INTERNAL_COMPILER_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang) KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple LLVM) KOKKOS_INTERNAL_COMPILER_HCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC) +KOKKOS_INTERNAL_COMPILER_GCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),GCC) # Check Host Compiler if using NVCC through nvcc_wrapper ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) @@ -175,20 +210,20 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS), 1) KOKKOS_INTERNAL_COMPILER_WARNINGS = else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) - KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized + KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized else ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1) - KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized + KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized else ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1) - KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized + KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1) # TODO check if cray accepts GNU style warnings KOKKOS_INTERNAL_COMPILER_WARNINGS = else #gcc - KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized + KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized endif endif endif @@ -225,7 +260,12 @@ ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1) KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_IBM_XL_OMP45_WORKAROUND -qsmp=omp -qoffload -qnoeh else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) - KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_BUG_WORKAROUND_IBM_CLANG_OMP45_VIEW_INIT -fopenmp-implicit-declare-target -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp -fopenmp=libomp + #KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_BUG_WORKAROUND_IBM_CLANG_OMP45_VIEW_INIT -fopenmp-implicit-declare-target -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp -fopenmp=libomp + KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_WORKAROUND_OPENMPTARGET_CLANG -fopenmp -fopenmp=libomp + KOKKOS_INTERNAL_OPENMPTARGET_LIB := -lomptarget + else + #Assume GCC + KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -fopenmp -foffload=nvptx-none endif endif @@ -348,11 +388,8 @@ KOKKOS_INTERNAL_USE_ARCH_IBM := $(strip $(shell echo $(KOKKOS_INTERNAL_USE_ARCH_ KOKKOS_INTERNAL_USE_ARCH_AMDAVX := $(call kokkos_has_string,$(KOKKOS_ARCH),AMDAVX) KOKKOS_INTERNAL_USE_ARCH_RYZEN := $(call kokkos_has_string,$(KOKKOS_ARCH),Ryzen) KOKKOS_INTERNAL_USE_ARCH_EPYC := $(call kokkos_has_string,$(KOKKOS_ARCH),EPYC) -KOKKOS_INTERNAL_USE_ARCH_KAVERI := $(call kokkos_has_string,$(KOKKOS_ARCH),Kaveri) -KOKKOS_INTERNAL_USE_ARCH_CARRIZO := $(call kokkos_has_string,$(KOKKOS_ARCH),Carrizo) -KOKKOS_INTERNAL_USE_ARCH_FIJI := $(call kokkos_has_string,$(KOKKOS_ARCH),Fiji) -KOKKOS_INTERNAL_USE_ARCH_VEGA := $(call kokkos_has_string,$(KOKKOS_ARCH),Vega) -KOKKOS_INTERNAL_USE_ARCH_GFX901 := $(call kokkos_has_string,$(KOKKOS_ARCH),gfx901) +KOKKOS_INTERNAL_USE_ARCH_VEGA900 := $(call kokkos_has_string,$(KOKKOS_ARCH),Vega900) +KOKKOS_INTERNAL_USE_ARCH_VEGA906 := $(call kokkos_has_string,$(KOKKOS_ARCH),Vega906) # Any AVX? KOKKOS_INTERNAL_USE_ARCH_SSE42 := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_WSM)) @@ -383,10 +420,10 @@ endif # Generating the list of Flags. -#CPPFLAGS is now unused KOKKOS_CPPFLAGS = +KOKKOS_LIBDIRS = ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src -I$(KOKKOS_ETI_PATH) + KOKKOS_CPPFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src -I$(KOKKOS_ETI_PATH) endif KOKKOS_TPL_INCLUDE_DIRS = KOKKOS_TPL_LIBRARY_DIRS = @@ -399,7 +436,7 @@ endif KOKKOS_LIBS = -ldl KOKKOS_TPL_LIBRARY_NAMES += dl ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_LDFLAGS = -L$(shell pwd) + KOKKOS_LIBDIRS = -L$(shell pwd) # CXXLDFLAGS is used together with CXXFLAGS in a combined compile/link command KOKKOS_CXXLDFLAGS = -L$(shell pwd) endif @@ -425,6 +462,10 @@ tmp := $(call kokkos_append_header,'\#error "Do not include $(KOKKOS_CONFIG_HEAD tmp := $(call kokkos_append_header,'\#else') tmp := $(call kokkos_append_header,'\#define KOKKOS_CORE_CONFIG_H') tmp := $(call kokkos_append_header,'\#endif') + +tmp := $(call kokkos_append_header,"") +tmp := $(call kokkos_append_header,"\#define KOKKOS_VERSION $(KOKKOS_VERSION)") +tmp := $(call kokkos_append_header,"") tmp := $(call kokkos_append_header,"/* Execution Spaces */") @@ -437,9 +478,15 @@ ifeq ($(KOKKOS_INTERNAL_USE_ROCM), 1) tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_ROCM') tmp := $(call kokkos_append_header,'\#define KOKKOS_IMPL_ROCM_CLANG_WORKAROUND 1') endif +ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) + tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_HIP') +endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_OPENMPTARGET') + ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_WORKAROUND_OPENMPTARGET_GCC") + endif endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) @@ -450,10 +497,6 @@ ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_THREADS") endif -ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_QTHREADS") -endif - ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HPX") endif @@ -492,28 +535,38 @@ ifeq ($(KOKKOS_INTERNAL_USE_ISA_POWERPCBE), 1) tmp := $(call kokkos_append_header,"\#endif") endif +#only add the c++ standard flags if this is not CMake tmp := $(call kokkos_append_header,"/* General Settings */") ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX11), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX11_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX11") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX14), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX14_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX14") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Y), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Y_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX14") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX17), 1) +ifneq ($(KOKKOS_STANDALONE_CMAKE), yes) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX17_FLAG) +endif tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX17") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Z), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Z_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX17") endif ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX2A), 1) + #I cannot make CMake add this in a good way - so add it here KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX2A_FLAG) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CXX20") endif @@ -531,23 +584,26 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK") endif endif +ifeq ($(KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN), 0) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_COMPLEX_ALIGN") +endif ifeq ($(KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_PROFILING_LOAD_PRINT") endif ifeq ($(KOKKOS_INTERNAL_USE_HWLOC), 1) - ifneq ($(HWLOC_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(HWLOC_PATH)/include + ifneq ($(KOKKOS_CMAKE), yes) + ifneq ($(HWLOC_PATH),) + KOKKOS_CPPFLAGS += -I$(HWLOC_PATH)/include + KOKKOS_LIBDIRS += -L$(HWLOC_PATH)/lib + KOKKOS_CXXLDFLAGS += -L$(HWLOC_PATH)/lib + KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include + KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib endif - KOKKOS_LDFLAGS += -L$(HWLOC_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(HWLOC_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib + KOKKOS_LIBS += -lhwloc + KOKKOS_TPL_LIBRARY_NAMES += hwloc endif - KOKKOS_LIBS += -lhwloc - KOKKOS_TPL_LIBRARY_NAMES += hwloc tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HWLOC") endif @@ -558,17 +614,17 @@ ifeq ($(KOKKOS_INTERNAL_USE_LIBRT), 1) endif ifeq ($(KOKKOS_INTERNAL_USE_MEMKIND), 1) - ifneq ($(MEMKIND_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(MEMKIND_PATH)/include + ifneq ($(KOKKOS_CMAKE), yes) + ifneq ($(MEMKIND_PATH),) + KOKKOS_CPPFLAGS += -I$(MEMKIND_PATH)/include + KOKKOS_LIBDIRS += -L$(MEMKIND_PATH)/lib + KOKKOS_CXXLDFLAGS += -L$(MEMKIND_PATH)/lib + KOKKOS_TPL_INCLUDE_DIRS += $(MEMKIND_PATH)/include + KOKKOS_TPL_LIBRARY_DIRS += $(MEMKIND_PATH)/lib endif - KOKKOS_LDFLAGS += -L$(MEMKIND_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(MEMKIND_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(MEMKIND_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(MEMKIND_PATH)/lib + KOKKOS_LIBS += -lmemkind -lnuma + KOKKOS_TPL_LIBRARY_NAMES += memkind numa endif - KOKKOS_LIBS += -lmemkind -lnuma - KOKKOS_TPL_LIBRARY_NAMES += memkind numa tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HBWSPACE") endif @@ -580,9 +636,6 @@ ifeq ($(KOKKOS_INTERNAL_USE_HPX), 0) ifeq ($(KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEPRECATED_CODE") endif - ifeq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 0) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEPRECATED_CODE") - endif endif ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) @@ -648,6 +701,21 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) endif endif + ifeq ($(KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) + ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_NVCC_VERSION) -ge 80; echo $$?),0) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CUDA_CONSTEXPR") + KOKKOS_CXXFLAGS += -expt-relaxed-constexpr + else + $(warning Warning: Cuda relaxed constexpr support was requested but NVCC version is too low. This requires NVCC for Cuda version 8.0 or higher. Disabling relaxed constexpr support now.) + endif + endif + + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_CUDA_CONSTEXPR") + endif + endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_IMPL_CUDA_CLANG_WORKAROUND") endif @@ -936,6 +1004,14 @@ endif # Figure out the architecture flag for Cuda. ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) + KOKKOS_INTERNAL_USE_CUDA_ARCH=1 +endif +ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) + KOKKOS_INTERNAL_USE_CUDA_ARCH=1 + endif +endif +ifeq ($(KOKKOS_INTERNAL_USE_CUDA_ARCH), 1) ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-arch else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) @@ -944,7 +1020,17 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) else $(error Makefile.kokkos: CUDA is enabled but the compiler is neither NVCC nor Clang (got version string $(KOKKOS_CXX_VERSION)) ) endif + KOKKOS_INTERNAL_USE_CUDA_ARCH = 1 +endif +ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march + endif + KOKKOS_INTERNAL_USE_CUDA_ARCH = 1 +endif + +ifeq ($(KOKKOS_INTERNAL_USE_CUDA_ARCH), 1) ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER30), 1) tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER") tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER30") @@ -1012,55 +1098,49 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG) endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) + KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG) + endif + endif + endif + ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) + KOKKOS_CXXFLAGS += --expt-extended-lambda endif endif + # Figure out the architecture flag for ROCm. -ifeq ($(KOKKOS_INTERNAL_USE_ROCM), 1) +ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) # Lets start with adding architecture defines - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KAVERI), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 701") - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KAVERI") - KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx701 + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VEGA900), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_HIP 900") + tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VEGA900") + KOKKOS_INTERNAL_HIP_ARCH_FLAG := --amdgpu-target=gfx900 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_CARRIZO), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 801") - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_CARRIZO") - KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx801 + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VEGA906), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_HIP 906") + tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VEGA906") + KOKKOS_INTERNAL_HIP_ARCH_FLAG := --amdgpu-target=gfx906 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_FIJI), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 803") - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_FIJI") - KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx803 - endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VEGA), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 900") - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VEGA") - KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx900 - endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_GFX901), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 901") - tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_GFX901") - KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx901 - endif - - - KOKKOS_INTERNAL_HCC_PATH := $(shell which $(CXX)) - ROCM_HCC_PATH ?= $(KOKKOS_INTERNAL_HCC_PATH:/bin/clang++=) - KOKKOS_CXXFLAGS += $(shell $(ROCM_HCC_PATH)/bin/hcc-config --cxxflags) - KOKKOS_LDFLAGS += $(shell $(ROCM_HCC_PATH)/bin/hcc-config --ldflags) -lhc_am -lm - KOKKOS_CXXLDFLAGS += $(shell $(ROCM_HCC_PATH)/bin/hcc-config --ldflags) -lhc_am -lm - KOKKOS_TPL_LIBRARY_NAMES += hc_am m - KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_ROCM_ARCH_FLAG) + KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/HIP/*.cpp) + KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/HIP/*.hpp) - KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/ROCm/*.cpp) -ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) - KOKKOS_SRC += $(wildcard $(KOKKOS_ETI_PATH)/ROCm/*.cpp) -endif - KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/ROCm/*.hpp) + KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG) + KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG) + + ifeq ($(KOKKOS_INTERNAL_HIP_USE_RELOC), 1) + tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE") + KOKKOS_CXXFLAGS+=-fgpu-rdc + KOKKOS_LDFLAGS+=-fgpu-rdc + else + KOKKOS_CXXFLAGS+=-fno-gpu-rdc + KOKKOS_LDFLAGS+=-fno-gpu-rdc + endif endif + KOKKOS_INTERNAL_LS_CONFIG := $(shell ls KokkosCore_config.h 2>&1) ifeq ($(KOKKOS_INTERNAL_LS_CONFIG), KokkosCore_config.h) @@ -1089,15 +1169,13 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) endif KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.hpp) ifneq ($(CUDA_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(CUDA_PATH)/include - endif + KOKKOS_CPPLAGS += -I$(CUDA_PATH)/include ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib64), 1) - KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib64 + KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib64 KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib64 KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib64 else ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib), 1) - KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib + KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib else @@ -1113,7 +1191,7 @@ endif endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) - KOKKOS_SRC += $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp + KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/OpenMPTarget/*.cpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMPTarget/*.hpp) ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) KOKKOS_CXXFLAGS += -Xcompiler $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) @@ -1121,6 +1199,7 @@ ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) endif KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) + KOKKOS_LIBS += $(KOKKOS_INTERNAL_OPENMPTARGET_LIB) endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) @@ -1150,22 +1229,6 @@ endif KOKKOS_TPL_LIBRARY_NAMES += pthread endif -ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1) - KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.cpp) - KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.hpp) - ifneq ($(QTHREADS_PATH),) - ifneq ($(KOKKOS_CMAKE), yes) - KOKKOS_CXXFLAGS += -I$(QTHREADS_PATH)/include - endif - KOKKOS_LDFLAGS += -L$(QTHREADS_PATH)/lib - KOKKOS_CXXLDFLAGS += -L$(QTHREADS_PATH)/lib - KOKKOS_TPL_INCLUDE_DIRS += $(QTHREADS_PATH)/include - KOKKOS_TPL_LIBRARY_DIRS += $(QTHREADS_PATH)/lib64 - endif - KOKKOS_LIBS += -lqthread - KOKKOS_TPL_LIBRARY_NAMES += qthread -endif - ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/HPX/*.cpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/HPX/*.hpp) @@ -1173,21 +1236,21 @@ ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application_debug) KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) - KOKKOS_LDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) + KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug) else KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application) KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) - KOKKOS_LDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) + KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application) endif else ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1) KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application_debug) KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application_debug) - KOKKOS_LDFLAGS += $(shell pkg-config --libs hpx_application_debug) + KOKKOS_LIBS += $(shell pkg-config --libs hpx_application_debug) else KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application) KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application) - KOKKOS_LDFLAGS += $(shell pkg-config --libs hpx_application) + KOKKOS_LIBS += $(shell pkg-config --libs hpx_application) endif endif KOKKOS_TPL_LIBRARY_NAMES += hpx @@ -1248,4 +1311,16 @@ libkokkos.a: $(KOKKOS_OBJ_LINK) $(KOKKOS_SRC) $(KOKKOS_HEADERS) ar cr libkokkos.a $(KOKKOS_OBJ_LINK) ranlib libkokkos.a +print-cxx-flags: + echo "$(KOKKOS_CXXFLAGS)" + KOKKOS_LINK_DEPENDS=libkokkos.a + +#we have carefully separated LDFLAGS from LIBS and LIBDIRS +#we have also separated CPPFLAGS from CXXFLAGS +#if this is not cmake, for backwards compatibility +#we just jam everything together into the CXXFLAGS and LDFLAGS +ifneq ($(KOKKOS_CMAKE), yes) + KOKKOS_CXXFLAGS += $(KOKKOS_CPPFLAGS) + KOKKOS_LDFLAGS += $(KOKKOS_LIBDIRS) +endif diff --git a/lib/kokkos/Makefile.targets b/lib/kokkos/Makefile.targets index e7d5a3c907..18e37a71f7 100644 --- a/lib/kokkos/Makefile.targets +++ b/lib/kokkos/Makefile.targets @@ -6,6 +6,8 @@ Kokkos_CPUDiscovery.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_CPUDiscovery.cpp Kokkos_Error.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Error.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_Error.cpp +Kokkos_Stacktrace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Stacktrace.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_Stacktrace.cpp Kokkos_ExecPolicy.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_ExecPolicy.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_ExecPolicy.cpp Kokkos_HostSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HostSpace.cpp @@ -53,6 +55,17 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) endif endif +ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) +Kokkos_HIP_Space.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Space.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Space.cpp +Kokkos_HIP_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Instance.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Instance.cpp +Kokkos_HIP_KernelLaunch.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_KernelLaunch.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_KernelLaunch.cpp +Kokkos_HIP_Locks.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Locks.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Locks.cpp +endif + ifeq ($(KOKKOS_INTERNAL_USE_ROCM), 1) Kokkos_ROCm_Exec.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/ROCm/Kokkos_ROCm_Exec.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/ROCm/Kokkos_ROCm_Exec.cpp @@ -77,13 +90,6 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_ETI), 1) endif endif -ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1) -Kokkos_QthreadsExec.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Qthreads/Kokkos_QthreadsExec.cpp - $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Qthreads/Kokkos_QthreadsExec.cpp -Kokkos_Qthreads_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Qthreads/Kokkos_Qthreads_Task.cpp - $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Qthreads/Kokkos_Qthreads_Task.cpp -endif - ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) Kokkos_OpenMP_Exec.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Exec.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Exec.cpp @@ -104,10 +110,12 @@ endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) Kokkos_OpenMPTarget_Exec.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp +Kokkos_OpenMPTarget_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp Kokkos_OpenMPTargetSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp -#Kokkos_OpenMPTarget_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp -# $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp +Kokkos_OpenMPTarget_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp endif Kokkos_HBWSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HBWSpace.cpp diff --git a/lib/kokkos/README b/lib/kokkos/README deleted file mode 100644 index cb6ceb5581..0000000000 --- a/lib/kokkos/README +++ /dev/null @@ -1,193 +0,0 @@ -Kokkos Core implements a programming model in C++ for writing performance portable -applications targeting all major HPC platforms. For that purpose it provides -abstractions for both parallel execution of code and data management. -Kokkos is designed to target complex node architectures with N-level memory -hierarchies and multiple types of execution resources. It currently can use -OpenMP, Pthreads and CUDA as backend programming models. - -Kokkos Core is part of the Kokkos C++ Performance Portability Programming EcoSystem, -which also provides math kernels (https://github.com/kokkos/kokkos-kernels), as well as -profiling and debugging tools (https://github.com/kokkos/kokkos-tools). - -# Learning about Kokkos - -A programming guide can be found on the Wiki, the API reference is under development. - -For questions find us on Slack: https://kokkosteam.slack.com or open a github issue. - -For non-public questions send an email to -crtrott(at)sandia.gov - -A separate repository with extensive tutorial material can be found under -https://github.com/kokkos/kokkos-tutorials. - -Furthermore, the 'example/tutorial' directory provides step by step tutorial -examples which explain many of the features of Kokkos. They work with -simple Makefiles. To build with g++ and OpenMP simply type 'make' -in the 'example/tutorial' directory. This will build all examples in the -subfolders. To change the build options refer to the Programming Guide -in the compilation section. - -To learn more about Kokkos consider watching one of our presentations: -* GTC 2015: - - http://on-demand.gputechconf.com/gtc/2015/video/S5166.html - - http://on-demand.gputechconf.com/gtc/2015/presentation/S5166-H-Carter-Edwards.pdf - - -# Contributing to Kokkos - -We are open and try to encourage contributions from external developers. -To do so please first open an issue describing the contribution and then issue -a pull request against the develop branch. For larger features it may be good -to get guidance from the core development team first through the github issue. - -Note that Kokkos Core is licensed under standard 3-clause BSD terms of use. -Which means contributing to Kokkos allows anyone else to use your contributions -not just for public purposes but also for closed source commercial projects. -For specifics see the LICENSE file contained in the repository or distribution. - -# Requirements - -### Primary tested compilers on X86 are: - * GCC 4.8.4 - * GCC 4.9.3 - * GCC 5.1.0 - * GCC 5.5.0 - * GCC 6.1.0 - * GCC 7.2.0 - * GCC 7.3.0 - * GCC 8.1.0 - * Intel 15.0.2 - * Intel 16.0.1 - * Intel 17.0.1 - * Intel 17.4.196 - * Intel 18.2.128 - * Clang 3.6.1 - * Clang 3.7.1 - * Clang 3.8.1 - * Clang 3.9.0 - * Clang 4.0.0 - * Clang 6.0.0 for CUDA (CUDA Toolkit 9.0) - * Clang 7.0.0 for CUDA (CUDA Toolkit 9.1) - * PGI 18.7 - * NVCC 7.5 for CUDA (with gcc 4.8.4) - * NVCC 8.0.44 for CUDA (with gcc 5.3.0) - * NVCC 9.1 for CUDA (with gcc 6.1.0) - * NVCC 9.2 for CUDA (with gcc 7.2.0) - * NVCC 10.0 for CUDA (with gcc 7.4.0) - -### Primary tested compilers on Power 8 are: - * GCC 6.4.0 (OpenMP,Serial) - * GCC 7.2.0 (OpenMP,Serial) - * IBM XL 16.1.0 (OpenMP, Serial) - * NVCC 9.2.88 for CUDA (with gcc 7.2.0 and XL 16.1.0) - -### Primary tested compilers on Intel KNL are: - * Intel 16.4.258 (with gcc 4.7.2) - * Intel 17.2.174 (with gcc 4.9.3) - * Intel 18.2.199 (with gcc 4.9.3) - -### Primary tested compilers on ARM (Cavium ThunderX2) - * GCC 7.2.0 - * ARM/Clang 18.4.0 - -### Other compilers working: - * X86: - - Cygwin 2.1.0 64bit with gcc 4.9.3 - - GCC 8.1.0 (not warning free) - -### Known non-working combinations: - * Power8: - - Pthreads backend - * ARM - - Pthreads backend - - -Primary tested compiler are passing in release mode -with warnings as errors. They also are tested with a comprehensive set of -backend combinations (i.e. OpenMP, Pthreads, Serial, OpenMP+Serial, ...). -We are using the following set of flags: -GCC: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits - -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized -Intel: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized -Clang: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized -NVCC: -Wall -Wshadow -pedantic -Werror -Wsign-compare -Wtype-limits -Wuninitialized - -Other compilers are tested occasionally, in particular when pushing from develop to -master branch, without -Werror and only for a select set of backends. - -# Running Unit Tests - -To run the unit tests create a build directory and run the following commands - -KOKKOS_PATH/generate_makefile.bash -make build-test -make test - -Run KOKKOS_PATH/generate_makefile.bash --help for more detailed options such as -changing the device type for which to build. - -# Installing the library - -To install Kokkos as a library create a build directory and run the following - -KOKKOS_PATH/generate_makefile.bash --prefix=INSTALL_PATH -make kokkoslib -make install - -KOKKOS_PATH/generate_makefile.bash --help for more detailed options such as -changing the device type for which to build. - -Note that in many cases it is preferable to build Kokkos inline with an -application. The main reason is that you may otherwise need many different -configurations of Kokkos installed depending on the required compile time -features an application needs. For example there is only one default -execution space, which means you need different installations to have OpenMP -or Pthreads as the default space. Also for the CUDA backend there are certain -choices, such as allowing relocatable device code, which must be made at -installation time. Building Kokkos inline uses largely the same process -as compiling an application against an installed Kokkos library. See for -example benchmarks/bytes_and_flops/Makefile which can be used with an installed -library and for an inline build. - -### CMake - -Kokkos supports being build as part of a CMake applications. An example can -be found in example/cmake_build. - -# Kokkos and CUDA UVM - -Kokkos does support UVM as a specific memory space called CudaUVMSpace. -Allocations made with that space are accessible from host and device. -You can tell Kokkos to use that as the default space for Cuda allocations. -In either case UVM comes with a number of restrictions: -(i) You can't access allocations on the host while a kernel is potentially -running. This will lead to segfaults. To avoid that you either need to -call Kokkos::Cuda::fence() (or just Kokkos::fence()), after kernels, or -you can set the environment variable CUDA_LAUNCH_BLOCKING=1. -Furthermore in multi socket multi GPU machines without NVLINK, UVM defaults -to using zero copy allocations for technical reasons related to using multiple -GPUs from the same process. If an executable doesn't do that (e.g. each -MPI rank of an application uses a single GPU [can be the same GPU for -multiple MPI ranks]) you can set CUDA_MANAGED_FORCE_DEVICE_ALLOC=1. -This will enforce proper UVM allocations, but can lead to errors if -more than a single GPU is used by a single process. - - -# Citing Kokkos - -If you publish work which mentions Kokkos, please cite the following paper: - -@article{CarterEdwards20143202, -title = "Kokkos: Enabling manycore performance portability through polymorphic memory access patterns ", -journal = "Journal of Parallel and Distributed Computing ", -volume = "74", -number = "12", -pages = "3202 - 3216", -year = "2014", -note = "Domain-Specific Languages and High-Level Frameworks for High-Performance Computing ", -issn = "0743-7315", -doi = "https://doi.org/10.1016/j.jpdc.2014.07.003", -url = "http://www.sciencedirect.com/science/article/pii/S0743731514001257", -author = "H. Carter Edwards and Christian R. Trott and Daniel Sunderland" -} diff --git a/lib/kokkos/README.md b/lib/kokkos/README.md new file mode 100644 index 0000000000..a04df9eb9b --- /dev/null +++ b/lib/kokkos/README.md @@ -0,0 +1,299 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Kokkos: Core Libraries + +Kokkos Core implements a programming model in C++ for writing performance portable +applications targeting all major HPC platforms. For that purpose it provides +abstractions for both parallel execution of code and data management. +Kokkos is designed to target complex node architectures with N-level memory +hierarchies and multiple types of execution resources. It currently can use +CUDA, HPX, OpenMP and Pthreads as backend programming models with several other +backends in development. + +Kokkos Core is part of the Kokkos C++ Performance Portability Programming EcoSystem, +which also provides math kernels (https://github.com/kokkos/kokkos-kernels), as well as +profiling and debugging tools (https://github.com/kokkos/kokkos-tools). + +# Learning about Kokkos + +A programming guide can be found on the Wiki, the API reference is under development. + +For questions find us on Slack: https://kokkosteam.slack.com or open a github issue. + +For non-public questions send an email to +crtrott(at)sandia.gov + +A separate repository with extensive tutorial material can be found under +https://github.com/kokkos/kokkos-tutorials. + +Furthermore, the 'example/tutorial' directory provides step by step tutorial +examples which explain many of the features of Kokkos. They work with +simple Makefiles. To build with g++ and OpenMP simply type 'make' +in the 'example/tutorial' directory. This will build all examples in the +subfolders. To change the build options refer to the Programming Guide +in the compilation section. + +To learn more about Kokkos consider watching one of our presentations: +* GTC 2015: + - http://on-demand.gputechconf.com/gtc/2015/video/S5166.html + - http://on-demand.gputechconf.com/gtc/2015/presentation/S5166-H-Carter-Edwards.pdf + + +# Contributing to Kokkos + +We are open and try to encourage contributions from external developers. +To do so please first open an issue describing the contribution and then issue +a pull request against the develop branch. For larger features it may be good +to get guidance from the core development team first through the github issue. + +Note that Kokkos Core is licensed under standard 3-clause BSD terms of use. +Which means contributing to Kokkos allows anyone else to use your contributions +not just for public purposes but also for closed source commercial projects. +For specifics see the LICENSE file contained in the repository or distribution. + +# Requirements + +### Primary tested compilers on X86 are: +* GCC 4.8.4 +* GCC 4.9.3 +* GCC 5.1.0 +* GCC 5.4.0 +* GCC 5.5.0 +* GCC 6.1.0 +* GCC 7.2.0 +* GCC 7.3.0 +* GCC 8.1.0 +* Intel 15.0.2 +* Intel 16.0.1 +* Intel 17.0.1 +* Intel 17.4.196 +* Intel 18.2.128 +* Clang 3.6.1 +* Clang 3.7.1 +* Clang 3.8.1 +* Clang 3.9.0 +* Clang 4.0.0 +* Clang 6.0.0 for CUDA (CUDA Toolkit 9.0) +* Clang 7.0.0 for CUDA (CUDA Toolkit 9.1) +* Clang 8.0.0 for CUDA (CUDA Toolkit 9.2) +* PGI 18.7 +* NVCC 9.1 for CUDA (with gcc 6.1.0) +* NVCC 9.2 for CUDA (with gcc 7.2.0) +* NVCC 10.0 for CUDA (with gcc 7.4.0) +* NVCC 10.1 for CUDA (with gcc 7.4.0) + +### Primary tested compilers on Power 8 are: +* GCC 6.4.0 (OpenMP,Serial) +* GCC 7.2.0 (OpenMP,Serial) +* IBM XL 16.1.0 (OpenMP, Serial) +* NVCC 9.2.88 for CUDA (with gcc 7.2.0 and XL 16.1.0) + +### Primary tested compilers on Intel KNL are: +* Intel 16.4.258 (with gcc 4.7.2) +* Intel 17.2.174 (with gcc 4.9.3) +* Intel 18.2.199 (with gcc 4.9.3) + +### Primary tested compilers on ARM (Cavium ThunderX2) +* GCC 7.2.0 +* ARM/Clang 18.4.0 + +### Other compilers working: +* X86: + * Cygwin 2.1.0 64bit with gcc 4.9.3 + * GCC 8.1.0 (not warning free) + +### Known non-working combinations: +* Power8: + * Pthreads backend +* ARM + * Pthreads backend + + +Primary tested compiler are passing in release mode +with warnings as errors. They also are tested with a comprehensive set of +backend combinations (i.e. OpenMP, Pthreads, Serial, OpenMP+Serial, ...). +We are using the following set of flags: +* GCC: + ```` + -Wall -Wunused-parameter -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wignored-qualifiers -Wempty-body + -Wclobbered -Wuninitialized + ```` +* Intel: + ```` + -Wall -Wunused-parameter -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` +* Clang: + ```` + -Wall -Wunused-parameter -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` + +* NVCC: + ```` + -Wall -Wunused-parameter -Wshadow -pedantic + -Werror -Wsign-compare -Wtype-limits + -Wuninitialized + ```` + +Other compilers are tested occasionally, in particular when pushing from develop to +master branch. These are tested less rigorously without `-Werror` and only for a select set of backends. + +# Building and Installing Kokkos +Kokkos provide a CMake build system and a raw Makefile build system. +The CMake build system is strongly encouraged and will be the most rigorously supported in future releases. +Full details are given in the [build instructions](BUILD.md). Basic setups are shown here: + +## CMake + +The best way to install Kokkos is using the CMake build system. Assuming Kokkos lives in `$srcdir`: +```` +cmake $srcdir \ + -DCMAKE_CXX_COMPILER=$path_to_compiler \ + -DCMAKE_INSTALL_PREFIX=$path_to_install \ + -DKokkos_ENABLE_OPENMP=On \ + -DKokkos_ARCH_HSW=On \ + -DKokkos_ENABLE_HWLOC=On \ + -DKokkos_HWLOC_DIR=$path_to_hwloc +```` +then simply type `make install`. The Kokkos CMake package will then be installed in `$path_to_install` to be used by downstream packages. + +To validate the Kokkos build, configure with +```` + -DKokkos_ENABLE_TESTS=On +```` +and run `make test` after completing the build. + +For your CMake project using Kokkos, code such as the following: + +```` +find_package(Kokkos) +... +target_link_libraries(myTarget Kokkos::kokkos) +```` +should be added to your CMakeLists.txt. Your configure should additionally include +```` +-DKokkos_DIR=$path_to_install/cmake/lib/Kokkos +```` +or +```` +-DKokkos_ROOT=$path_to_install +```` +for the install location given above. + +## Spack +An alternative to manually building with the CMake is to use the Spack package manager. +To do so, download the `kokkos-spack` git repo and add to the package list: +```` +spack repo add $path-to-kokkos-spack +```` +A basic installation would be done as: +```` +spack install kokkos +```` +Spack allows options and and compilers to be tuned in the install command. +```` +spack install kokkos@3.0 %gcc@7.3.0 +openmp +```` +This example illustrates the three most common parameters to Spack: +* Variants: specified with, e.g. `+openmp`, this activates (or deactivates with, e.g. `~openmp`) certain options. +* Version: immediately following `kokkos` the `@version` can specify a particular Kokkos to build +* Compiler: a default compiler will be chosen if not specified, but an exact compiler version can be given with the `%`option. + +For a complete list of Kokkos options, run: +```` +spack info kokkos +```` +Spack currently installs packages to a location determined by a unique hash. This hash name is not really "human readable". +Generally, Spack usage should never really require you to reference the computer-generated unique install folder. +More details are given in the [build instructions](BUILD.md). If you must know, you can locate Spack Kokkos installations with: +```` +spack find -p kokkos ... +```` +where `...` is the unique spec identifying the particular Kokkos configuration and version. + + +## Raw Makefile +A bash script is provided to generate raw makefiles. +To install Kokkos as a library create a build directory and run the following +```` +$KOKKOS_PATH/generate_makefile.bash --prefix=$path_to_install +```` +Once the Makefile is generated, run: +```` +make kokkoslib +make install +```` +To additionally run the unit tests: +```` +make build-test +make test +```` +Run `generate_makefile.bash --help` for more detailed options such as +changing the device type for which to build. + +## Inline Builds vs. Installed Package +For individual projects, it may be preferable to build Kokkos inline rather than link to an installed package. +The main reason is that you may otherwise need many different +configurations of Kokkos installed depending on the required compile time +features an application needs. For example there is only one default +execution space, which means you need different installations to have OpenMP +or Pthreads as the default space. Also for the CUDA backend there are certain +choices, such as allowing relocatable device code, which must be made at +installation time. Building Kokkos inline uses largely the same process +as compiling an application against an installed Kokkos library. + +For CMake, this means copying over the Kokkos source code into your project and adding `add_subdirectory(kokkos)` to your CMakeLists.txt. + +For raw Makefiles, see the example benchmarks/bytes_and_flops/Makefile which can be used with an installed library and or an inline build. + +# Kokkos and CUDA UVM + +Kokkos does support UVM as a specific memory space called CudaUVMSpace. +Allocations made with that space are accessible from host and device. +You can tell Kokkos to use that as the default space for Cuda allocations. +In either case UVM comes with a number of restrictions: +* You can't access allocations on the host while a kernel is potentially +running. This will lead to segfaults. To avoid that you either need to +call Kokkos::Cuda::fence() (or just Kokkos::fence()), after kernels, or +you can set the environment variable CUDA_LAUNCH_BLOCKING=1. +* In multi socket multi GPU machines without NVLINK, UVM defaults +to using zero copy allocations for technical reasons related to using multiple +GPUs from the same process. If an executable doesn't do that (e.g. each +MPI rank of an application uses a single GPU [can be the same GPU for +multiple MPI ranks]) you can set CUDA_MANAGED_FORCE_DEVICE_ALLOC=1. +This will enforce proper UVM allocations, but can lead to errors if +more than a single GPU is used by a single process. + + +# Citing Kokkos + +If you publish work which mentions Kokkos, please cite the following paper: + +```` +@article{CarterEdwards20143202, + title = "Kokkos: Enabling manycore performance portability through polymorphic memory access patterns ", + journal = "Journal of Parallel and Distributed Computing ", + volume = "74", + number = "12", + pages = "3202 - 3216", + year = "2014", + note = "Domain-Specific Languages and High-Level Frameworks for High-Performance Computing ", + issn = "0743-7315", + doi = "https://doi.org/10.1016/j.jpdc.2014.07.003", + url = "http://www.sciencedirect.com/science/article/pii/S0743731514001257", + author = "H. Carter Edwards and Christian R. Trott and Daniel Sunderland" +} +```` + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/master/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. + diff --git a/lib/kokkos/algorithms/CMakeLists.txt b/lib/kokkos/algorithms/CMakeLists.txt index 507c9f2fdb..38747c152c 100644 --- a/lib/kokkos/algorithms/CMakeLists.txt +++ b/lib/kokkos/algorithms/CMakeLists.txt @@ -1,12 +1,12 @@ - - -TRIBITS_SUBPACKAGE(Algorithms) - -IF(KOKKOS_HAS_TRILINOS) - ADD_SUBDIRECTORY(src) -ENDIF() - -TRIBITS_ADD_TEST_DIRECTORIES(unit_tests) -#TRIBITS_ADD_TEST_DIRECTORIES(performance_tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() + + +KOKKOS_SUBPACKAGE(Algorithms) + +ADD_SUBDIRECTORY(src) + +KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) + +KOKKOS_SUBPACKAGE_POSTPROCESS() + + + diff --git a/lib/kokkos/algorithms/src/CMakeLists.txt b/lib/kokkos/algorithms/src/CMakeLists.txt index dfbf3323c2..5afd319fcc 100644 --- a/lib/kokkos/algorithms/src/CMakeLists.txt +++ b/lib/kokkos/algorithms/src/CMakeLists.txt @@ -1,8 +1,9 @@ -TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) +KOKKOS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +#I have to leave these here for tribits +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) #----------------------------------------------------------------------------- @@ -12,10 +13,18 @@ LIST(APPEND HEADERS ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME}_config.h) #----------------------------------------------------------------------------- -TRIBITS_ADD_LIBRARY( - kokkosalgorithms - HEADERS ${HEADERS} - SOURCES ${SOURCES} - DEPLIBS - ) +# We have to pass the sources in here for Tribits +# These will get ignored for standalone CMake and a true interface library made +KOKKOS_ADD_INTERFACE_LIBRARY( + kokkosalgorithms + HEADERS ${HEADERS} + SOURCES ${SOURCES} +) +KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkosalgorithms + ${KOKKOS_TOP_BUILD_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) + + diff --git a/lib/kokkos/algorithms/src/Kokkos_Random.hpp b/lib/kokkos/algorithms/src/Kokkos_Random.hpp index da781de4fe..0a79675236 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Random.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Random.hpp @@ -1,13 +1,14 @@ /* //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -36,7 +37,7 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER */ @@ -59,6 +60,7 @@ namespace Kokkos { +// clang-format off /*Template functions to get equidistributed random numbers from a generator for a specific Scalar type template @@ -229,1798 +231,1510 @@ namespace Kokkos { ViewType::value_type start, ViewType::value_type end); */ - - template - struct rand; - - - template - struct rand { - - KOKKOS_INLINE_FUNCTION - static short max(){return 127;} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen) - {return short((gen.rand()&0xff+256)%256);} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const char& range) - {return char(gen.rand(range));} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const char& start, const char& end) - {return char(gen.rand(start,end));} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static short max(){return 32767;} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen) - {return short((gen.rand()&0xffff+65536)%32768);} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const short& range) - {return short(gen.rand(range));} - KOKKOS_INLINE_FUNCTION - static short draw(Generator& gen, const short& start, const short& end) - {return short(gen.rand(start,end));} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static int max(){return Generator::MAX_RAND;} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen) - {return gen.rand();} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen, const int& range) - {return gen.rand(range);} - KOKKOS_INLINE_FUNCTION - static int draw(Generator& gen, const int& start, const int& end) - {return gen.rand(start,end);} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned int max () { - return Generator::MAX_URAND; - } - KOKKOS_INLINE_FUNCTION - static unsigned int draw (Generator& gen) { - return gen.urand (); - } - KOKKOS_INLINE_FUNCTION - static unsigned int draw(Generator& gen, const unsigned int& range) { - return gen.urand (range); - } - KOKKOS_INLINE_FUNCTION - static unsigned int - draw (Generator& gen, const unsigned int& start, const unsigned int& end) { - return gen.urand (start, end); - } - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static long max () { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (Generator::MAX_RAND) : - static_cast (Generator::MAX_RAND64); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand ()) : - static_cast (gen.rand64 ()); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen, const long& range) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand (static_cast (range))) : - static_cast (gen.rand64 (range)); - } - KOKKOS_INLINE_FUNCTION - static long draw (Generator& gen, const long& start, const long& end) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (long) == 4 ? - static_cast (gen.rand (static_cast (start), - static_cast (end))) : - static_cast (gen.rand64 (start, end)); - } - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned long max () { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (Generator::MAX_URAND) : - static_cast (Generator::MAX_URAND64); - } - KOKKOS_INLINE_FUNCTION - static unsigned long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand ()) : - static_cast (gen.urand64 ()); - } - KOKKOS_INLINE_FUNCTION - static unsigned long draw(Generator& gen, const unsigned long& range) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand (static_cast (range))) : - static_cast (gen.urand64 (range)); - } - KOKKOS_INLINE_FUNCTION - static unsigned long - draw (Generator& gen, const unsigned long& start, const unsigned long& end) { - // FIXME (mfh 26 Oct 2014) It would be better to select the - // return value at compile time, using something like enable_if. - return sizeof (unsigned long) == 4 ? - static_cast (gen.urand (static_cast (start), - static_cast (end))) : - static_cast (gen.urand64 (start, end)); - } - }; - - // NOTE (mfh 26 oct 2014) This is a partial specialization for long - // long, a C99 / C++11 signed type which is guaranteed to be at - // least 64 bits. Do NOT write a partial specialization for - // int64_t!!! This is just a typedef! It could be either long or - // long long. We don't know which a priori, and I've seen both. - // The types long and long long are guaranteed to differ, so it's - // always safe to specialize for both. - template - struct rand { - KOKKOS_INLINE_FUNCTION - static long long max () { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return Generator::MAX_RAND64; - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (); - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen, const long long& range) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (range); - } - KOKKOS_INLINE_FUNCTION - static long long draw (Generator& gen, const long long& start, const long long& end) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.rand64 (start, end); - } - }; - - // NOTE (mfh 26 oct 2014) This is a partial specialization for - // unsigned long long, a C99 / C++11 unsigned type which is - // guaranteed to be at least 64 bits. Do NOT write a partial - // specialization for uint64_t!!! This is just a typedef! It could - // be either unsigned long or unsigned long long. We don't know - // which a priori, and I've seen both. The types unsigned long and - // unsigned long long are guaranteed to differ, so it's always safe - // to specialize for both. - template - struct rand { - KOKKOS_INLINE_FUNCTION - static unsigned long long max () { - // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. - return Generator::MAX_URAND64; - } - KOKKOS_INLINE_FUNCTION - static unsigned long long draw (Generator& gen) { - // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 bits. - return gen.urand64 (); - } - KOKKOS_INLINE_FUNCTION - static unsigned long long draw (Generator& gen, const unsigned long long& range) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.urand64 (range); - } - KOKKOS_INLINE_FUNCTION - static unsigned long long - draw (Generator& gen, const unsigned long long& start, const unsigned long long& end) { - // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. - return gen.urand64 (start, end); - } - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static float max(){return 1.0f;} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen) - {return gen.frand();} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen, const float& range) - {return gen.frand(range);} - KOKKOS_INLINE_FUNCTION - static float draw(Generator& gen, const float& start, const float& end) - {return gen.frand(start,end);} - - }; - - template - struct rand { - KOKKOS_INLINE_FUNCTION - static double max(){return 1.0;} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen) - {return gen.drand();} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen, const double& range) - {return gen.drand(range);} - KOKKOS_INLINE_FUNCTION - static double draw(Generator& gen, const double& start, const double& end) - {return gen.drand(start,end);} - - }; - - template - struct rand > { - KOKKOS_INLINE_FUNCTION - static Kokkos::complex max () { - return Kokkos::complex (1.0, 1.0); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen) { - const float re = gen.frand (); - const float im = gen.frand (); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& range) { - const float re = gen.frand (real (range)); - const float im = gen.frand (imag (range)); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& start, const Kokkos::complex& end) { - const float re = gen.frand (real (start), real (end)); - const float im = gen.frand (imag (start), imag (end)); - return Kokkos::complex (re, im); - } - }; - - template - struct rand > { - KOKKOS_INLINE_FUNCTION - static Kokkos::complex max () { - return Kokkos::complex (1.0, 1.0); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen) { - const double re = gen.drand (); - const double im = gen.drand (); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& range) { - const double re = gen.drand (real (range)); - const double im = gen.drand (imag (range)); - return Kokkos::complex (re, im); - } - KOKKOS_INLINE_FUNCTION - static Kokkos::complex draw (Generator& gen, const Kokkos::complex& start, const Kokkos::complex& end) { - const double re = gen.drand (real (start), real (end)); - const double im = gen.drand (imag (start), imag (end)); - return Kokkos::complex (re, im); - } - }; - - template - class Random_XorShift64_Pool; - - template - class Random_XorShift64 { - private: - uint64_t state_; - const int state_idx_; - friend class Random_XorShift64_Pool; - public: - - typedef DeviceType device_type; - - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffff/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffLL/2-1)}; - - KOKKOS_INLINE_FUNCTION - Random_XorShift64 (uint64_t state, int state_idx = 0) - : state_(state==0?uint64_t(1318319):state),state_idx_(state_idx){} - - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - state_ ^= state_ >> 12; - state_ ^= state_ << 25; - state_ ^= state_ >> 27; - - uint64_t tmp = state_ * 2685821657736338717ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - state_ ^= state_ >> 12; - state_ ^= state_ << 25; - state_ ^= state_ >> 27; - return (state_ * 2685821657736338717ULL) - 1; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - tmp = urand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - tmp = urand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - tmp = rand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - tmp = rand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return drand(end-start)+start; - } - - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } - - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - - }; - - template - class Random_XorShift64_Pool { - private: - typedef View lock_type; - typedef View state_data_type; - lock_type locks_; - state_data_type state_; - int num_states_; - - public: - typedef Random_XorShift64 generator_type; - typedef DeviceType device_type; - - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool() { - num_states_ = 0; - } - Random_XorShift64_Pool(uint64_t seed) { - num_states_ = 0; -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - init(seed,DeviceType::max_hardware_threads()); -#else - init(seed,DeviceType::impl_max_hardware_threads()); -#endif - } - - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool(const Random_XorShift64_Pool& src): - locks_(src.locks_), - state_(src.state_), - num_states_(src.num_states_) - {} - - KOKKOS_INLINE_FUNCTION - Random_XorShift64_Pool operator = (const Random_XorShift64_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - num_states_ = src.num_states_; - return *this; - } - - void init(uint64_t seed, int num_states) { - if(seed==0) - seed = uint64_t(1318319); - - num_states_ = num_states; - - locks_ = lock_type("Kokkos::Random_XorShift64::locks",num_states_); - state_ = state_data_type("Kokkos::Random_XorShift64::state",num_states_); - - typename state_data_type::HostMirror h_state = create_mirror_view(state_); - typename lock_type::HostMirror h_lock = create_mirror_view(locks_); - - // Execute on the HostMirror's default execution space. - Random_XorShift64 gen(seed,0); - for(int i = 0; i < 17; i++) - gen.rand(); - for(int i = 0; i < num_states_; i++) { - int n1 = gen.rand(); - int n2 = gen.rand(); - int n3 = gen.rand(); - int n4 = gen.rand(); - h_state(i) = (((static_cast(n1)) & 0xffff)<<00) | - (((static_cast(n2)) & 0xffff)<<16) | - (((static_cast(n3)) & 0xffff)<<32) | - (((static_cast(n4)) & 0xffff)<<48); - h_lock(i) = 0; - } - deep_copy(state_,h_state); - deep_copy(locks_,h_lock); - } - - KOKKOS_INLINE_FUNCTION - Random_XorShift64 get_state() const { -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - const int i = DeviceType::hardware_thread_id();; -#else - const int i = DeviceType::impl_hardware_thread_id();; -#endif - return Random_XorShift64(state_(i),i); - } - - // NOTE: state_idx MUST be unique and less than num_states - KOKKOS_INLINE_FUNCTION - Random_XorShift64 get_state(const int state_idx) const { - return Random_XorShift64(state_(state_idx),state_idx); - } - - KOKKOS_INLINE_FUNCTION - void free_state(const Random_XorShift64& state) const { - state_(state.state_idx_) = state.state_; - } - }; - - - template - class Random_XorShift1024_Pool; - - template - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t state_[16]; - friend class Random_XorShift1024_Pool; - public: - - typedef Random_XorShift1024_Pool pool_type; - typedef DeviceType device_type; - - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx){ - for(int i=0 ; i<16; i++) - state_[i] = state(state_idx,i); - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ ]; - uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ ]; - uint64_t state_1 = state_[ p_ = ( p_ + 1 ) & 15 ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - tmp = urand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - tmp = urand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - tmp = rand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - tmp = rand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } - - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } - - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; - - - template - class Random_XorShift1024_Pool { - private: - typedef View int_view_type; - typedef View state_data_type; - - int_view_type locks_; - state_data_type state_; - int_view_type p_; - int num_states_; - friend class Random_XorShift1024; - - public: - typedef Random_XorShift1024 generator_type; - - typedef DeviceType device_type; - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool() { - num_states_ = 0; - } - - inline - Random_XorShift1024_Pool(uint64_t seed){ - num_states_ = 0; -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - init(seed,DeviceType::max_hardware_threads()); -#else - init(seed,DeviceType::impl_max_hardware_threads()); -#endif - } - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src): - locks_(src.locks_), - state_(src.state_), - p_(src.p_), - num_states_(src.num_states_) - {} - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024_Pool operator = (const Random_XorShift1024_Pool& src) { - locks_ = src.locks_; - state_ = src.state_; - p_ = src.p_; - num_states_ = src.num_states_; - return *this; - } - - inline - void init(uint64_t seed, int num_states) { - if(seed==0) - seed = uint64_t(1318319); - num_states_ = num_states; - locks_ = int_view_type("Kokkos::Random_XorShift1024::locks",num_states_); - state_ = state_data_type("Kokkos::Random_XorShift1024::state",num_states_); - p_ = int_view_type("Kokkos::Random_XorShift1024::p",num_states_); - - typename state_data_type::HostMirror h_state = create_mirror_view(state_); - typename int_view_type::HostMirror h_lock = create_mirror_view(locks_); - typename int_view_type::HostMirror h_p = create_mirror_view(p_); - - // Execute on the HostMirror's default execution space. - Random_XorShift64 gen(seed,0); - for(int i = 0; i < 17; i++) - gen.rand(); - for(int i = 0; i < num_states_; i++) { - for(int j = 0; j < 16 ; j++) { - int n1 = gen.rand(); - int n2 = gen.rand(); - int n3 = gen.rand(); - int n4 = gen.rand(); - h_state(i,j) = (((static_cast(n1)) & 0xffff)<<00) | - (((static_cast(n2)) & 0xffff)<<16) | - (((static_cast(n3)) & 0xffff)<<32) | - (((static_cast(n4)) & 0xffff)<<48); - } - h_p(i) = 0; - h_lock(i) = 0; - } - deep_copy(state_,h_state); - deep_copy(locks_,h_lock); - } - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 get_state() const { -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - const int i = DeviceType::hardware_thread_id(); -#else - const int i = DeviceType::impl_hardware_thread_id(); -#endif - return Random_XorShift1024(state_,p_(i),i); - }; - - // NOTE: state_idx MUST be unique and less than num_states - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 get_state(const int state_idx) const { - return Random_XorShift1024(state_,p_(state_idx),state_idx); - } - - KOKKOS_INLINE_FUNCTION - void free_state(const Random_XorShift1024& state) const { - for(int i = 0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; - p_(state.state_idx_) = state.p_; - } - }; - -#if defined(KOKKOS_ENABLE_CUDA) && defined(__CUDACC__) - - template<> - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t* state_; - const int stride_; - friend class Random_XorShift1024_Pool; - public: - - typedef Kokkos::Cuda device_type; - typedef Random_XorShift1024_Pool pool_type; - - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx),state_(&state(state_idx,0)),stride_(state.stride_1()){ - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - urand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - urand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - rand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - rand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } - - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } - - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; - -template<> -inline -Random_XorShift64_Pool::Random_XorShift64_Pool(uint64_t seed) { - num_states_ = 0; - init(seed,4*32768); -} - -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift64 Random_XorShift64_Pool::get_state() const { -#ifdef __CUDA_ARCH__ - const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; - int i = (((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * - blockDim.x*blockDim.y*blockDim.z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim.x*blockDim.y*blockDim.z; - if(i>=num_states_) {i = i_offset;} +// clang-format on + +template +struct rand; + +template +struct rand { + KOKKOS_INLINE_FUNCTION + static short max() { return 127; } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) { + return short((gen.rand() & 0xff + 256) % 256); } - - return Random_XorShift64(state_(i),i); -#else - return Random_XorShift64(state_(0),0); -#endif -} - -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift64_Pool::free_state(const Random_XorShift64 &state) const { - state_(state.state_idx_) = state.state_; -#ifdef __CUDA_ARCH__ - locks_(state.state_idx_) = 0; - return; -#endif -} - - -template<> -inline -Random_XorShift1024_Pool::Random_XorShift1024_Pool(uint64_t seed) { - num_states_ = 0; - init(seed,4*32768); -} - -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift1024 Random_XorShift1024_Pool::get_state() const { -#ifdef __CUDA_ARCH__ - const int i_offset = (threadIdx.x*blockDim.y + threadIdx.y)*blockDim.z+threadIdx.z; - int i = (((blockIdx.x*gridDim.y+blockIdx.y)*gridDim.z + blockIdx.z) * - blockDim.x*blockDim.y*blockDim.z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim.x*blockDim.y*blockDim.z; - if(i>=num_states_) {i = i_offset;} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& range) { + return char(gen.rand(range)); } - - return Random_XorShift1024(state_, p_(i), i); -#else - return Random_XorShift1024(state_, p_(0), 0); -#endif -} - -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift1024_Pool::free_state(const Random_XorShift1024 &state) const { - for(int i=0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; -#ifdef __CUDA_ARCH__ - locks_(state.state_idx_) = 0; - return; -#endif -} - - -#endif - -#if defined(KOKKOS_ENABLE_ROCM) - - template<> - class Random_XorShift1024 { - private: - int p_; - const int state_idx_; - uint64_t* state_; - const int stride_; - friend class Random_XorShift1024_Pool; - public: - - typedef Kokkos::Experimental::ROCm device_type; - typedef Random_XorShift1024_Pool pool_type; - - enum {MAX_URAND = 0xffffffffU}; - enum {MAX_URAND64 = 0xffffffffffffffffULL-1}; - enum {MAX_RAND = static_cast(0xffffffffU/2)}; - enum {MAX_RAND64 = static_cast(0xffffffffffffffffULL/2-1)}; - - KOKKOS_INLINE_FUNCTION - Random_XorShift1024 (const typename pool_type::state_data_type& state, int p, int state_idx = 0): - p_(p),state_idx_(state_idx),state_(&state(state_idx,0)),stride_(state.stride_1()){ - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - uint64_t tmp = ( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981ULL; - tmp = tmp>>16; - return static_cast(tmp&MAX_URAND); - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64() { - uint64_t state_0 = state_[ p_ * stride_ ]; - uint64_t state_1 = state_[ (p_ = ( p_ + 1 ) & 15) * stride_ ]; - state_1 ^= state_1 << 31; - state_1 ^= state_1 >> 11; - state_0 ^= state_0 >> 30; - return (( state_[ p_ * stride_ ] = state_0 ^ state_1 ) * 1181783497276652981LL) - 1; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& range) { - const uint32_t max_val = (MAX_URAND/range)*range; - uint32_t tmp = urand(); - while(tmp>=max_val) - urand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint32_t urand(const uint32_t& start, const uint32_t& end ) { - return urand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& range) { - const uint64_t max_val = (MAX_URAND64/range)*range; - uint64_t tmp = urand64(); - while(tmp>=max_val) - urand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - uint64_t urand64(const uint64_t& start, const uint64_t& end ) { - return urand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int rand() { - return static_cast(urand()/2); - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& range) { - const int max_val = (MAX_RAND/range)*range; - int tmp = rand(); - while(tmp>=max_val) - rand(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int rand(const int& start, const int& end ) { - return rand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64() { - return static_cast(urand64()/2); - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& range) { - const int64_t max_val = (MAX_RAND64/range)*range; - int64_t tmp = rand64(); - while(tmp>=max_val) - rand64(); - return tmp%range; - } - - KOKKOS_INLINE_FUNCTION - int64_t rand64(const int64_t& start, const int64_t& end ) { - return rand64(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - float frand() { - return 1.0f * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - float frand(const float& start, const float& end ) { - return frand(end-start)+start; - } - - KOKKOS_INLINE_FUNCTION - double drand() { - return 1.0 * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& range) { - return range * urand64()/MAX_URAND64; - } - - KOKKOS_INLINE_FUNCTION - double drand(const double& start, const double& end ) { - return frand(end-start)+start; - } - - //Marsaglia polar method for drawing a standard normal distributed random number - KOKKOS_INLINE_FUNCTION - double normal() { - double S = 2.0; - double U; - while(S>=1.0) { - U = 2.0*drand() - 1.0; - const double V = 2.0*drand() - 1.0; - S = U*U+V*V; - } - return U*std::sqrt(-2.0*log(S)/S); - } - - KOKKOS_INLINE_FUNCTION - double normal(const double& mean, const double& std_dev=1.0) { - return mean + normal()*std_dev; - } - }; - -template<> -inline -Random_XorShift64_Pool::Random_XorShift64_Pool(uint64_t seed) { - num_states_ = 0; - init(seed,4*32768); -} - -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift64 Random_XorShift64_Pool::get_state() const { -#ifdef __HCC_ACCELERATOR__ - const int i_offset = (threadIdx_x*blockDim_y + threadIdx_y)*blockDim_z+threadIdx_z; - int i = (((blockIdx_x*gridDim_y+blockIdx_y)*gridDim_z + blockIdx_z) * - blockDim_x*blockDim_y*blockDim_z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim_x*blockDim_y*blockDim_z; - if(i>=num_states_) {i = i_offset;} + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const char& start, const char& end) { + return char(gen.rand(start, end)); } +}; - return Random_XorShift64(state_(i),i); -#else - return Random_XorShift64(state_(0),0); -#endif -} - -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift64_Pool::free_state(const Random_XorShift64 &state) const { -#ifdef __HCC_ACCELERATOR__ - state_(state.state_idx_) = state.state_; - locks_(state.state_idx_) = 0; - return; -#endif -} - - -template<> -inline -Random_XorShift1024_Pool::Random_XorShift1024_Pool(uint64_t seed) { - num_states_ = 0; - init(seed,4*32768); -} - -template<> -KOKKOS_INLINE_FUNCTION -Random_XorShift1024 Random_XorShift1024_Pool::get_state() const { -#ifdef __HCC_ACCELERATOR__ - const int i_offset = (threadIdx_x*blockDim_y + threadIdx_y)*blockDim_z+threadIdx_z; - int i = (((blockIdx_x*gridDim_y+blockIdx_y)*gridDim_z + blockIdx_z) * - blockDim_x*blockDim_y*blockDim_z + i_offset)%num_states_; - while(Kokkos::atomic_compare_exchange(&locks_(i),0,1)) { - i+=blockDim_x*blockDim_y*blockDim_z; - if(i>=num_states_) {i = i_offset;} +template +struct rand { + KOKKOS_INLINE_FUNCTION + static short max() { return 32767; } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen) { + return short((gen.rand() & 0xffff + 65536) % 32768); } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& range) { + return short(gen.rand(range)); + } + KOKKOS_INLINE_FUNCTION + static short draw(Generator& gen, const short& start, const short& end) { + return short(gen.rand(start, end)); + } +}; - return Random_XorShift1024(state_, p_(i), i); -#else - return Random_XorShift1024(state_, p_(0), 0); -#endif -} +template +struct rand { + KOKKOS_INLINE_FUNCTION + static int max() { return Generator::MAX_RAND; } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen) { return gen.rand(); } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& range) { return gen.rand(range); } + KOKKOS_INLINE_FUNCTION + static int draw(Generator& gen, const int& start, const int& end) { + return gen.rand(start, end); + } +}; -template<> -KOKKOS_INLINE_FUNCTION -void Random_XorShift1024_Pool::free_state(const Random_XorShift1024 &state) const { -#ifdef __HCC_ACCELERATOR__ - for(int i=0; i<16; i++) - state_(state.state_idx_,i) = state.state_[i]; - locks_(state.state_idx_) = 0; - return; -#endif -} +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned int max() { return Generator::MAX_URAND; } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen) { return gen.urand(); } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen, const unsigned int& range) { + return gen.urand(range); + } + KOKKOS_INLINE_FUNCTION + static unsigned int draw(Generator& gen, const unsigned int& start, + const unsigned int& end) { + return gen.urand(start, end); + } +}; +template +struct rand { + KOKKOS_INLINE_FUNCTION + static long max() { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 ? static_cast(Generator::MAX_RAND) + : static_cast(Generator::MAX_RAND64); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 ? static_cast(gen.rand()) + : static_cast(gen.rand64()); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen, const long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 + ? static_cast(gen.rand(static_cast(range))) + : static_cast(gen.rand64(range)); + } + KOKKOS_INLINE_FUNCTION + static long draw(Generator& gen, const long& start, const long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(long) == 4 + ? static_cast( + gen.rand(static_cast(start), static_cast(end))) + : static_cast(gen.rand64(start, end)); + } +}; -#endif +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long max() { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast(Generator::MAX_URAND) + : static_cast(Generator::MAX_URAND64); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast(gen.urand()) + : static_cast(gen.urand64()); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen, const unsigned long& range) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast( + gen.urand(static_cast(range))) + : static_cast(gen.urand64(range)); + } + KOKKOS_INLINE_FUNCTION + static unsigned long draw(Generator& gen, const unsigned long& start, + const unsigned long& end) { + // FIXME (mfh 26 Oct 2014) It would be better to select the + // return value at compile time, using something like enable_if. + return sizeof(unsigned long) == 4 + ? static_cast( + gen.urand(static_cast(start), + static_cast(end))) + : static_cast(gen.urand64(start, end)); + } +}; +// NOTE (mfh 26 oct 2014) This is a partial specialization for long +// long, a C99 / C++11 signed type which is guaranteed to be at +// least 64 bits. Do NOT write a partial specialization for +// int64_t!!! This is just a typedef! It could be either long or +// long long. We don't know which a priori, and I've seen both. +// The types long and long long are guaranteed to differ, so it's +// always safe to specialize for both. +template +struct rand { + KOKKOS_INLINE_FUNCTION + static long long max() { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return Generator::MAX_RAND64; + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(); + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen, const long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(range); + } + KOKKOS_INLINE_FUNCTION + static long long draw(Generator& gen, const long long& start, + const long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.rand64(start, end); + } +}; + +// NOTE (mfh 26 oct 2014) This is a partial specialization for +// unsigned long long, a C99 / C++11 unsigned type which is +// guaranteed to be at least 64 bits. Do NOT write a partial +// specialization for uint64_t!!! This is just a typedef! It could +// be either unsigned long or unsigned long long. We don't know +// which a priori, and I've seen both. The types unsigned long and +// unsigned long long are guaranteed to differ, so it's always safe +// to specialize for both. +template +struct rand { + KOKKOS_INLINE_FUNCTION + static unsigned long long max() { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 + // bits. + return Generator::MAX_URAND64; + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen) { + // FIXME (mfh 26 Oct 2014) It's legal for unsigned long long to be > 64 + // bits. + return gen.urand64(); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen, + const unsigned long long& range) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64(range); + } + KOKKOS_INLINE_FUNCTION + static unsigned long long draw(Generator& gen, + const unsigned long long& start, + const unsigned long long& end) { + // FIXME (mfh 26 Oct 2014) It's legal for long long to be > 64 bits. + return gen.urand64(start, end); + } +}; + +template +struct rand { + KOKKOS_INLINE_FUNCTION + static float max() { return 1.0f; } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen) { return gen.frand(); } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& range) { + return gen.frand(range); + } + KOKKOS_INLINE_FUNCTION + static float draw(Generator& gen, const float& start, const float& end) { + return gen.frand(start, end); + } +}; + +template +struct rand { + KOKKOS_INLINE_FUNCTION + static double max() { return 1.0; } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen) { return gen.drand(); } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& range) { + return gen.drand(range); + } + KOKKOS_INLINE_FUNCTION + static double draw(Generator& gen, const double& start, const double& end) { + return gen.drand(start, end); + } +}; + +template +struct rand > { + KOKKOS_INLINE_FUNCTION + static Kokkos::complex max() { + return Kokkos::complex(1.0, 1.0); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen) { + const float re = gen.frand(); + const float im = gen.frand(); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& range) { + const float re = gen.frand(real(range)); + const float im = gen.frand(imag(range)); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& start, + const Kokkos::complex& end) { + const float re = gen.frand(real(start), real(end)); + const float im = gen.frand(imag(start), imag(end)); + return Kokkos::complex(re, im); + } +}; + +template +struct rand > { + KOKKOS_INLINE_FUNCTION + static Kokkos::complex max() { + return Kokkos::complex(1.0, 1.0); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen) { + const double re = gen.drand(); + const double im = gen.drand(); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& range) { + const double re = gen.drand(real(range)); + const double im = gen.drand(imag(range)); + return Kokkos::complex(re, im); + } + KOKKOS_INLINE_FUNCTION + static Kokkos::complex draw(Generator& gen, + const Kokkos::complex& start, + const Kokkos::complex& end) { + const double re = gen.drand(real(start), real(end)); + const double im = gen.drand(imag(start), imag(end)); + return Kokkos::complex(re, im); + } +}; + +template +class Random_XorShift1024_Pool; namespace Impl { -template +template +struct Random_XorShift1024_State { + uint64_t state_[16]; + KOKKOS_DEFAULTED_FUNCTION + Random_XorShift1024_State() = default; + + template + KOKKOS_FUNCTION Random_XorShift1024_State(const StateViewType& v, + int state_idx) { + for (int i = 0; i < 16; i++) state_[i] = v(state_idx, i); + } + + KOKKOS_FUNCTION + uint64_t operator[](const int i) const { return state_[i]; } + + KOKKOS_FUNCTION + uint64_t& operator[](const int i) { return state_[i]; } +}; + +template <> +struct Random_XorShift1024_State { + uint64_t* state_; + const int stride_; + KOKKOS_FUNCTION + Random_XorShift1024_State() : state_(nullptr), stride_(1){}; + + template + KOKKOS_FUNCTION Random_XorShift1024_State(const StateViewType& v, + int state_idx) + : state_(&v(state_idx, 0)), stride_(v.stride_1()) {} + + KOKKOS_FUNCTION + uint64_t operator[](const int i) const { return state_[i * stride_]; } + + KOKKOS_FUNCTION + uint64_t& operator[](const int i) { return state_[i * stride_]; } +}; + +template +struct Random_XorShift1024_UseCArrayState : std::true_type {}; + +#ifdef KOKKOS_ENABLE_CUDA +template <> +struct Random_XorShift1024_UseCArrayState : std::false_type {}; +#endif +#ifdef KOKKOS_ENABLE_HIP +template <> +struct Random_XorShift1024_UseCArrayState + : std::false_type {}; +#endif +#ifdef KOKKOS_ENABLE_OPENMPTARGET +template <> +struct Random_XorShift1024_UseCArrayState + : std::false_type {}; +#endif + +template +struct Random_UniqueIndex { + using locks_view_type = View; + KOKKOS_FUNCTION + static int get_state_idx(const locks_view_type) { +#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE + const int i = ExecutionSpace::hardware_thread_id(); +#else + const int i = ExecutionSpace::impl_hardware_thread_id(); +#endif + return i; +#else + return 0; +#endif + } +}; + +#ifdef KOKKOS_ENABLE_CUDA +template <> +struct Random_UniqueIndex { + using locks_view_type = View; + KOKKOS_FUNCTION + static int get_state_idx(const locks_view_type& locks_) { +#ifdef __CUDA_ARCH__ + const int i_offset = + (threadIdx.x * blockDim.y + threadIdx.y) * blockDim.z + threadIdx.z; + int i = (((blockIdx.x * gridDim.y + blockIdx.y) * gridDim.z + blockIdx.z) * + blockDim.x * blockDim.y * blockDim.z + + i_offset) % + locks_.extent(0); + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += blockDim.x * blockDim.y * blockDim.z; + if (i >= static_cast(locks_.extent(0))) { + i = i_offset; + } + } + return i; +#else + (void)locks_; + return 0; +#endif + } +}; +#endif + +#ifdef KOKKOS_ENABLE_HIP +template <> +struct Random_UniqueIndex { + using locks_view_type = View; + KOKKOS_FUNCTION + static int get_state_idx(const locks_view_type& locks_) { +#ifdef __HIP_DEVICE_COMPILE__ + const int i_offset = + (hipThreadIdx_x * hipBlockDim_y + hipThreadIdx_y) * hipBlockDim_z + + hipThreadIdx_z; + int i = (((hipBlockIdx_x * hipGridDim_y + hipBlockIdx_y) * hipGridDim_z + + hipBlockIdx_z) * + hipBlockDim_x * hipBlockDim_y * hipBlockDim_z + + i_offset) % + locks_.extent(0); + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i += hipBlockDim_x * hipBlockDim_y * hipBlockDim_z; + if (i >= static_cast(locks_.extent(0))) { + i = i_offset; + } + } + return i; +#else + (void)locks_; + return 0; +#endif + } +}; +#endif + +} // namespace Impl + +template +class Random_XorShift64_Pool; + +template +class Random_XorShift64 { + private: + uint64_t state_; + const int state_idx_; + friend class Random_XorShift64_Pool; + + public: + typedef DeviceType device_type; + + constexpr static uint32_t MAX_URAND = std::numeric_limits::max(); + constexpr static uint64_t MAX_URAND64 = std::numeric_limits::max(); + constexpr static int32_t MAX_RAND = std::numeric_limits::max(); + constexpr static int64_t MAX_RAND64 = std::numeric_limits::max(); + + KOKKOS_INLINE_FUNCTION + Random_XorShift64(uint64_t state, int state_idx = 0) + : state_(state == 0 ? uint64_t(1318319) : state), state_idx_(state_idx) {} + + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + + uint64_t tmp = state_ * 2685821657736338717ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + state_ ^= state_ >> 12; + state_ ^= state_ << 25; + state_ ^= state_ >> 27; + return (state_ * 2685821657736338717ULL) - 1; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) tmp = urand(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) tmp = urand64(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } + + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) tmp = rand(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) tmp = rand64(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + float frand() { return urand64() / static_cast(MAX_URAND64); } + + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { + return range * urand64() / static_cast(MAX_URAND64); + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + double drand() { return urand64() / static_cast(MAX_URAND64); } + + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { + return range * urand64() / static_cast(MAX_URAND64); + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return drand(end - start) + start; + } + + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { +#ifndef __HIP_DEVICE_COMPILE__ // FIXME_HIP + using std::sqrt; +#else + using ::sqrt; +#endif + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * sqrt(-2.0 * log(S) / S); + } + + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; + +template +class Random_XorShift64_Pool { + private: + using execution_space = typename DeviceType::execution_space; + typedef View locks_type; + typedef View state_data_type; + locks_type locks_; + state_data_type state_; + int num_states_; + + public: + typedef Random_XorShift64 generator_type; + typedef DeviceType device_type; + + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool() { num_states_ = 0; } + Random_XorShift64_Pool(uint64_t seed) { + num_states_ = 0; + + init(seed, execution_space().concurrency()); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool(const Random_XorShift64_Pool& src) + : locks_(src.locks_), state_(src.state_), num_states_(src.num_states_) {} + + KOKKOS_INLINE_FUNCTION + Random_XorShift64_Pool operator=(const Random_XorShift64_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + num_states_ = src.num_states_; + return *this; + } + + void init(uint64_t seed, int num_states) { + if (seed == 0) seed = uint64_t(1318319); + + num_states_ = num_states; + + locks_ = locks_type("Kokkos::Random_XorShift64::locks", num_states_); + state_ = state_data_type("Kokkos::Random_XorShift64::state", num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename locks_type::HostMirror h_lock = create_mirror_view(locks_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 + gen(seed, 0); + for (int i = 0; i < 17; i++) gen.rand(); + for (int i = 0; i < num_states_; i++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i) = (((static_cast(n1)) & 0xffff) << 00) | + (((static_cast(n2)) & 0xffff) << 16) | + (((static_cast(n3)) & 0xffff) << 32) | + (((static_cast(n4)) & 0xffff) << 48); + h_lock(i) = 0; + } + deep_copy(state_, h_state); + deep_copy(locks_, h_lock); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift64 get_state() const { + const int i = + Impl::Random_UniqueIndex::get_state_idx(locks_); + return Random_XorShift64(state_(i), i); + } + + // NOTE: state_idx MUST be unique and less than num_states + KOKKOS_INLINE_FUNCTION + Random_XorShift64 get_state(const int state_idx) const { + return Random_XorShift64(state_(state_idx), state_idx); + } + + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift64& state) const { + state_(state.state_idx_) = state.state_; + locks_(state.state_idx_) = 0; + } +}; + +template +class Random_XorShift1024 { + using execution_space = typename DeviceType::execution_space; + + private: + int p_; + const int state_idx_; + Impl::Random_XorShift1024_State< + Impl::Random_XorShift1024_UseCArrayState::value> + state_; + friend class Random_XorShift1024_Pool; + + public: + typedef Random_XorShift1024_Pool pool_type; + typedef DeviceType device_type; + + constexpr static uint32_t MAX_URAND = std::numeric_limits::max(); + constexpr static uint64_t MAX_URAND64 = std::numeric_limits::max(); + constexpr static int32_t MAX_RAND = std::numeric_limits::max(); + constexpr static int64_t MAX_RAND64 = std::numeric_limits::max(); + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024(const typename pool_type::state_data_type& state, int p, + int state_idx = 0) + : p_(p), state_idx_(state_idx), state_(state, state_idx) {} + + KOKKOS_INLINE_FUNCTION + uint32_t urand() { + uint64_t state_0 = state_[p_]; + uint64_t state_1 = state_[p_ = (p_ + 1) & 15]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + uint64_t tmp = (state_[p_] = state_0 ^ state_1) * 1181783497276652981ULL; + tmp = tmp >> 16; + return static_cast(tmp & MAX_URAND); + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64() { + uint64_t state_0 = state_[p_]; + uint64_t state_1 = state_[p_ = (p_ + 1) & 15]; + state_1 ^= state_1 << 31; + state_1 ^= state_1 >> 11; + state_0 ^= state_0 >> 30; + return ((state_[p_] = state_0 ^ state_1) * 1181783497276652981LL) - 1; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& range) { + const uint32_t max_val = (MAX_URAND / range) * range; + uint32_t tmp = urand(); + while (tmp >= max_val) tmp = urand(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + uint32_t urand(const uint32_t& start, const uint32_t& end) { + return urand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& range) { + const uint64_t max_val = (MAX_URAND64 / range) * range; + uint64_t tmp = urand64(); + while (tmp >= max_val) tmp = urand64(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + uint64_t urand64(const uint64_t& start, const uint64_t& end) { + return urand64(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + int rand() { return static_cast(urand() / 2); } + + KOKKOS_INLINE_FUNCTION + int rand(const int& range) { + const int max_val = (MAX_RAND / range) * range; + int tmp = rand(); + while (tmp >= max_val) tmp = rand(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + int rand(const int& start, const int& end) { + return rand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64() { return static_cast(urand64() / 2); } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& range) { + const int64_t max_val = (MAX_RAND64 / range) * range; + int64_t tmp = rand64(); + while (tmp >= max_val) tmp = rand64(); + return tmp % range; + } + + KOKKOS_INLINE_FUNCTION + int64_t rand64(const int64_t& start, const int64_t& end) { + return rand64(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + float frand() { return urand64() / static_cast(MAX_URAND64); } + + KOKKOS_INLINE_FUNCTION + float frand(const float& range) { + return range * urand64() / static_cast(MAX_URAND64); + } + + KOKKOS_INLINE_FUNCTION + float frand(const float& start, const float& end) { + return frand(end - start) + start; + } + + KOKKOS_INLINE_FUNCTION + double drand() { return urand64() / static_cast(MAX_URAND64); } + + KOKKOS_INLINE_FUNCTION + double drand(const double& range) { + return range * urand64() / static_cast(MAX_URAND64); + } + + KOKKOS_INLINE_FUNCTION + double drand(const double& start, const double& end) { + return frand(end - start) + start; + } + + // Marsaglia polar method for drawing a standard normal distributed random + // number + KOKKOS_INLINE_FUNCTION + double normal() { +#ifndef KOKKOS_ENABLE_HIP // FIXME_HIP + using std::sqrt; +#else + using ::sqrt; +#endif + double S = 2.0; + double U; + while (S >= 1.0) { + U = 2.0 * drand() - 1.0; + const double V = 2.0 * drand() - 1.0; + S = U * U + V * V; + } + return U * sqrt(-2.0 * log(S) / S); + } + + KOKKOS_INLINE_FUNCTION + double normal(const double& mean, const double& std_dev = 1.0) { + return mean + normal() * std_dev; + } +}; + +template +class Random_XorShift1024_Pool { + private: + using execution_space = typename DeviceType::execution_space; + typedef View locks_type; + typedef View int_view_type; + typedef View state_data_type; + + locks_type locks_; + state_data_type state_; + int_view_type p_; + int num_states_; + friend class Random_XorShift1024; + + public: + typedef Random_XorShift1024 generator_type; + + typedef DeviceType device_type; + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool() { num_states_ = 0; } + + inline Random_XorShift1024_Pool(uint64_t seed) { + num_states_ = 0; + + init(seed, execution_space().concurrency()); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool(const Random_XorShift1024_Pool& src) + : locks_(src.locks_), + state_(src.state_), + p_(src.p_), + num_states_(src.num_states_) {} + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024_Pool operator=(const Random_XorShift1024_Pool& src) { + locks_ = src.locks_; + state_ = src.state_; + p_ = src.p_; + num_states_ = src.num_states_; + return *this; + } + + inline void init(uint64_t seed, int num_states) { + if (seed == 0) seed = uint64_t(1318319); + num_states_ = num_states; + locks_ = locks_type("Kokkos::Random_XorShift1024::locks", num_states_); + state_ = state_data_type("Kokkos::Random_XorShift1024::state", num_states_); + p_ = int_view_type("Kokkos::Random_XorShift1024::p", num_states_); + + typename state_data_type::HostMirror h_state = create_mirror_view(state_); + typename locks_type::HostMirror h_lock = create_mirror_view(locks_); + typename int_view_type::HostMirror h_p = create_mirror_view(p_); + + // Execute on the HostMirror's default execution space. + Random_XorShift64 + gen(seed, 0); + for (int i = 0; i < 17; i++) gen.rand(); + for (int i = 0; i < num_states_; i++) { + for (int j = 0; j < 16; j++) { + int n1 = gen.rand(); + int n2 = gen.rand(); + int n3 = gen.rand(); + int n4 = gen.rand(); + h_state(i, j) = (((static_cast(n1)) & 0xffff) << 00) | + (((static_cast(n2)) & 0xffff) << 16) | + (((static_cast(n3)) & 0xffff) << 32) | + (((static_cast(n4)) & 0xffff) << 48); + } + h_p(i) = 0; + h_lock(i) = 0; + } + deep_copy(state_, h_state); + deep_copy(locks_, h_lock); + } + + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 get_state() const { + const int i = + Impl::Random_UniqueIndex::get_state_idx(locks_); + return Random_XorShift1024(state_, p_(i), i); + }; + + // NOTE: state_idx MUST be unique and less than num_states + KOKKOS_INLINE_FUNCTION + Random_XorShift1024 get_state(const int state_idx) const { + return Random_XorShift1024(state_, p_(state_idx), state_idx); + } + + KOKKOS_INLINE_FUNCTION + void free_state(const Random_XorShift1024& state) const { + for (int i = 0; i < 16; i++) state_(state.state_idx_, i) = state.state_[i]; + p_(state.state_idx_) = state.p_; + locks_(state.state_idx_) = 0; + } +}; + +namespace Impl { + +template struct fill_random_functor_range; -template +template struct fill_random_functor_begin_end; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (const IndexType& i) const { + void operator()(const IndexType& i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) - a(idx) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) + a(idx) = Rand::draw(gen, range); } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - a(idx,k) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + a(idx, k) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - a(idx,k,l) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + a(idx, k, l) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - a(idx,k,l,m) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + a(idx, k, l, m) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - a(idx,k,l,m,n) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + a(idx, k, l, m, n) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - a(idx,k,l,m,n,o) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + a(idx, k, l, m, n, o) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - a(idx,k,l,m,n,o,p) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + a(idx, k, l, m, n, o, p) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_range{ +template +struct fill_random_functor_range { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; typename ViewType::const_value_type range; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_range(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type range_): - a(a_),rand_pool(rand_pool_),range(range_) {} + typename ViewType::const_value_type range_) + : a(a_), rand_pool(rand_pool_), range(range_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - for(IndexType q=0;q(a.extent(7));q++) - a(idx,k,l,m,n,o,p,q) = Rand::draw(gen,range); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + for (IndexType q = 0; + q < static_cast(a.extent(7)); q++) + a(idx, k, l, m, n, o, p, q) = Rand::draw(gen, range); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) - a(idx) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) + a(idx) = Rand::draw(gen, begin, end); } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - a(idx,k) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + a(idx, k) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - a(idx,k,l) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + a(idx, k, l) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - a(idx,k,l,m) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + a(idx, k, l, m) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))){ - for(IndexType l=0;l(a.extent(1));l++) - for(IndexType m=0;m(a.extent(2));m++) - for(IndexType n=0;n(a.extent(3));n++) - for(IndexType o=0;o(a.extent(4));o++) - a(idx,l,m,n,o) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType l = 0; l < static_cast(a.extent(1)); l++) + for (IndexType m = 0; m < static_cast(a.extent(2)); m++) + for (IndexType n = 0; n < static_cast(a.extent(3)); n++) + for (IndexType o = 0; o < static_cast(a.extent(4)); + o++) + a(idx, l, m, n, o) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - a(idx,k,l,m,n,o) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + a(idx, k, l, m, n, o) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; - -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - a(idx,k,l,m,n,o,p) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + a(idx, k, l, m, n, o, p) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -template -struct fill_random_functor_begin_end{ +template +struct fill_random_functor_begin_end { typedef typename ViewType::execution_space execution_space; ViewType a; RandomPool rand_pool; - typename ViewType::const_value_type begin,end; + typename ViewType::const_value_type begin, end; - typedef rand Rand; + typedef rand + Rand; fill_random_functor_begin_end(ViewType a_, RandomPool rand_pool_, - typename ViewType::const_value_type begin_, typename ViewType::const_value_type end_): - a(a_),rand_pool(rand_pool_),begin(begin_),end(end_) {} + typename ViewType::const_value_type begin_, + typename ViewType::const_value_type end_) + : a(a_), rand_pool(rand_pool_), begin(begin_), end(end_) {} KOKKOS_INLINE_FUNCTION - void operator() (IndexType i) const { + void operator()(IndexType i) const { typename RandomPool::generator_type gen = rand_pool.get_state(); - for(IndexType j=0;j(a.extent(0))) { - for(IndexType k=0;k(a.extent(1));k++) - for(IndexType l=0;l(a.extent(2));l++) - for(IndexType m=0;m(a.extent(3));m++) - for(IndexType n=0;n(a.extent(4));n++) - for(IndexType o=0;o(a.extent(5));o++) - for(IndexType p=0;p(a.extent(6));p++) - for(IndexType q=0;q(a.extent(7));q++) - a(idx,k,l,m,n,o,p,q) = Rand::draw(gen,begin,end); + for (IndexType j = 0; j < loops; j++) { + const IndexType idx = i * loops + j; + if (idx < static_cast(a.extent(0))) { + for (IndexType k = 0; k < static_cast(a.extent(1)); k++) + for (IndexType l = 0; l < static_cast(a.extent(2)); l++) + for (IndexType m = 0; m < static_cast(a.extent(3)); m++) + for (IndexType n = 0; n < static_cast(a.extent(4)); + n++) + for (IndexType o = 0; o < static_cast(a.extent(5)); + o++) + for (IndexType p = 0; p < static_cast(a.extent(6)); + p++) + for (IndexType q = 0; + q < static_cast(a.extent(7)); q++) + a(idx, k, l, m, n, o, p, q) = Rand::draw(gen, begin, end); } } rand_pool.free_state(gen); } }; -} +} // namespace Impl -template -void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type range) { +template +void fill_random(ViewType a, RandomPool g, + typename ViewType::const_value_type range) { int64_t LDA = a.extent(0); - if(LDA>0) - parallel_for((LDA+127)/128,Impl::fill_random_functor_range(a,g,range)); + if (LDA > 0) + parallel_for("Kokkos::fill_random", (LDA + 127) / 128, + Impl::fill_random_functor_range( + a, g, range)); } -template -void fill_random(ViewType a, RandomPool g, typename ViewType::const_value_type begin,typename ViewType::const_value_type end ) { +template +void fill_random(ViewType a, RandomPool g, + typename ViewType::const_value_type begin, + typename ViewType::const_value_type end) { int64_t LDA = a.extent(0); - if(LDA>0) - parallel_for((LDA+127)/128,Impl::fill_random_functor_begin_end(a,g,begin,end)); -} + if (LDA > 0) + parallel_for("Kokkos::fill_random", (LDA + 127) / 128, + Impl::fill_random_functor_begin_end( + a, g, begin, end)); } +} // namespace Kokkos #endif diff --git a/lib/kokkos/algorithms/src/Kokkos_Sort.hpp b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp index 7fb8505fe5..1c79a505bb 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Sort.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Sort.hpp @@ -1,13 +1,14 @@ /* //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -36,12 +37,11 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER */ - #ifndef KOKKOS_SORT_HPP_ #define KOKKOS_SORT_HPP_ @@ -51,125 +51,107 @@ namespace Kokkos { - namespace Impl { +namespace Impl { - template< class DstViewType , class SrcViewType - , int Rank = DstViewType::Rank > - struct CopyOp; +template +struct CopyOp; - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - dst(i_dst) = src(i_src); - } - }; - - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - for(int j = 0;j< (int) dst.extent(1); j++) - dst(i_dst,j) = src(i_src,j); - } - }; - - template< class DstViewType , class SrcViewType > - struct CopyOp { - KOKKOS_INLINE_FUNCTION - static void copy(DstViewType const& dst, size_t i_dst, - SrcViewType const& src, size_t i_src ) { - for(int j = 0; j +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + dst(i_dst) = src(i_src); } +}; + +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < (int)dst.extent(1); j++) dst(i_dst, j) = src(i_src, j); + } +}; + +template +struct CopyOp { + KOKKOS_INLINE_FUNCTION + static void copy(DstViewType const& dst, size_t i_dst, SrcViewType const& src, + size_t i_src) { + for (int j = 0; j < dst.extent(1); j++) + for (int k = 0; k < dst.extent(2); k++) + dst(i_dst, j, k) = src(i_src, j, k); + } +}; +} // namespace Impl //---------------------------------------------------------------------------- -template< class KeyViewType - , class BinSortOp - , class Space = typename KeyViewType::device_type - , class SizeType = typename KeyViewType::memory_space::size_type - > +template class BinSort { -public: - - template< class DstViewType , class SrcViewType > + public: + template struct copy_functor { + typedef typename SrcViewType::const_type src_view_type; - typedef typename SrcViewType::const_type src_view_type ; + typedef Impl::CopyOp copy_op; - typedef Impl::CopyOp< DstViewType , src_view_type > copy_op ; + DstViewType dst_values; + src_view_type src_values; + int dst_offset; - DstViewType dst_values ; - src_view_type src_values ; - int dst_offset ; - - copy_functor( DstViewType const & dst_values_ - , int const & dst_offset_ - , SrcViewType const & src_values_ - ) - : dst_values( dst_values_ ) - , src_values( src_values_ ) - , dst_offset( dst_offset_ ) - {} + copy_functor(DstViewType const& dst_values_, int const& dst_offset_, + SrcViewType const& src_values_) + : dst_values(dst_values_), + src_values(src_values_), + dst_offset(dst_offset_) {} KOKKOS_INLINE_FUNCTION - void operator() (const int& i) const { - copy_op::copy(dst_values,i+dst_offset,src_values,i); + void operator()(const int& i) const { + copy_op::copy(dst_values, i + dst_offset, src_values, i); } }; - template< class DstViewType - , class PermuteViewType - , class SrcViewType - > + template struct copy_permute_functor { - // If a Kokkos::View then can generate constant random access // otherwise can only use the constant type. - typedef typename std::conditional - < Kokkos::is_view< SrcViewType >::value - , Kokkos::View< typename SrcViewType::const_data_type - , typename SrcViewType::array_layout - , typename SrcViewType::device_type - , Kokkos::MemoryTraits - > - , typename SrcViewType::const_type - >::type src_view_type ; + typedef typename std::conditional< + Kokkos::is_view::value, + Kokkos::View >, + typename SrcViewType::const_type>::type src_view_type; - typedef typename PermuteViewType::const_type perm_view_type ; + typedef typename PermuteViewType::const_type perm_view_type; - typedef Impl::CopyOp< DstViewType , src_view_type > copy_op ; + typedef Impl::CopyOp copy_op; - DstViewType dst_values ; - perm_view_type sort_order ; - src_view_type src_values ; - int src_offset ; + DstViewType dst_values; + perm_view_type sort_order; + src_view_type src_values; + int src_offset; - copy_permute_functor( DstViewType const & dst_values_ - , PermuteViewType const & sort_order_ - , SrcViewType const & src_values_ - , int const & src_offset_ - ) - : dst_values( dst_values_ ) - , sort_order( sort_order_ ) - , src_values( src_values_ ) - , src_offset( src_offset_ ) - {} + copy_permute_functor(DstViewType const& dst_values_, + PermuteViewType const& sort_order_, + SrcViewType const& src_values_, int const& src_offset_) + : dst_values(dst_values_), + sort_order(sort_order_), + src_values(src_values_), + src_offset(src_offset_) {} KOKKOS_INLINE_FUNCTION - void operator() (const int& i) const { - copy_op::copy(dst_values,i,src_values,src_offset+sort_order(i)); + void operator()(const int& i) const { + copy_op::copy(dst_values, i, src_values, src_offset + sort_order(i)); } }; - typedef typename Space::execution_space execution_space; + typedef typename Space::execution_space execution_space; typedef BinSortOp bin_op_type; struct bin_count_tag {}; @@ -177,221 +159,236 @@ public: struct bin_binning_tag {}; struct bin_sort_bins_tag {}; -public: - + public: typedef SizeType size_type; typedef size_type value_type; typedef Kokkos::View offset_type; typedef Kokkos::View bin_count_type; - typedef typename KeyViewType::const_type const_key_view_type ; + typedef typename KeyViewType::const_type const_key_view_type; // If a Kokkos::View then can generate constant random access // otherwise can only use the constant type. - typedef typename std::conditional - < Kokkos::is_view< KeyViewType >::value - , Kokkos::View< typename KeyViewType::const_data_type, - typename KeyViewType::array_layout, - typename KeyViewType::device_type, - Kokkos::MemoryTraits > - , const_key_view_type - >::type const_rnd_key_view_type; + typedef typename std::conditional< + Kokkos::is_view::value, + Kokkos::View >, + const_key_view_type>::type const_rnd_key_view_type; typedef typename KeyViewType::non_const_value_type non_const_key_scalar; - typedef typename KeyViewType::const_value_type const_key_scalar; + typedef typename KeyViewType::const_value_type const_key_scalar; - typedef Kokkos::View > bin_count_atomic_type ; - -private: + typedef Kokkos::View > + bin_count_atomic_type; + private: const_key_view_type keys; const_rnd_key_view_type keys_rnd; -public: + public: + BinSortOp bin_op; + offset_type bin_offsets; + bin_count_atomic_type bin_count_atomic; + bin_count_type bin_count_const; + offset_type sort_order; - BinSortOp bin_op ; - offset_type bin_offsets ; - bin_count_atomic_type bin_count_atomic ; - bin_count_type bin_count_const ; - offset_type sort_order ; + int range_begin; + int range_end; + bool sort_within_bins; - int range_begin ; - int range_end ; - bool sort_within_bins ; - -public: - - BinSort() {} + public: + BinSort() = default; //---------------------------------------- - // Constructor: takes the keys, the binning_operator and optionally whether to sort within bins (default false) - BinSort( const_key_view_type keys_ - , int range_begin_ - , int range_end_ - , BinSortOp bin_op_ - , bool sort_within_bins_ = false - ) - : keys(keys_) - , keys_rnd(keys_) - , bin_op(bin_op_) - , bin_offsets() - , bin_count_atomic() - , bin_count_const() - , sort_order() - , range_begin( range_begin_ ) - , range_end( range_end_ ) - , sort_within_bins( sort_within_bins_ ) - { - bin_count_atomic = Kokkos::View("Kokkos::SortImpl::BinSortFunctor::bin_count",bin_op.max_bins()); - bin_count_const = bin_count_atomic; - bin_offsets = offset_type(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::bin_offsets"),bin_op.max_bins()); - sort_order = offset_type(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sort_order"),range_end-range_begin); + // Constructor: takes the keys, the binning_operator and optionally whether to + // sort within bins (default false) + BinSort(const_key_view_type keys_, int range_begin_, int range_end_, + BinSortOp bin_op_, bool sort_within_bins_ = false) + : keys(keys_), + keys_rnd(keys_), + bin_op(bin_op_), + bin_offsets(), + bin_count_atomic(), + bin_count_const(), + sort_order(), + range_begin(range_begin_), + range_end(range_end_), + sort_within_bins(sort_within_bins_) { + bin_count_atomic = Kokkos::View( + "Kokkos::SortImpl::BinSortFunctor::bin_count", bin_op.max_bins()); + bin_count_const = bin_count_atomic; + bin_offsets = + offset_type(ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::bin_offsets"), + bin_op.max_bins()); + sort_order = + offset_type(ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sort_order"), + range_end - range_begin); } - BinSort( const_key_view_type keys_ - , BinSortOp bin_op_ - , bool sort_within_bins_ = false - ) - : BinSort( keys_ , 0 , keys_.extent(0), bin_op_ , sort_within_bins_ ) {} + BinSort(const_key_view_type keys_, BinSortOp bin_op_, + bool sort_within_bins_ = false) + : BinSort(keys_, 0, keys_.extent(0), bin_op_, sort_within_bins_) {} //---------------------------------------- - // Create the permutation vector, the bin_offset array and the bin_count array. Can be called again if keys changed + // Create the permutation vector, the bin_offset array and the bin_count + // array. Can be called again if keys changed void create_permute_vector() { - const size_t len = range_end - range_begin ; - Kokkos::parallel_for ("Kokkos::Sort::BinCount",Kokkos::RangePolicy (0,len),*this); - Kokkos::parallel_scan("Kokkos::Sort::BinOffset",Kokkos::RangePolicy (0,bin_op.max_bins()) ,*this); + const size_t len = range_end - range_begin; + Kokkos::parallel_for( + "Kokkos::Sort::BinCount", + Kokkos::RangePolicy(0, len), *this); + Kokkos::parallel_scan("Kokkos::Sort::BinOffset", + Kokkos::RangePolicy( + 0, bin_op.max_bins()), + *this); - Kokkos::deep_copy(bin_count_atomic,0); - Kokkos::parallel_for ("Kokkos::Sort::BinBinning",Kokkos::RangePolicy (0,len),*this); + Kokkos::deep_copy(bin_count_atomic, 0); + Kokkos::parallel_for( + "Kokkos::Sort::BinBinning", + Kokkos::RangePolicy(0, len), *this); - if(sort_within_bins) - Kokkos::parallel_for ("Kokkos::Sort::BinSort",Kokkos::RangePolicy(0,bin_op.max_bins()) ,*this); + if (sort_within_bins) + Kokkos::parallel_for( + "Kokkos::Sort::BinSort", + Kokkos::RangePolicy( + 0, bin_op.max_bins()), + *this); } - // Sort a subset of a view with respect to the first dimension using the permutation array - template - void sort( ValuesViewType const & values - , int values_range_begin - , int values_range_end) const - { - typedef - Kokkos::View< typename ValuesViewType::data_type, - typename ValuesViewType::array_layout, - typename ValuesViewType::device_type > - scratch_view_type ; + // Sort a subset of a view with respect to the first dimension using the + // permutation array + template + void sort(ValuesViewType const& values, int values_range_begin, + int values_range_end) const { + typedef Kokkos::View + scratch_view_type; - const size_t len = range_end - range_begin ; - const size_t values_len = values_range_end - values_range_begin ; + const size_t len = range_end - range_begin; + const size_t values_len = values_range_end - values_range_begin; if (len != values_len) { - Kokkos::abort("BinSort::sort: values range length != permutation vector length"); + Kokkos::abort( + "BinSort::sort: values range length != permutation vector length"); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - scratch_view_type - sorted_values(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sorted_values"), - len, - values.extent(1), - values.extent(2), - values.extent(3), - values.extent(4), - values.extent(5), - values.extent(6), - values.extent(7)); + scratch_view_type sorted_values( + ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sorted_values"), + len, values.extent(1), values.extent(2), values.extent(3), + values.extent(4), values.extent(5), values.extent(6), values.extent(7)); #else - scratch_view_type - sorted_values(ViewAllocateWithoutInitializing("Kokkos::SortImpl::BinSortFunctor::sorted_values"), - values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 1 ? values.extent(1) : KOKKOS_IMPL_CTOR_DEFAULT_ARG , - values.rank_dynamic > 2 ? values.extent(2) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 3 ? values.extent(3) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 4 ? values.extent(4) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 5 ? values.extent(5) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 6 ? values.extent(6) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, - values.rank_dynamic > 7 ? values.extent(7) : KOKKOS_IMPL_CTOR_DEFAULT_ARG); + scratch_view_type sorted_values( + ViewAllocateWithoutInitializing( + "Kokkos::SortImpl::BinSortFunctor::sorted_values"), + values.rank_dynamic > 0 ? len : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 1 ? values.extent(1) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 2 ? values.extent(2) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 3 ? values.extent(3) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 4 ? values.extent(4) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 5 ? values.extent(5) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 6 ? values.extent(6) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + values.rank_dynamic > 7 ? values.extent(7) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG); #endif { - copy_permute_functor< scratch_view_type /* DstViewType */ - , offset_type /* PermuteViewType */ - , ValuesViewType /* SrcViewType */ - > - functor( sorted_values , sort_order , values, values_range_begin - range_begin ); + copy_permute_functor + functor(sorted_values, sort_order, values, + values_range_begin - range_begin); - parallel_for("Kokkos::Sort::CopyPermute", Kokkos::RangePolicy(0,len),functor); + parallel_for("Kokkos::Sort::CopyPermute", + Kokkos::RangePolicy(0, len), functor); } { - copy_functor< ValuesViewType , scratch_view_type > - functor( values , range_begin , sorted_values ); + copy_functor functor( + values, range_begin, sorted_values); - parallel_for("Kokkos::Sort::Copy", Kokkos::RangePolicy(0,len),functor); + parallel_for("Kokkos::Sort::Copy", + Kokkos::RangePolicy(0, len), functor); } - Kokkos::fence(); + execution_space().fence(); } - template - void sort( ValuesViewType const & values ) const - { - this->sort( values, 0, /*values.extent(0)*/ range_end - range_begin ); + template + void sort(ValuesViewType const& values) const { + this->sort(values, 0, /*values.extent(0)*/ range_end - range_begin); } // Get the permutation vector KOKKOS_INLINE_FUNCTION - offset_type get_permute_vector() const { return sort_order;} + offset_type get_permute_vector() const { return sort_order; } // Get the start offsets for each bin KOKKOS_INLINE_FUNCTION - offset_type get_bin_offsets() const { return bin_offsets;} + offset_type get_bin_offsets() const { return bin_offsets; } // Get the count for each bin KOKKOS_INLINE_FUNCTION - bin_count_type get_bin_count() const {return bin_count_const;} - -public: + bin_count_type get_bin_count() const { return bin_count_const; } + public: KOKKOS_INLINE_FUNCTION - void operator() (const bin_count_tag& tag, const int& i) const { - const int j = range_begin + i ; + void operator()(const bin_count_tag& /*tag*/, const int i) const { + const int j = range_begin + i; bin_count_atomic(bin_op.bin(keys, j))++; } KOKKOS_INLINE_FUNCTION - void operator() (const bin_offset_tag& tag, const int& i, value_type& offset, const bool& final) const { - if(final) { + void operator()(const bin_offset_tag& /*tag*/, const int i, + value_type& offset, const bool& final) const { + if (final) { bin_offsets(i) = offset; } - offset+=bin_count_const(i); + offset += bin_count_const(i); } KOKKOS_INLINE_FUNCTION - void operator() (const bin_binning_tag& tag, const int& i) const { - const int j = range_begin + i ; - const int bin = bin_op.bin(keys,j); + void operator()(const bin_binning_tag& /*tag*/, const int i) const { + const int j = range_begin + i; + const int bin = bin_op.bin(keys, j); const int count = bin_count_atomic(bin)++; - sort_order(bin_offsets(bin) + count) = j ; + sort_order(bin_offsets(bin) + count) = j; } KOKKOS_INLINE_FUNCTION - void operator() (const bin_sort_bins_tag& tag, const int&i ) const { + void operator()(const bin_sort_bins_tag& /*tag*/, const int i) const { auto bin_size = bin_count_const(i); if (bin_size <= 1) return; - int upper_bound = bin_offsets(i)+bin_size; - bool sorted = false; - while(!sorted) { - sorted = true; + int upper_bound = bin_offsets(i) + bin_size; + bool sorted = false; + while (!sorted) { + sorted = true; int old_idx = sort_order(bin_offsets(i)); - int new_idx; - for(int k=bin_offsets(i)+1; k +template struct BinOp1D { int max_bins_; double mul_; typename KeyViewType::const_value_type range_; typename KeyViewType::const_value_type min_; - BinOp1D():max_bins_(0),mul_(0.0), - range_(typename KeyViewType::const_value_type()), - min_(typename KeyViewType::const_value_type()) {} + BinOp1D() + : max_bins_(0), + mul_(0.0), + range_(typename KeyViewType::const_value_type()), + min_(typename KeyViewType::const_value_type()) {} - //Construct BinOp with number of bins, minimum value and maxuimum value + // Construct BinOp with number of bins, minimum value and maxuimum value BinOp1D(int max_bins__, typename KeyViewType::const_value_type min, - typename KeyViewType::const_value_type max ) - :max_bins_(max_bins__+1),mul_(1.0*max_bins__/(max-min)),range_(max-min),min_(min) {} + typename KeyViewType::const_value_type max) + : max_bins_(max_bins__ + 1), + mul_(1.0 * max_bins__ / (max - min)), + range_(max - min), + min_(min) {} - //Determine bin index from key value - template - KOKKOS_INLINE_FUNCTION - int bin(ViewType& keys, const int& i) const { - return int(mul_*(keys(i)-min_)); + // Determine bin index from key value + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return int(mul_ * (keys(i) - min_)); } - //Return maximum bin index + 1 + // Return maximum bin index + 1 KOKKOS_INLINE_FUNCTION - int max_bins() const { - return max_bins_; - } + int max_bins() const { return max_bins_; } - //Compare to keys within a bin if true new_val will be put before old_val - template - KOKKOS_INLINE_FUNCTION - bool operator()(ViewType& keys, iType1& i1, iType2& i2) const { - return keys(i1) + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + return keys(i1) < keys(i2); } }; -template +template struct BinOp3D { int max_bins_[3]; double mul_[3]; typename KeyViewType::non_const_value_type range_[3]; typename KeyViewType::non_const_value_type min_[3]; - BinOp3D() {} + BinOp3D() = default; BinOp3D(int max_bins__[], typename KeyViewType::const_value_type min[], - typename KeyViewType::const_value_type max[] ) - { + typename KeyViewType::const_value_type max[]) { max_bins_[0] = max_bins__[0]; max_bins_[1] = max_bins__[1]; max_bins_[2] = max_bins__[2]; - mul_[0] = 1.0*max_bins__[0]/(max[0]-min[0]); - mul_[1] = 1.0*max_bins__[1]/(max[1]-min[1]); - mul_[2] = 1.0*max_bins__[2]/(max[2]-min[2]); - range_[0] = max[0]-min[0]; - range_[1] = max[1]-min[1]; - range_[2] = max[2]-min[2]; - min_[0] = min[0]; - min_[1] = min[1]; - min_[2] = min[2]; + mul_[0] = 1.0 * max_bins__[0] / (max[0] - min[0]); + mul_[1] = 1.0 * max_bins__[1] / (max[1] - min[1]); + mul_[2] = 1.0 * max_bins__[2] / (max[2] - min[2]); + range_[0] = max[0] - min[0]; + range_[1] = max[1] - min[1]; + range_[2] = max[2] - min[2]; + min_[0] = min[0]; + min_[1] = min[1]; + min_[2] = min[2]; } - template - KOKKOS_INLINE_FUNCTION - int bin(ViewType& keys, const int& i) const { - return int( (((int(mul_[0]*(keys(i,0)-min_[0]))*max_bins_[1]) + - int(mul_[1]*(keys(i,1)-min_[1])))*max_bins_[2]) + - int(mul_[2]*(keys(i,2)-min_[2]))); + template + KOKKOS_INLINE_FUNCTION int bin(ViewType& keys, const int& i) const { + return int((((int(mul_[0] * (keys(i, 0) - min_[0])) * max_bins_[1]) + + int(mul_[1] * (keys(i, 1) - min_[1]))) * + max_bins_[2]) + + int(mul_[2] * (keys(i, 2) - min_[2]))); } KOKKOS_INLINE_FUNCTION - int max_bins() const { - return max_bins_[0]*max_bins_[1]*max_bins_[2]; - } + int max_bins() const { return max_bins_[0] * max_bins_[1] * max_bins_[2]; } - template - KOKKOS_INLINE_FUNCTION - bool operator()(ViewType& keys, iType1& i1 , iType2& i2) const { - if (keys(i1,0)>keys(i2,0)) return true; - else if (keys(i1,0)==keys(i2,0)) { - if (keys(i1,1)>keys(i2,1)) return true; - else if (keys(i1,1)==keys(i2,1)) { - if (keys(i1,2)>keys(i2,2)) return true; + template + KOKKOS_INLINE_FUNCTION bool operator()(ViewType& keys, iType1& i1, + iType2& i2) const { + if (keys(i1, 0) > keys(i2, 0)) + return true; + else if (keys(i1, 0) == keys(i2, 0)) { + if (keys(i1, 1) > keys(i2, 1)) + return true; + else if (keys(i1, 1) == keys(i2, 1)) { + if (keys(i1, 2) > keys(i2, 2)) return true; } } return false; @@ -495,85 +493,80 @@ struct BinOp3D { namespace Impl { -template +template bool try_std_sort(ViewType view) { - bool possible = true; - size_t stride[8] = { view.stride_0() - , view.stride_1() - , view.stride_2() - , view.stride_3() - , view.stride_4() - , view.stride_5() - , view.stride_6() - , view.stride_7() - }; - possible = possible && std::is_same::value; - possible = possible && (ViewType::Rank == 1); - possible = possible && (stride[0] == 1); - if(possible) { - std::sort(view.data(),view.data()+view.extent(0)); + bool possible = true; + size_t stride[8] = {view.stride_0(), view.stride_1(), view.stride_2(), + view.stride_3(), view.stride_4(), view.stride_5(), + view.stride_6(), view.stride_7()}; + possible = possible && + std::is_same::value; + possible = possible && (ViewType::Rank == 1); + possible = possible && (stride[0] == 1); + if (possible) { + std::sort(view.data(), view.data() + view.extent(0)); } return possible; } -template +template struct min_max_functor { - typedef Kokkos::MinMaxScalar minmax_scalar; + typedef Kokkos::MinMaxScalar + minmax_scalar; ViewType view; - min_max_functor(const ViewType& view_):view(view_) {} + min_max_functor(const ViewType& view_) : view(view_) {} KOKKOS_INLINE_FUNCTION - void operator() (const size_t& i, minmax_scalar& minmax) const { - if(view(i) < minmax.min_val) minmax.min_val = view(i); - if(view(i) > minmax.max_val) minmax.max_val = view(i); + void operator()(const size_t& i, minmax_scalar& minmax) const { + if (view(i) < minmax.min_val) minmax.min_val = view(i); + if (view(i) > minmax.max_val) minmax.max_val = view(i); } }; -} +} // namespace Impl -template -void sort( ViewType const & view , bool const always_use_kokkos_sort = false) -{ - if(!always_use_kokkos_sort) { - if(Impl::try_std_sort(view)) return; +template +void sort(ViewType const& view, bool const always_use_kokkos_sort = false) { + if (!always_use_kokkos_sort) { + if (Impl::try_std_sort(view)) return; } typedef BinOp1D CompType; Kokkos::MinMaxScalar result; Kokkos::MinMax reducer(result); - parallel_reduce("Kokkos::Sort::FindExtent",Kokkos::RangePolicy(0,view.extent(0)), - Impl::min_max_functor(view),reducer); - if(result.min_val == result.max_val) return; - BinSort bin_sort(view,CompType(view.extent(0)/2,result.min_val,result.max_val),true); + parallel_reduce("Kokkos::Sort::FindExtent", + Kokkos::RangePolicy( + 0, view.extent(0)), + Impl::min_max_functor(view), reducer); + if (result.min_val == result.max_val) return; + BinSort bin_sort( + view, CompType(view.extent(0) / 2, result.min_val, result.max_val), true); bin_sort.create_permute_vector(); bin_sort.sort(view); } -template -void sort( ViewType view - , size_t const begin - , size_t const end - ) -{ - typedef Kokkos::RangePolicy range_policy ; +template +void sort(ViewType view, size_t const begin, size_t const end) { + typedef Kokkos::RangePolicy range_policy; typedef BinOp1D CompType; Kokkos::MinMaxScalar result; Kokkos::MinMax reducer(result); - parallel_reduce("Kokkos::Sort::FindExtent", range_policy( begin , end ) - , Impl::min_max_functor(view),reducer ); + parallel_reduce("Kokkos::Sort::FindExtent", range_policy(begin, end), + Impl::min_max_functor(view), reducer); - if(result.min_val == result.max_val) return; + if (result.min_val == result.max_val) return; - BinSort - bin_sort(view,begin,end,CompType((end-begin)/2,result.min_val,result.max_val),true); + BinSort bin_sort( + view, begin, end, + CompType((end - begin) / 2, result.min_val, result.max_val), true); bin_sort.create_permute_vector(); - bin_sort.sort(view,begin,end); + bin_sort.sort(view, begin, end); } -} +} // namespace Kokkos #endif diff --git a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt index e238b37c8e..e3563a8b98 100644 --- a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt +++ b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt @@ -1,18 +1,12 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +#Leave these here for now - I don't need transitive deps anyway +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) -IF(NOT KOKKOS_HAS_TRILINOS) - IF(KOKKOS_SEPARATE_LIBS) - set(TEST_LINK_TARGETS kokkoscore) - ELSE() - set(TEST_LINK_TARGETS kokkos) - ENDIF() -ENDIF() SET(GTEST_SOURCE_DIR ${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/tpls/gtest) -INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) # mfh 03 Nov 2017: The gtest library used here must have a different # name than that of the gtest library built in KokkosCore. We can't @@ -20,25 +14,44 @@ INCLUDE_DIRECTORIES(${GTEST_SOURCE_DIR}) # possible to build only (e.g.,) KokkosAlgorithms tests, without # building KokkosCore tests. -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_PTHREAD=0") -TRIBITS_ADD_LIBRARY( +KOKKOS_ADD_TEST_LIBRARY( kokkosalgorithms_gtest HEADERS ${GTEST_SOURCE_DIR}/gtest/gtest.h SOURCES ${GTEST_SOURCE_DIR}/gtest/gtest-all.cc - TESTONLY - ) +) +# WORKAROUND FOR HIPCC +IF(Kokkos_ENABLE_HIP) + TARGET_COMPILE_DEFINITIONS(kokkosalgorithms_gtest PUBLIC "-DGTEST_HAS_PTHREAD=0 --amdgpu-target=gfx906") +ELSE() + TARGET_COMPILE_DEFINITIONS(kokkosalgorithms_gtest PUBLIC "-DGTEST_HAS_PTHREAD=0") +ENDIF() + +TARGET_COMPILE_FEATURES(kokkosalgorithms_gtest PUBLIC cxx_std_11) SET(SOURCES - UnitTestMain.cpp - TestCuda.cpp - ) + UnitTestMain.cpp +) -SET(LIBRARIES kokkoscore) - -IF(Kokkos_ENABLE_OpenMP) +IF(Kokkos_ENABLE_OPENMP) LIST( APPEND SOURCES TestOpenMP.cpp + TestOpenMP_Sort1D.cpp + TestOpenMP_Sort3D.cpp + TestOpenMP_SortDynamicView.cpp + TestOpenMP_Random.cpp + ) +ENDIF() + +IF(Kokkos_ENABLE_HIP) + LIST( APPEND SOURCES + TestHIP.cpp + ) +ENDIF() + +IF(Kokkos_ENABLE_CUDA) + LIST( APPEND SOURCES + TestCuda.cpp ) ENDIF() @@ -48,23 +61,19 @@ IF(Kokkos_ENABLE_HPX) ) ENDIF() -IF(Kokkos_ENABLE_Serial) +IF(Kokkos_ENABLE_SERIAL) LIST( APPEND SOURCES TestSerial.cpp ) ENDIF() -IF(Kokkos_ENABLE_Pthread) +IF(Kokkos_ENABLE_PTHREAD) LIST( APPEND SOURCES TestThreads.cpp ) ENDIF() -TRIBITS_ADD_EXECUTABLE_AND_TEST( +KOKKOS_ADD_EXECUTABLE_AND_TEST( UnitTest SOURCES ${SOURCES} - COMM serial mpi - NUM_MPI_PROCS 1 - FAIL_REGULAR_EXPRESSION " FAILED " - TESTONLYLIBS kokkosalgorithms_gtest ${TEST_LINK_TARGETS} - ) +) diff --git a/lib/kokkos/algorithms/unit_tests/Makefile b/lib/kokkos/algorithms/unit_tests/Makefile index 3c862d03dc..4a192b08ec 100644 --- a/lib/kokkos/algorithms/unit_tests/Makefile +++ b/lib/kokkos/algorithms/unit_tests/Makefile @@ -44,7 +44,7 @@ ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1) endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) - OBJ_OPENMP = TestOpenMP.o UnitTestMain.o gtest-all.o + OBJ_OPENMP = TestOpenMP.o TestOpenMP_Random.o TestOpenMP_Sort1D.o TestOpenMP_Sort3D.o TestOpenMP_SortDynamicView.o UnitTestMain.o gtest-all.o TARGETS += KokkosAlgorithms_UnitTest_OpenMP TEST_TARGETS += test-openmp endif diff --git a/lib/kokkos/algorithms/unit_tests/TestCuda.cpp b/lib/kokkos/algorithms/unit_tests/TestCuda.cpp index 86fdccd0e7..86cee61f64 100644 --- a/lib/kokkos/algorithms/unit_tests/TestCuda.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestCuda.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -57,51 +58,35 @@ namespace Test { -class cuda : public ::testing::Test { -protected: - static void SetUpTestCase() - { - } - static void TearDownTestCase() - { - } -}; - -void cuda_test_random_xorshift64( int num_draws ) -{ - Impl::test_random >(num_draws); +void cuda_test_random_xorshift64(int num_draws) { + Impl::test_random>(num_draws); + Impl::test_random>>(num_draws); } -void cuda_test_random_xorshift1024( int num_draws ) -{ - Impl::test_random >(num_draws); +void cuda_test_random_xorshift1024(int num_draws) { + Impl::test_random>(num_draws); + Impl::test_random>>(num_draws); } +#define CUDA_RANDOM_XORSHIFT64(num_draws) \ + TEST(cuda, Random_XorShift64) { cuda_test_random_xorshift64(num_draws); } -#define CUDA_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( cuda, Random_XorShift64 ) { \ - cuda_test_random_xorshift64(num_draws); \ - } +#define CUDA_RANDOM_XORSHIFT1024(num_draws) \ + TEST(cuda, Random_XorShift1024) { cuda_test_random_xorshift1024(num_draws); } -#define CUDA_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( cuda, Random_XorShift1024 ) { \ - cuda_test_random_xorshift1024(num_draws); \ - } +#define CUDA_SORT_UNSIGNED(size) \ + TEST(cuda, SortUnsigned) { Impl::test_sort(size); } -#define CUDA_SORT_UNSIGNED( size ) \ - TEST_F( cuda, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Cuda, unsigned >(size); \ - } - -CUDA_RANDOM_XORSHIFT64( 132141141 ) -CUDA_RANDOM_XORSHIFT1024( 52428813 ) +CUDA_RANDOM_XORSHIFT64(132141141) +CUDA_RANDOM_XORSHIFT1024(52428813) CUDA_SORT_UNSIGNED(171) #undef CUDA_RANDOM_XORSHIFT64 #undef CUDA_RANDOM_XORSHIFT1024 #undef CUDA_SORT_UNSIGNED -} +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTCUDA_PREVENT_LINK_ERROR() {} -#endif /* #ifdef KOKKOS_ENABLE_CUDA */ - +#endif /* #ifdef KOKKOS_ENABLE_CUDA */ diff --git a/lib/kokkos/example/sort_array/main.cpp b/lib/kokkos/algorithms/unit_tests/TestHIP.cpp similarity index 53% rename from lib/kokkos/example/sort_array/main.cpp rename to lib/kokkos/algorithms/unit_tests/TestHIP.cpp index 331b2ef62d..5e5ccb6a2e 100644 --- a/lib/kokkos/example/sort_array/main.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestHIP.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,55 +42,42 @@ //@HEADER */ -#include -#include +#include +#ifdef KOKKOS_ENABLE_HIP + +#include #include -#include +#include + +#include #include -#include +#include +#include +namespace Test { -int main( int argc , char ** argv ) -{ -#if defined( KOKKOS_ENABLE_CUDA ) || defined( KOKKOS_ENABLE_THREADS ) || defined( KOKKOS_ENABLE_OPENMP ) - Kokkos::initialize( argc , argv ); - - int length_array = 100000 ; - - for ( int i = 0 ; i < argc ; ++i ) { - if ( 0 == strcmp( argv[i] , "length_array" ) ) { - length_array = atoi( argv[i+1] ); - } - } - - int length_total_array = length_array * 100; - -#if defined( KOKKOS_ENABLE_CUDA ) - if ( Kokkos::Cuda::is_initialized() ) { - std::cout << "Kokkos::Cuda" << std::endl ; - Example::sort_array< Kokkos::Cuda >( length_array , length_total_array ); - } -#endif - -#if defined( KOKKOS_ENABLE_THREADS ) - if ( Kokkos::Threads::is_initialized() ) { - std::cout << "Kokkos::Threads" << std::endl ; - Example::sort_array< Kokkos::Threads >( length_array , length_total_array ); - } -#endif - -#if defined( KOKKOS_ENABLE_OPENMP ) - if ( Kokkos::OpenMP::is_initialized() ) { - std::cout << "Kokkos::OpenMP" << std::endl ; - Example::sort_array< Kokkos::OpenMP >( length_array , length_total_array ); - } -#endif - - Kokkos::finalize(); -#endif - - return 0 ; +void hip_test_random_xorshift64(size_t num_draws) { + Impl::test_random>( + num_draws); + Impl::test_random>>(num_draws); } +void hip_test_random_xorshift1024(size_t num_draws) { + Impl::test_random< + Kokkos::Random_XorShift1024_Pool>(num_draws); + Impl::test_random>>(num_draws); +} + +TEST(hip, Random_XorShift64) { hip_test_random_xorshift64(132141141); } +TEST(hip, Random_XorShift1024_0) { hip_test_random_xorshift1024(52428813); } +TEST(hip, SortUnsigned) { + Impl::test_sort(171); +} +} // namespace Test +#else +void KOKKOS_ALGORITHMS_UNITTESTS_TESTHIP_PREVENT_LINK_ERROR() {} +#endif /* #ifdef KOKKOS_ENABLE_HIP */ diff --git a/lib/kokkos/algorithms/unit_tests/TestHPX.cpp b/lib/kokkos/algorithms/unit_tests/TestHPX.cpp index e5b7dbdb7a..2981e97945 100644 --- a/lib/kokkos/algorithms/unit_tests/TestHPX.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestHPX.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,7 +42,6 @@ //@HEADER */ - #include #ifdef KOKKOS_ENABLE_HPX @@ -55,42 +55,33 @@ namespace Test { -class hpx : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; +#define HPX_RANDOM_XORSHIFT64(num_draws) \ + TEST(hpx, Random_XorShift64) { \ + Impl::test_random< \ + Kokkos::Random_XorShift64_Pool >( \ + num_draws); \ } - static void TearDownTestCase() - { - } -}; - -#define HPX_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( hpx, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define HPX_RANDOM_XORSHIFT1024(num_draws) \ + TEST(hpx, Random_XorShift1024) { \ + Impl::test_random< \ + Kokkos::Random_XorShift1024_Pool >( \ + num_draws); \ } -#define HPX_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( hpx, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define HPX_SORT_UNSIGNED(size) \ + TEST(hpx, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define HPX_SORT_UNSIGNED( size ) \ - TEST_F( hpx, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Experimental::HPX, unsigned >(size); \ - } - -HPX_RANDOM_XORSHIFT64( 10240000 ) -HPX_RANDOM_XORSHIFT1024( 10130144 ) +HPX_RANDOM_XORSHIFT64(10240000) +HPX_RANDOM_XORSHIFT1024(10130144) HPX_SORT_UNSIGNED(171) #undef HPX_RANDOM_XORSHIFT64 #undef HPX_RANDOM_XORSHIFT1024 #undef HPX_SORT_UNSIGNED -} // namespace test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTHPX_PREVENT_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp index c4ddde7b7f..5ded3ce390 100644 --- a/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,7 +42,6 @@ //@HEADER */ - #include #ifdef KOKKOS_ENABLE_OPENMP @@ -55,42 +55,9 @@ namespace Test { -class openmp : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; - } +TEST(openmp, SortIssue1160) { Impl::test_issue_1160_sort(); } - static void TearDownTestCase() - { - } -}; - -#define OPENMP_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( openmp, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ - } - -#define OPENMP_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( openmp, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ - } - -#define OPENMP_SORT_UNSIGNED( size ) \ - TEST_F( openmp, SortUnsigned ) { \ - Impl::test_sort< Kokkos::OpenMP, unsigned >(size); \ - } - -OPENMP_RANDOM_XORSHIFT64( 10240000 ) -OPENMP_RANDOM_XORSHIFT1024( 10130144 ) -OPENMP_SORT_UNSIGNED(171) - -#undef OPENMP_RANDOM_XORSHIFT64 -#undef OPENMP_RANDOM_XORSHIFT1024 -#undef OPENMP_SORT_UNSIGNED -} // namespace test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP_Random.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Random.cpp new file mode 100644 index 0000000000..1ca8e0a828 --- /dev/null +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Random.cpp @@ -0,0 +1,77 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#ifdef KOKKOS_ENABLE_OPENMP + +#include +#include + +//---------------------------------------------------------------------------- +#include +#include + +namespace Test { + +#define OPENMP_RANDOM_XORSHIFT64(num_draws) \ + TEST(openmp, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ + } + +#define OPENMP_RANDOM_XORSHIFT1024(num_draws) \ + TEST(openmp, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ + } + +OPENMP_RANDOM_XORSHIFT64(10240000) +OPENMP_RANDOM_XORSHIFT1024(10130144) + +#undef OPENMP_RANDOM_XORSHIFT64 +#undef OPENMP_RANDOM_XORSHIFT1024 +} // namespace Test +#else +void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} +#endif diff --git a/lib/kokkos/example/feint/feint_openmp.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp similarity index 70% rename from lib/kokkos/example/feint/feint_openmp.cpp rename to lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp index 8c7e0b6a08..a9b2010ad0 100644 --- a/lib/kokkos/example/feint/feint_openmp.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp @@ -1,13 +1,14 @@ /* //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -36,32 +37,29 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER */ -#include - +#include #ifdef KOKKOS_ENABLE_OPENMP -#include +#include +#include -namespace Kokkos { -namespace Example { +//---------------------------------------------------------------------------- +#include +#include +#include -template void feint( - const unsigned global_elem_nx , - const unsigned global_elem_ny , - const unsigned global_elem_nz ); +namespace Test { -template void feint( - const unsigned global_elem_nx , - const unsigned global_elem_ny , - const unsigned global_elem_nz ); - -} /* namespace Example */ -} /* namespace Kokkos */ +TEST(openmp, SortUnsigned1D) { + Impl::test_1D_sort(171); +} +} // namespace Test +#else +void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort3D.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort3D.cpp new file mode 100644 index 0000000000..127d911d7c --- /dev/null +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort3D.cpp @@ -0,0 +1,65 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#ifdef KOKKOS_ENABLE_OPENMP + +#include +#include + +//---------------------------------------------------------------------------- +#include +#include +#include + +namespace Test { + +TEST(openmp, SortUnsigned3D) { + Impl::test_3D_sort(171); +} + +} // namespace Test +#else +void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} +#endif diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP_SortDynamicView.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP_SortDynamicView.cpp new file mode 100644 index 0000000000..3dc8854044 --- /dev/null +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP_SortDynamicView.cpp @@ -0,0 +1,65 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#include +#ifdef KOKKOS_ENABLE_OPENMP + +#include +#include + +//---------------------------------------------------------------------------- +#include +#include +#include + +namespace Test { + +TEST(openmp, SortUnsignedDynamicView) { + Impl::test_dynamic_view_sort(171); +} + +} // namespace Test +#else +void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} +#endif diff --git a/lib/kokkos/algorithms/unit_tests/TestROCm.cpp b/lib/kokkos/algorithms/unit_tests/TestROCm.cpp index 15179509bb..29814cca3e 100644 --- a/lib/kokkos/algorithms/unit_tests/TestROCm.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestROCm.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -57,52 +58,35 @@ namespace Test { -class rocm : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; - } - static void TearDownTestCase() - { - } -}; - -void rocm_test_random_xorshift64( int num_draws ) -{ - Impl::test_random >(num_draws); +void rocm_test_random_xorshift64(int num_draws) { + Impl::test_random< + Kokkos::Random_XorShift64_Pool >(num_draws); } -void rocm_test_random_xorshift1024( int num_draws ) -{ - Impl::test_random >(num_draws); +void rocm_test_random_xorshift1024(int num_draws) { + Impl::test_random< + Kokkos::Random_XorShift1024_Pool >(num_draws); } +#define ROCM_RANDOM_XORSHIFT64(num_draws) \ + TEST(rocm, Random_XorShift64) { rocm_test_random_xorshift64(num_draws); } -#define ROCM_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( rocm, Random_XorShift64 ) { \ - rocm_test_random_xorshift64(num_draws); \ +#define ROCM_RANDOM_XORSHIFT1024(num_draws) \ + TEST(rocm, Random_XorShift1024) { rocm_test_random_xorshift1024(num_draws); } + +#define ROCM_SORT_UNSIGNED(size) \ + TEST(rocm, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define ROCM_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( rocm, Random_XorShift1024 ) { \ - rocm_test_random_xorshift1024(num_draws); \ - } - -#define ROCM_SORT_UNSIGNED( size ) \ - TEST_F( rocm, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Experimental::ROCm, unsigned >(size); \ - } - -ROCM_RANDOM_XORSHIFT64( 132141141 ) -ROCM_RANDOM_XORSHIFT1024( 52428813 ) +ROCM_RANDOM_XORSHIFT64(132141141) +ROCM_RANDOM_XORSHIFT1024(52428813) ROCM_SORT_UNSIGNED(171) #undef ROCM_RANDOM_XORSHIFT64 #undef ROCM_RANDOM_XORSHIFT1024 #undef ROCM_SORT_UNSIGNED -} +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTROCM_PREVENT_LINK_ERROR() {} -#endif /* #ifdef KOKKOS_ENABLE_ROCM */ - +#endif /* #ifdef KOKKOS_ENABLE_ROCM */ diff --git a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp index 73bd416f2a..10a496242b 100644 --- a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -54,18 +55,19 @@ namespace Test { -namespace Impl{ +namespace Impl { // This test runs the random number generators and uses some statistic tests to // check the 'goodness' of the random numbers: // (i) mean: the mean is expected to be 0.5*RAND_MAX // (ii) variance: the variance is 1/3*mean*mean // (iii) covariance: the covariance is 0 -// (iv) 1-tupledistr: the mean, variance and covariance of a 1D Histrogram of random numbers -// (v) 3-tupledistr: the mean, variance and covariance of a 3D Histrogram of random numbers +// (iv) 1-tupledistr: the mean, variance and covariance of a 1D Histrogram +// of random numbers (v) 3-tupledistr: the mean, variance and covariance of +// a 3D Histrogram of random numbers #define HIST_DIM3D 24 -#define HIST_DIM1D (HIST_DIM3D*HIST_DIM3D*HIST_DIM3D) +#define HIST_DIM1D (HIST_DIM3D * HIST_DIM3D * HIST_DIM3D) struct RandomProperties { uint64_t count; @@ -77,37 +79,37 @@ struct RandomProperties { KOKKOS_INLINE_FUNCTION RandomProperties() { - count = 0; - mean = 0.0; - variance = 0.0; + count = 0; + mean = 0.0; + variance = 0.0; covariance = 0.0; - min = 1e64; - max = -1e64; + min = 1e64; + max = -1e64; } KOKKOS_INLINE_FUNCTION RandomProperties& operator+=(const RandomProperties& add) { - count += add.count; - mean += add.mean; - variance += add.variance; + count += add.count; + mean += add.mean; + variance += add.variance; covariance += add.covariance; - min = add.minmax?add.max:max; + min = add.min < min ? add.min : min; + max = add.max > max ? add.max : max; return *this; } KOKKOS_INLINE_FUNCTION void operator+=(const volatile RandomProperties& add) volatile { - count += add.count; - mean += add.mean; - variance += add.variance; + count += add.count; + mean += add.mean; + variance += add.variance; covariance += add.covariance; - min = add.minmax?add.max:max; + min = add.min < min ? add.min : min; + max = add.max > max ? add.max : max; } }; -template +template struct test_random_functor { typedef typename GeneratorPool::generator_type rnd_type; @@ -123,38 +125,40 @@ struct test_random_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View types below. - typedef Kokkos::View type_1d; + typedef Kokkos::View + type_1d; type_1d density_1d; - typedef Kokkos::View type_3d; + typedef Kokkos::View + type_3d; type_3d density_3d; - test_random_functor (GeneratorPool rand_pool_, type_1d d1d, type_3d d3d) : - rand_pool (rand_pool_), - mean (0.5*Kokkos::rand::max ()), - density_1d (d1d), - density_3d (d3d) - {} + test_random_functor(GeneratorPool rand_pool_, type_1d d1d, type_3d d3d) + : rand_pool(rand_pool_), + mean(0.5 * Kokkos::rand::max()), + density_1d(d1d), + density_3d(d3d) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, RandomProperties& prop) const { + void operator()(int /*i*/, RandomProperties& prop) const { using Kokkos::atomic_fetch_add; rnd_type rand_gen = rand_pool.get_state(); for (int k = 0; k < 1024; ++k) { - const Scalar tmp = Kokkos::rand::draw(rand_gen); + const Scalar tmp = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp; - prop.variance += (tmp-mean)*(tmp-mean); - const Scalar tmp2 = Kokkos::rand::draw(rand_gen); + prop.variance += (tmp - mean) * (tmp - mean); + const Scalar tmp2 = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp2; - prop.variance += (tmp2-mean)*(tmp2-mean); - prop.covariance += (tmp-mean)*(tmp2-mean); - const Scalar tmp3 = Kokkos::rand::draw(rand_gen); + prop.variance += (tmp2 - mean) * (tmp2 - mean); + prop.covariance += (tmp - mean) * (tmp2 - mean); + const Scalar tmp3 = Kokkos::rand::draw(rand_gen); prop.count++; prop.mean += tmp3; - prop.variance += (tmp3-mean)*(tmp3-mean); - prop.covariance += (tmp2-mean)*(tmp3-mean); + prop.variance += (tmp3 - mean) * (tmp3 - mean); + prop.covariance += (tmp2 - mean) * (tmp3 - mean); // NOTE (mfh 03 Nov 2014): Kokkos::rand::max() is supposed to // define an exclusive upper bound on the range of random @@ -169,26 +173,32 @@ struct test_random_functor { // returns values of max(), the histograms will still catch this // indirectly, since none of the other values will be filled in. - const Scalar theMax = Kokkos::rand::max (); + const Scalar theMax = Kokkos::rand::max(); - const uint64_t ind1_1d = static_cast (1.0 * HIST_DIM1D * tmp / theMax); - const uint64_t ind2_1d = static_cast (1.0 * HIST_DIM1D * tmp2 / theMax); - const uint64_t ind3_1d = static_cast (1.0 * HIST_DIM1D * tmp3 / theMax); + const uint64_t ind1_1d = + static_cast(1.0 * HIST_DIM1D * tmp / theMax); + const uint64_t ind2_1d = + static_cast(1.0 * HIST_DIM1D * tmp2 / theMax); + const uint64_t ind3_1d = + static_cast(1.0 * HIST_DIM1D * tmp3 / theMax); - const uint64_t ind1_3d = static_cast (1.0 * HIST_DIM3D * tmp / theMax); - const uint64_t ind2_3d = static_cast (1.0 * HIST_DIM3D * tmp2 / theMax); - const uint64_t ind3_3d = static_cast (1.0 * HIST_DIM3D * tmp3 / theMax); + const uint64_t ind1_3d = + static_cast(1.0 * HIST_DIM3D * tmp / theMax); + const uint64_t ind2_3d = + static_cast(1.0 * HIST_DIM3D * tmp2 / theMax); + const uint64_t ind3_3d = + static_cast(1.0 * HIST_DIM3D * tmp3 / theMax); - atomic_fetch_add (&density_1d(ind1_1d), 1); - atomic_fetch_add (&density_1d(ind2_1d), 1); - atomic_fetch_add (&density_1d(ind3_1d), 1); - atomic_fetch_add (&density_3d(ind1_3d, ind2_3d, ind3_3d), 1); + atomic_fetch_add(&density_1d(ind1_1d), 1); + atomic_fetch_add(&density_1d(ind2_1d), 1); + atomic_fetch_add(&density_1d(ind3_1d), 1); + atomic_fetch_add(&density_3d(ind1_3d, ind2_3d, ind3_3d), 1); } rand_pool.free_state(rand_gen); } }; -template +template struct test_histogram1d_functor { typedef RandomProperties value_type; typedef typename DeviceType::execution_space execution_space; @@ -200,34 +210,29 @@ struct test_histogram1d_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View type below. - typedef Kokkos::View type_1d; + typedef Kokkos::View type_1d; type_1d density_1d; double mean; - test_histogram1d_functor (type_1d d1d, int num_draws) : - density_1d (d1d), - mean (1.0*num_draws/HIST_DIM1D*3) - { - } + test_histogram1d_functor(type_1d d1d, int num_draws) + : density_1d(d1d), mean(1.0 * num_draws / HIST_DIM1D * 3) {} - KOKKOS_INLINE_FUNCTION void - operator() (const typename memory_space::size_type i, - RandomProperties& prop) const - { + KOKKOS_INLINE_FUNCTION void operator()( + const typename memory_space::size_type i, RandomProperties& prop) const { typedef typename memory_space::size_type size_type; const double count = density_1d(i); prop.mean += count; prop.variance += 1.0 * (count - mean) * (count - mean); - //prop.covariance += 1.0*count*count; + // prop.covariance += 1.0*count*count; prop.min = count < prop.min ? count : prop.min; prop.max = count > prop.max ? count : prop.max; - if (i < static_cast (HIST_DIM1D-1)) { - prop.covariance += (count - mean) * (density_1d(i+1) - mean); + if (i < static_cast(HIST_DIM1D - 1)) { + prop.covariance += (count - mean) * (density_1d(i + 1) - mean); } } }; -template +template struct test_histogram3d_functor { typedef RandomProperties value_type; typedef typename DeviceType::execution_space execution_space; @@ -239,29 +244,28 @@ struct test_histogram3d_functor { // implementations might violate this upper bound, due to rounding // error. Just in case, we leave an extra space at the end of each // dimension, in the View type below. - typedef Kokkos::View type_3d; + typedef Kokkos::View + type_3d; type_3d density_3d; double mean; - test_histogram3d_functor (type_3d d3d, int num_draws) : - density_3d (d3d), - mean (1.0*num_draws/HIST_DIM1D) - {} + test_histogram3d_functor(type_3d d3d, int num_draws) + : density_3d(d3d), mean(1.0 * num_draws / HIST_DIM1D) {} - KOKKOS_INLINE_FUNCTION void - operator() (const typename memory_space::size_type i, - RandomProperties& prop) const - { + KOKKOS_INLINE_FUNCTION void operator()( + const typename memory_space::size_type i, RandomProperties& prop) const { typedef typename memory_space::size_type size_type; - const double count = density_3d(i/(HIST_DIM3D*HIST_DIM3D), - (i % (HIST_DIM3D*HIST_DIM3D))/HIST_DIM3D, - i % HIST_DIM3D); + const double count = density_3d( + i / (HIST_DIM3D * HIST_DIM3D), + (i % (HIST_DIM3D * HIST_DIM3D)) / HIST_DIM3D, i % HIST_DIM3D); prop.mean += count; prop.variance += (count - mean) * (count - mean); - if (i < static_cast (HIST_DIM1D-1)) { - const double count_next = density_3d((i+1)/(HIST_DIM3D*HIST_DIM3D), - ((i+1)%(HIST_DIM3D*HIST_DIM3D))/HIST_DIM3D, - (i+1)%HIST_DIM3D); + if (i < static_cast(HIST_DIM1D - 1)) { + const double count_next = + density_3d((i + 1) / (HIST_DIM3D * HIST_DIM3D), + ((i + 1) % (HIST_DIM3D * HIST_DIM3D)) / HIST_DIM3D, + (i + 1) % HIST_DIM3D); prop.covariance += (count - mean) * (count_next - mean); } } @@ -270,212 +274,223 @@ struct test_histogram3d_functor { // // Templated test that uses the above functors. // -template +template struct test_random_scalar { typedef typename RandomGenerator::generator_type rnd_type; - int pass_mean,pass_var,pass_covar; - int pass_hist1d_mean,pass_hist1d_var,pass_hist1d_covar; - int pass_hist3d_mean,pass_hist3d_var,pass_hist3d_covar; + int pass_mean, pass_var, pass_covar; + int pass_hist1d_mean, pass_hist1d_var, pass_hist1d_covar; + int pass_hist3d_mean, pass_hist3d_var, pass_hist3d_covar; - test_random_scalar (typename test_random_functor::type_1d& density_1d, - typename test_random_functor::type_3d& density_3d, - RandomGenerator& pool, - unsigned int num_draws) - { + test_random_scalar( + typename test_random_functor::type_1d& density_1d, + typename test_random_functor::type_3d& density_3d, + RandomGenerator& pool, unsigned int num_draws) { + using Kokkos::parallel_reduce; using std::cout; using std::endl; - using Kokkos::parallel_reduce; { cout << " -- Testing randomness properties" << endl; RandomProperties result; typedef test_random_functor functor_type; - parallel_reduce (num_draws/1024, functor_type (pool, density_1d, density_3d), result); + parallel_reduce(num_draws / 1024, + functor_type(pool, density_1d, density_3d), result); - //printf("Result: %lf %lf %lf\n",result.mean/num_draws/3,result.variance/num_draws/3,result.covariance/num_draws/2); - double tolerance = 1.6*std::sqrt(1.0/num_draws); - double mean_expect = 0.5*Kokkos::rand::max(); - double variance_expect = 1.0/3.0*mean_expect*mean_expect; - double mean_eps = mean_expect/(result.mean/num_draws/3)-1.0; - double variance_eps = variance_expect/(result.variance/num_draws/3)-1.0; - double covariance_eps = result.covariance/num_draws/2/variance_expect; - pass_mean = ((-tolerance < mean_eps) && - ( tolerance > mean_eps)) ? 1:0; - pass_var = ((-1.5*tolerance < variance_eps) && - ( 1.5*tolerance > variance_eps)) ? 1:0; - pass_covar = ((-2.0*tolerance < covariance_eps) && - ( 2.0*tolerance > covariance_eps)) ? 1:0; - cout << "Pass: " << pass_mean - << " " << pass_var - << " " << mean_eps - << " " << variance_eps - << " " << covariance_eps - << " || " << tolerance << endl; + // printf("Result: %lf %lf + // %lf\n",result.mean/num_draws/3,result.variance/num_draws/3,result.covariance/num_draws/2); + double tolerance = 1.6 * std::sqrt(1.0 / num_draws); + double mean_expect = 0.5 * Kokkos::rand::max(); + double variance_expect = 1.0 / 3.0 * mean_expect * mean_expect; + double mean_eps = mean_expect / (result.mean / num_draws / 3) - 1.0; + double variance_eps = + variance_expect / (result.variance / num_draws / 3) - 1.0; + double covariance_eps = + result.covariance / num_draws / 2 / variance_expect; + pass_mean = ((-tolerance < mean_eps) && (tolerance > mean_eps)) ? 1 : 0; + pass_var = ((-1.5 * tolerance < variance_eps) && + (1.5 * tolerance > variance_eps)) + ? 1 + : 0; + pass_covar = ((-2.0 * tolerance < covariance_eps) && + (2.0 * tolerance > covariance_eps)) + ? 1 + : 0; + cout << "Pass: " << pass_mean << " " << pass_var << " " << mean_eps << " " + << variance_eps << " " << covariance_eps << " || " << tolerance + << endl; } { cout << " -- Testing 1-D histogram" << endl; RandomProperties result; - typedef test_histogram1d_functor functor_type; - parallel_reduce (HIST_DIM1D, functor_type (density_1d, num_draws), result); + typedef test_histogram1d_functor + functor_type; + parallel_reduce(HIST_DIM1D, functor_type(density_1d, num_draws), result); - double tolerance = 6*std::sqrt(1.0/HIST_DIM1D); - double mean_expect = 1.0*num_draws*3/HIST_DIM1D; - double variance_expect = 1.0*num_draws*3/HIST_DIM1D*(1.0-1.0/HIST_DIM1D); - double covariance_expect = -1.0*num_draws*3/HIST_DIM1D/HIST_DIM1D; - double mean_eps = mean_expect/(result.mean/HIST_DIM1D)-1.0; - double variance_eps = variance_expect/(result.variance/HIST_DIM1D)-1.0; - double covariance_eps = (result.covariance/HIST_DIM1D - covariance_expect)/mean_expect; - pass_hist1d_mean = ((-0.0001 < mean_eps) && - ( 0.0001 > mean_eps)) ? 1:0; - pass_hist1d_var = ((-0.07 < variance_eps) && - ( 0.07 > variance_eps)) ? 1:0; - pass_hist1d_covar = ((-0.06 < covariance_eps) && - ( 0.06 > covariance_eps)) ? 1:0; + double tolerance = 6 * std::sqrt(1.0 / HIST_DIM1D); + double mean_expect = 1.0 * num_draws * 3 / HIST_DIM1D; + double variance_expect = + 1.0 * num_draws * 3 / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D); + double covariance_expect = -1.0 * num_draws * 3 / HIST_DIM1D / HIST_DIM1D; + double mean_eps = mean_expect / (result.mean / HIST_DIM1D) - 1.0; + double variance_eps = + variance_expect / (result.variance / HIST_DIM1D) - 1.0; + double covariance_eps = + (result.covariance / HIST_DIM1D - covariance_expect) / mean_expect; + pass_hist1d_mean = ((-0.0001 < mean_eps) && (0.0001 > mean_eps)) ? 1 : 0; + pass_hist1d_var = + ((-0.07 < variance_eps) && (0.07 > variance_eps)) ? 1 : 0; + pass_hist1d_covar = + ((-0.06 < covariance_eps) && (0.06 > covariance_eps)) ? 1 : 0; - cout << "Density 1D: " << mean_eps - << " " << variance_eps - << " " << (result.covariance/HIST_DIM1D/HIST_DIM1D) - << " || " << tolerance - << " " << result.min - << " " << result.max - << " || " << result.variance/HIST_DIM1D - << " " << 1.0*num_draws*3/HIST_DIM1D*(1.0-1.0/HIST_DIM1D) - << " || " << result.covariance/HIST_DIM1D - << " " << -1.0*num_draws*3/HIST_DIM1D/HIST_DIM1D - << endl; + cout << "Density 1D: " << mean_eps << " " << variance_eps << " " + << (result.covariance / HIST_DIM1D / HIST_DIM1D) << " || " + << tolerance << " " << result.min << " " << result.max << " || " + << result.variance / HIST_DIM1D << " " + << 1.0 * num_draws * 3 / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D) + << " || " << result.covariance / HIST_DIM1D << " " + << -1.0 * num_draws * 3 / HIST_DIM1D / HIST_DIM1D << endl; } { cout << " -- Testing 3-D histogram" << endl; RandomProperties result; - typedef test_histogram3d_functor functor_type; - parallel_reduce (HIST_DIM1D, functor_type (density_3d, num_draws), result); + typedef test_histogram3d_functor + functor_type; + parallel_reduce(HIST_DIM1D, functor_type(density_3d, num_draws), result); - double tolerance = 6*std::sqrt(1.0/HIST_DIM1D); - double mean_expect = 1.0*num_draws/HIST_DIM1D; - double variance_expect = 1.0*num_draws/HIST_DIM1D*(1.0-1.0/HIST_DIM1D); - double covariance_expect = -1.0*num_draws/HIST_DIM1D/HIST_DIM1D; - double mean_eps = mean_expect/(result.mean/HIST_DIM1D)-1.0; - double variance_eps = variance_expect/(result.variance/HIST_DIM1D)-1.0; - double covariance_eps = (result.covariance/HIST_DIM1D - covariance_expect)/mean_expect; - pass_hist3d_mean = ((-tolerance < mean_eps) && - ( tolerance > mean_eps)) ? 1:0; - pass_hist3d_var = ((-1.2*tolerance < variance_eps) && - ( 1.2*tolerance > variance_eps)) ? 1:0; - pass_hist3d_covar = ((-tolerance < covariance_eps) && - ( tolerance > covariance_eps)) ? 1:0; + double tolerance = 6 * std::sqrt(1.0 / HIST_DIM1D); + double mean_expect = 1.0 * num_draws / HIST_DIM1D; + double variance_expect = + 1.0 * num_draws / HIST_DIM1D * (1.0 - 1.0 / HIST_DIM1D); + double covariance_expect = -1.0 * num_draws / HIST_DIM1D / HIST_DIM1D; + double mean_eps = mean_expect / (result.mean / HIST_DIM1D) - 1.0; + double variance_eps = + variance_expect / (result.variance / HIST_DIM1D) - 1.0; + double covariance_eps = + (result.covariance / HIST_DIM1D - covariance_expect) / mean_expect; + pass_hist3d_mean = + ((-tolerance < mean_eps) && (tolerance > mean_eps)) ? 1 : 0; + pass_hist3d_var = ((-1.2 * tolerance < variance_eps) && + (1.2 * tolerance > variance_eps)) + ? 1 + : 0; + pass_hist3d_covar = + ((-tolerance < covariance_eps) && (tolerance > covariance_eps)) ? 1 + : 0; - cout << "Density 3D: " << mean_eps - << " " << variance_eps - << " " << result.covariance/HIST_DIM1D/HIST_DIM1D - << " || " << tolerance - << " " << result.min - << " " << result.max << endl; + cout << "Density 3D: " << mean_eps << " " << variance_eps << " " + << result.covariance / HIST_DIM1D / HIST_DIM1D << " || " << tolerance + << " " << result.min << " " << result.max << endl; } } }; template -void test_random(unsigned int num_draws) -{ +void test_random(unsigned int num_draws) { using std::cout; using std::endl; - typename test_random_functor::type_1d density_1d("D1d"); - typename test_random_functor::type_3d density_3d("D3d"); + typename test_random_functor::type_1d density_1d("D1d"); + typename test_random_functor::type_3d density_3d("D3d"); - - uint64_t ticks = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + uint64_t ticks = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); cout << "Test Seed:" << ticks << endl; RandomGenerator pool(ticks); cout << "Test Scalar=int" << endl; - test_random_scalar test_int(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_int.pass_mean,1); - ASSERT_EQ( test_int.pass_var,1); - ASSERT_EQ( test_int.pass_covar,1); - ASSERT_EQ( test_int.pass_hist1d_mean,1); - ASSERT_EQ( test_int.pass_hist1d_var,1); - ASSERT_EQ( test_int.pass_hist1d_covar,1); - ASSERT_EQ( test_int.pass_hist3d_mean,1); - ASSERT_EQ( test_int.pass_hist3d_var,1); - ASSERT_EQ( test_int.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_int(density_1d, density_3d, + pool, num_draws); + ASSERT_EQ(test_int.pass_mean, 1); + ASSERT_EQ(test_int.pass_var, 1); + ASSERT_EQ(test_int.pass_covar, 1); + ASSERT_EQ(test_int.pass_hist1d_mean, 1); + ASSERT_EQ(test_int.pass_hist1d_var, 1); + ASSERT_EQ(test_int.pass_hist1d_covar, 1); + ASSERT_EQ(test_int.pass_hist3d_mean, 1); + ASSERT_EQ(test_int.pass_hist3d_var, 1); + ASSERT_EQ(test_int.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=unsigned int" << endl; - test_random_scalar test_uint(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_uint.pass_mean,1); - ASSERT_EQ( test_uint.pass_var,1); - ASSERT_EQ( test_uint.pass_covar,1); - ASSERT_EQ( test_uint.pass_hist1d_mean,1); - ASSERT_EQ( test_uint.pass_hist1d_var,1); - ASSERT_EQ( test_uint.pass_hist1d_covar,1); - ASSERT_EQ( test_uint.pass_hist3d_mean,1); - ASSERT_EQ( test_uint.pass_hist3d_var,1); - ASSERT_EQ( test_uint.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_uint( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_uint.pass_mean, 1); + ASSERT_EQ(test_uint.pass_var, 1); + ASSERT_EQ(test_uint.pass_covar, 1); + ASSERT_EQ(test_uint.pass_hist1d_mean, 1); + ASSERT_EQ(test_uint.pass_hist1d_var, 1); + ASSERT_EQ(test_uint.pass_hist1d_covar, 1); + ASSERT_EQ(test_uint.pass_hist3d_mean, 1); + ASSERT_EQ(test_uint.pass_hist3d_var, 1); + ASSERT_EQ(test_uint.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=int64_t" << endl; - test_random_scalar test_int64(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_int64.pass_mean,1); - ASSERT_EQ( test_int64.pass_var,1); - ASSERT_EQ( test_int64.pass_covar,1); - ASSERT_EQ( test_int64.pass_hist1d_mean,1); - ASSERT_EQ( test_int64.pass_hist1d_var,1); - ASSERT_EQ( test_int64.pass_hist1d_covar,1); - ASSERT_EQ( test_int64.pass_hist3d_mean,1); - ASSERT_EQ( test_int64.pass_hist3d_var,1); - ASSERT_EQ( test_int64.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_int64( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_int64.pass_mean, 1); + ASSERT_EQ(test_int64.pass_var, 1); + ASSERT_EQ(test_int64.pass_covar, 1); + ASSERT_EQ(test_int64.pass_hist1d_mean, 1); + ASSERT_EQ(test_int64.pass_hist1d_var, 1); + ASSERT_EQ(test_int64.pass_hist1d_covar, 1); + ASSERT_EQ(test_int64.pass_hist3d_mean, 1); + ASSERT_EQ(test_int64.pass_hist3d_var, 1); + ASSERT_EQ(test_int64.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=uint64_t" << endl; - test_random_scalar test_uint64(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_uint64.pass_mean,1); - ASSERT_EQ( test_uint64.pass_var,1); - ASSERT_EQ( test_uint64.pass_covar,1); - ASSERT_EQ( test_uint64.pass_hist1d_mean,1); - ASSERT_EQ( test_uint64.pass_hist1d_var,1); - ASSERT_EQ( test_uint64.pass_hist1d_covar,1); - ASSERT_EQ( test_uint64.pass_hist3d_mean,1); - ASSERT_EQ( test_uint64.pass_hist3d_var,1); - ASSERT_EQ( test_uint64.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_uint64( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_uint64.pass_mean, 1); + ASSERT_EQ(test_uint64.pass_var, 1); + ASSERT_EQ(test_uint64.pass_covar, 1); + ASSERT_EQ(test_uint64.pass_hist1d_mean, 1); + ASSERT_EQ(test_uint64.pass_hist1d_var, 1); + ASSERT_EQ(test_uint64.pass_hist1d_covar, 1); + ASSERT_EQ(test_uint64.pass_hist3d_mean, 1); + ASSERT_EQ(test_uint64.pass_hist3d_var, 1); + ASSERT_EQ(test_uint64.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=float" << endl; - test_random_scalar test_float(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_float.pass_mean,1); - ASSERT_EQ( test_float.pass_var,1); - ASSERT_EQ( test_float.pass_covar,1); - ASSERT_EQ( test_float.pass_hist1d_mean,1); - ASSERT_EQ( test_float.pass_hist1d_var,1); - ASSERT_EQ( test_float.pass_hist1d_covar,1); - ASSERT_EQ( test_float.pass_hist3d_mean,1); - ASSERT_EQ( test_float.pass_hist3d_var,1); - ASSERT_EQ( test_float.pass_hist3d_covar,1); - deep_copy(density_1d,0); - deep_copy(density_3d,0); + test_random_scalar test_float(density_1d, density_3d, + pool, num_draws); + ASSERT_EQ(test_float.pass_mean, 1); + ASSERT_EQ(test_float.pass_var, 1); + ASSERT_EQ(test_float.pass_covar, 1); + ASSERT_EQ(test_float.pass_hist1d_mean, 1); + ASSERT_EQ(test_float.pass_hist1d_var, 1); + ASSERT_EQ(test_float.pass_hist1d_covar, 1); + ASSERT_EQ(test_float.pass_hist3d_mean, 1); + ASSERT_EQ(test_float.pass_hist3d_var, 1); + ASSERT_EQ(test_float.pass_hist3d_covar, 1); + deep_copy(density_1d, 0); + deep_copy(density_3d, 0); cout << "Test Scalar=double" << endl; - test_random_scalar test_double(density_1d,density_3d,pool,num_draws); - ASSERT_EQ( test_double.pass_mean,1); - ASSERT_EQ( test_double.pass_var,1); - ASSERT_EQ( test_double.pass_covar,1); - ASSERT_EQ( test_double.pass_hist1d_mean,1); - ASSERT_EQ( test_double.pass_hist1d_var,1); - ASSERT_EQ( test_double.pass_hist1d_covar,1); - ASSERT_EQ( test_double.pass_hist3d_mean,1); - ASSERT_EQ( test_double.pass_hist3d_var,1); - ASSERT_EQ( test_double.pass_hist3d_covar,1); -} + test_random_scalar test_double( + density_1d, density_3d, pool, num_draws); + ASSERT_EQ(test_double.pass_mean, 1); + ASSERT_EQ(test_double.pass_var, 1); + ASSERT_EQ(test_double.pass_covar, 1); + ASSERT_EQ(test_double.pass_hist1d_mean, 1); + ASSERT_EQ(test_double.pass_hist1d_var, 1); + ASSERT_EQ(test_double.pass_hist1d_covar, 1); + ASSERT_EQ(test_double.pass_hist3d_mean, 1); + ASSERT_EQ(test_double.pass_hist3d_var, 1); + ASSERT_EQ(test_double.pass_hist3d_covar, 1); } +} // namespace Impl -} // namespace Test +} // namespace Test -#endif //KOKKOS_TEST_UNORDERED_MAP_HPP +#endif // KOKKOS_TEST_UNORDERED_MAP_HPP diff --git a/lib/kokkos/algorithms/unit_tests/TestSerial.cpp b/lib/kokkos/algorithms/unit_tests/TestSerial.cpp index 9cf998f773..2eacdc2677 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSerial.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestSerial.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -52,49 +53,36 @@ #include #include - //---------------------------------------------------------------------------- - namespace Test { -class serial : public ::testing::Test { -protected: - static void SetUpTestCase() - { +#define SERIAL_RANDOM_XORSHIFT64(num_draws) \ + TEST(serial, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ } - static void TearDownTestCase () - { - } -}; - -#define SERIAL_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( serial, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define SERIAL_RANDOM_XORSHIFT1024(num_draws) \ + TEST(serial, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ } -#define SERIAL_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( serial, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define SERIAL_SORT_UNSIGNED(size) \ + TEST(serial, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define SERIAL_SORT_UNSIGNED( size ) \ - TEST_F( serial, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Serial, unsigned >(size); \ - } - -SERIAL_RANDOM_XORSHIFT64( 10240000 ) -SERIAL_RANDOM_XORSHIFT1024( 10130144 ) +SERIAL_RANDOM_XORSHIFT64(10240000) +SERIAL_RANDOM_XORSHIFT1024(10130144) SERIAL_SORT_UNSIGNED(171) #undef SERIAL_RANDOM_XORSHIFT64 #undef SERIAL_RANDOM_XORSHIFT1024 #undef SERIAL_SORT_UNSIGNED -} // namespace Test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTSERIAL_PREVENT_LINK_ERROR() {} -#endif // KOKKOS_ENABLE_SERIAL - - +#endif // KOKKOS_ENABLE_SERIAL diff --git a/lib/kokkos/algorithms/unit_tests/TestSort.hpp b/lib/kokkos/algorithms/unit_tests/TestSort.hpp index 5fd7f09b50..b6ff91c25f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSort.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestSort.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -43,235 +44,248 @@ #define KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_HPP #include -#include -#include -#include -#include +#include +#include +#include +#include namespace Test { -namespace Impl{ +namespace Impl { -template +template struct is_sorted_struct { typedef unsigned int value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - is_sorted_struct(Kokkos::View keys_):keys(keys_) {} + is_sorted_struct(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, unsigned int& count) const { - if(keys(i)>keys(i+1)) count++; + void operator()(int i, unsigned int& count) const { + if (keys(i) > keys(i + 1)) count++; } }; -template +template struct sum { typedef double value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - sum(Kokkos::View keys_):keys(keys_) {} + sum(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, double& count) const { - count+=keys(i); - } + void operator()(int i, double& count) const { count += keys(i); } }; -template +template struct bin3d_is_sorted_struct { typedef unsigned int value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; int max_bins; Scalar min; Scalar max; - bin3d_is_sorted_struct(Kokkos::View keys_,int max_bins_,Scalar min_,Scalar max_): - keys(keys_),max_bins(max_bins_),min(min_),max(max_) { - } + bin3d_is_sorted_struct(Kokkos::View keys_, + int max_bins_, Scalar min_, Scalar max_) + : keys(keys_), max_bins(max_bins_), min(min_), max(max_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, unsigned int& count) const { - int ix1 = int ((keys(i,0)-min)/max * max_bins); - int iy1 = int ((keys(i,1)-min)/max * max_bins); - int iz1 = int ((keys(i,2)-min)/max * max_bins); - int ix2 = int ((keys(i+1,0)-min)/max * max_bins); - int iy2 = int ((keys(i+1,1)-min)/max * max_bins); - int iz2 = int ((keys(i+1,2)-min)/max * max_bins); + void operator()(int i, unsigned int& count) const { + int ix1 = int((keys(i, 0) - min) / max * max_bins); + int iy1 = int((keys(i, 1) - min) / max * max_bins); + int iz1 = int((keys(i, 2) - min) / max * max_bins); + int ix2 = int((keys(i + 1, 0) - min) / max * max_bins); + int iy2 = int((keys(i + 1, 1) - min) / max * max_bins); + int iz2 = int((keys(i + 1, 2) - min) / max * max_bins); - if (ix1>ix2) count++; - else if(ix1==ix2) { - if (iy1>iy2) count++; - else if ((iy1==iy2) && (iz1>iz2)) count++; + if (ix1 > ix2) + count++; + else if (ix1 == ix2) { + if (iy1 > iy2) + count++; + else if ((iy1 == iy2) && (iz1 > iz2)) + count++; } } }; -template +template struct sum3D { typedef double value_type; typedef ExecutionSpace execution_space; - Kokkos::View keys; + Kokkos::View keys; - sum3D(Kokkos::View keys_):keys(keys_) {} + sum3D(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION - void operator() (int i, double& count) const { - count+=keys(i,0); - count+=keys(i,1); - count+=keys(i,2); + void operator()(int i, double& count) const { + count += keys(i, 0); + count += keys(i, 1); + count += keys(i, 2); } }; -template -void test_1D_sort(unsigned int n,bool force_kokkos) { - typedef Kokkos::View KeyViewType; - KeyViewType keys("Keys",n); +template +void test_1D_sort_impl(unsigned int n, bool force_kokkos) { + typedef Kokkos::View KeyViewType; + KeyViewType keys("Keys", n); // Test sorting array with all numbers equal - Kokkos::deep_copy(keys,KeyType(1)); - Kokkos::sort(keys,force_kokkos); + Kokkos::deep_copy(keys, KeyType(1)); + Kokkos::sort(keys, force_kokkos); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys,g,Kokkos::Random_XorShift64_Pool::generator_type::MAX_URAND); + Kokkos::fill_random(keys, g, + Kokkos::Random_XorShift64_Pool< + ExecutionSpace>::generator_type::MAX_URAND); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(n,sum(keys),sum_before); + Kokkos::parallel_reduce(n, sum(keys), sum_before); - Kokkos::sort(keys,force_kokkos); + Kokkos::sort(keys, force_kokkos); - Kokkos::parallel_reduce(n,sum(keys),sum_after); - Kokkos::parallel_reduce(n-1,is_sorted_struct(keys),sort_fails); + Kokkos::parallel_reduce(n, sum(keys), sum_after); + Kokkos::parallel_reduce( + n - 1, is_sorted_struct(keys), sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } -template -void test_3D_sort(unsigned int n) { - typedef Kokkos::View KeyViewType; +template +void test_3D_sort_impl(unsigned int n) { + typedef Kokkos::View KeyViewType; - KeyViewType keys("Keys",n*n*n); + KeyViewType keys("Keys", n * n * n); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys,g,100.0); + Kokkos::fill_random(keys, g, 100.0); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(keys.extent(0),sum3D(keys),sum_before); + Kokkos::parallel_reduce(keys.extent(0), sum3D(keys), + sum_before); int bin_1d = 1; - while( bin_1d*bin_1d*bin_1d*4< (int) keys.extent(0) ) bin_1d*=2; - int bin_max[3] = {bin_1d,bin_1d,bin_1d}; - typename KeyViewType::value_type min[3] = {0,0,0}; - typename KeyViewType::value_type max[3] = {100,100,100}; + while (bin_1d * bin_1d * bin_1d * 4 < (int)keys.extent(0)) bin_1d *= 2; + int bin_max[3] = {bin_1d, bin_1d, bin_1d}; + typename KeyViewType::value_type min[3] = {0, 0, 0}; + typename KeyViewType::value_type max[3] = {100, 100, 100}; - typedef Kokkos::BinOp3D< KeyViewType > BinOp; - BinOp bin_op(bin_max,min,max); - Kokkos::BinSort< KeyViewType , BinOp > - Sorter(keys,bin_op,false); + typedef Kokkos::BinOp3D BinOp; + BinOp bin_op(bin_max, min, max); + Kokkos::BinSort Sorter(keys, bin_op, false); Sorter.create_permute_vector(); - Sorter.template sort< KeyViewType >(keys); + Sorter.template sort(keys); - Kokkos::parallel_reduce(keys.extent(0),sum3D(keys),sum_after); - Kokkos::parallel_reduce(keys.extent(0)-1,bin3d_is_sorted_struct(keys,bin_1d,min[0],max[0]),sort_fails); + Kokkos::parallel_reduce(keys.extent(0), sum3D(keys), + sum_after); + Kokkos::parallel_reduce(keys.extent(0) - 1, + bin3d_is_sorted_struct( + keys, bin_1d, min[0], max[0]), + sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; - if ( sort_fails ) - printf("3D Sort Sum: %f %f Fails: %u\n",sum_before,sum_after,sort_fails); + if (sort_fails) + printf("3D Sort Sum: %f %f Fails: %u\n", sum_before, sum_after, sort_fails); - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } //---------------------------------------------------------------------------- -template -void test_dynamic_view_sort(unsigned int n ) -{ - typedef Kokkos::Experimental::DynamicView KeyDynamicViewType; - typedef Kokkos::View KeyViewType; +template +void test_dynamic_view_sort_impl(unsigned int n) { + typedef Kokkos::Experimental::DynamicView + KeyDynamicViewType; + typedef Kokkos::View KeyViewType; - const size_t upper_bound = 2 * n ; + const size_t upper_bound = 2 * n; const size_t min_chunk_size = 1024; KeyDynamicViewType keys("Keys", min_chunk_size, upper_bound); keys.resize_serial(n); - KeyViewType keys_view("KeysTmp", n ); + KeyViewType keys_view("KeysTmp", n); // Test sorting array with all numbers equal - Kokkos::deep_copy(keys_view,KeyType(1)); - Kokkos::deep_copy(keys,keys_view); - Kokkos::sort(keys, 0 /* begin */ , n /* end */ ); + Kokkos::deep_copy(keys_view, KeyType(1)); + Kokkos::deep_copy(keys, keys_view); + Kokkos::sort(keys, 0 /* begin */, n /* end */); Kokkos::Random_XorShift64_Pool g(1931); - Kokkos::fill_random(keys_view,g,Kokkos::Random_XorShift64_Pool::generator_type::MAX_URAND); + Kokkos::fill_random(keys_view, g, + Kokkos::Random_XorShift64_Pool< + ExecutionSpace>::generator_type::MAX_URAND); ExecutionSpace().fence(); - Kokkos::deep_copy(keys,keys_view); - //ExecutionSpace().fence(); + Kokkos::deep_copy(keys, keys_view); + // ExecutionSpace().fence(); - double sum_before = 0.0; - double sum_after = 0.0; + double sum_before = 0.0; + double sum_after = 0.0; unsigned int sort_fails = 0; - Kokkos::parallel_reduce(n,sum(keys_view),sum_before); + Kokkos::parallel_reduce(n, sum(keys_view), + sum_before); - Kokkos::sort(keys, 0 /* begin */ , n /* end */ ); + Kokkos::sort(keys, 0 /* begin */, n /* end */); - ExecutionSpace().fence(); // Need this fence to prevent BusError with Cuda - Kokkos::deep_copy( keys_view , keys ); - //ExecutionSpace().fence(); + ExecutionSpace().fence(); // Need this fence to prevent BusError with Cuda + Kokkos::deep_copy(keys_view, keys); + // ExecutionSpace().fence(); - Kokkos::parallel_reduce(n,sum(keys_view),sum_after); - Kokkos::parallel_reduce(n-1,is_sorted_struct(keys_view),sort_fails); + Kokkos::parallel_reduce(n, sum(keys_view), + sum_after); + Kokkos::parallel_reduce( + n - 1, is_sorted_struct(keys_view), sort_fails); - double ratio = sum_before/sum_after; + double ratio = sum_before / sum_after; double epsilon = 1e-10; - unsigned int equal_sum = (ratio > (1.0-epsilon)) && (ratio < (1.0+epsilon)) ? 1 : 0; + unsigned int equal_sum = + (ratio > (1.0 - epsilon)) && (ratio < (1.0 + epsilon)) ? 1 : 0; - if ( sort_fails != 0 || equal_sum != 1 ) { - std::cout << " N = " << n - << " ; sum_before = " << sum_before - << " ; sum_after = " << sum_after - << " ; ratio = " << ratio - << std::endl ; + if (sort_fails != 0 || equal_sum != 1) { + std::cout << " N = " << n << " ; sum_before = " << sum_before + << " ; sum_after = " << sum_after << " ; ratio = " << ratio + << std::endl; } - ASSERT_EQ(sort_fails,0); - ASSERT_EQ(equal_sum,1); + ASSERT_EQ(sort_fails, 0); + ASSERT_EQ(equal_sum, 1); } //---------------------------------------------------------------------------- -template -void test_issue_1160() -{ +template +void test_issue_1160_impl() { Kokkos::View element_("element", 10); Kokkos::View x_("x", 10); Kokkos::View v_("y", 10); auto h_element = Kokkos::create_mirror_view(element_); - auto h_x = Kokkos::create_mirror_view(x_); - auto h_v = Kokkos::create_mirror_view(v_); + auto h_x = Kokkos::create_mirror_view(x_); + auto h_v = Kokkos::create_mirror_view(v_); h_element(0) = 9; h_element(1) = 8; @@ -292,20 +306,21 @@ void test_issue_1160() Kokkos::deep_copy(v_, h_v); typedef decltype(element_) KeyViewType; - typedef Kokkos::BinOp1D< KeyViewType > BinOp; + typedef Kokkos::BinOp1D BinOp; int begin = 3; - int end = 8; - auto max = h_element(begin); - auto min = h_element(end - 1); + int end = 8; + auto max = h_element(begin); + auto min = h_element(end - 1); BinOp binner(end - begin, min, max); - Kokkos::BinSort Sorter(element_,begin,end,binner,false); + Kokkos::BinSort Sorter(element_, begin, end, binner, + false); Sorter.create_permute_vector(); - Sorter.sort(element_,begin,end); + Sorter.sort(element_, begin, end); - Sorter.sort(x_,begin,end); - Sorter.sort(v_,begin,end); + Sorter.sort(x_, begin, end); + Sorter.sort(v_, begin, end); Kokkos::deep_copy(h_element, element_); Kokkos::deep_copy(h_x, x_); @@ -330,18 +345,34 @@ void test_issue_1160() //---------------------------------------------------------------------------- -template -void test_sort(unsigned int N) -{ - test_1D_sort(N*N*N, true); - test_1D_sort(N*N*N, false); -#if !defined(KOKKOS_ENABLE_ROCM) - test_3D_sort(N); - test_dynamic_view_sort(N*N); -#endif - test_issue_1160(); +template +void test_1D_sort(unsigned int N) { + test_1D_sort_impl(N * N * N, true); + test_1D_sort_impl(N * N * N, false); } +template +void test_3D_sort(unsigned int N) { + test_3D_sort_impl(N); } + +template +void test_dynamic_view_sort(unsigned int N) { + test_dynamic_view_sort_impl(N * N); } + +template +void test_issue_1160_sort() { + test_issue_1160_impl(); +} + +template +void test_sort(unsigned int N) { + test_1D_sort(N); + test_3D_sort(N); + test_dynamic_view_sort(N); + test_issue_1160_sort(); +} +} // namespace Impl +} // namespace Test #endif /* KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_HPP */ diff --git a/lib/kokkos/algorithms/unit_tests/TestThreads.cpp b/lib/kokkos/algorithms/unit_tests/TestThreads.cpp index 99cdb7da92..c75e6e8dfb 100644 --- a/lib/kokkos/algorithms/unit_tests/TestThreads.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestThreads.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -52,51 +53,36 @@ #include #include - //---------------------------------------------------------------------------- - namespace Test { -class threads : public ::testing::Test { -protected: - static void SetUpTestCase() - { - std::cout << std::setprecision(5) << std::scientific; +#define THREADS_RANDOM_XORSHIFT64(num_draws) \ + TEST(threads, Random_XorShift64) { \ + Impl::test_random >( \ + num_draws); \ } - static void TearDownTestCase() - { - } -}; - -#define THREADS_RANDOM_XORSHIFT64( num_draws ) \ - TEST_F( threads, Random_XorShift64 ) { \ - Impl::test_random >(num_draws); \ +#define THREADS_RANDOM_XORSHIFT1024(num_draws) \ + TEST(threads, Random_XorShift1024) { \ + Impl::test_random >( \ + num_draws); \ } -#define THREADS_RANDOM_XORSHIFT1024( num_draws ) \ - TEST_F( threads, Random_XorShift1024 ) { \ - Impl::test_random >(num_draws); \ +#define THREADS_SORT_UNSIGNED(size) \ + TEST(threads, SortUnsigned) { \ + Impl::test_sort(size); \ } -#define THREADS_SORT_UNSIGNED( size ) \ - TEST_F( threads, SortUnsigned ) { \ - Impl::test_sort< Kokkos::Threads, double >(size); \ - } - - -THREADS_RANDOM_XORSHIFT64( 10240000 ) -THREADS_RANDOM_XORSHIFT1024( 10130144 ) +THREADS_RANDOM_XORSHIFT64(10240000) +THREADS_RANDOM_XORSHIFT1024(10130144) THREADS_SORT_UNSIGNED(171) #undef THREADS_RANDOM_XORSHIFT64 #undef THREADS_RANDOM_XORSHIFT1024 #undef THREADS_SORT_UNSIGNED -} // namespace Test +} // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTTHREADS_PREVENT_LINK_ERROR() {} #endif - - diff --git a/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp b/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp index 8feb08332f..e245aad35f 100644 --- a/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp +++ b/lib/kokkos/algorithms/unit_tests/UnitTestMain.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -45,10 +46,9 @@ #include int main(int argc, char *argv[]) { - Kokkos::initialize(argc,argv); - ::testing::InitGoogleTest(&argc,argv); + Kokkos::initialize(argc, argv); + ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); Kokkos::finalize(); return result; } - diff --git a/lib/kokkos/appveyor.yml b/lib/kokkos/appveyor.yml new file mode 100644 index 0000000000..8f139ba6ab --- /dev/null +++ b/lib/kokkos/appveyor.yml @@ -0,0 +1,10 @@ +image: + - Visual Studio 2019 +clone_folder: c:\projects\source +build_script: +- cmd: >- + mkdir build && + cd build && + cmake c:\projects\source -DKokkos_ENABLE_TESTS=ON -DKokkos_ENABLE_LIBDL=OFF -DKokkos_ENABLE_PROFILING=OFF && + cmake --build . --target install && + ctest -C Debug -V diff --git a/lib/kokkos/benchmarks/atomic/main.cpp b/lib/kokkos/benchmarks/atomic/main.cpp index d86d196249..5f0977f754 100644 --- a/lib/kokkos/benchmarks/atomic/main.cpp +++ b/lib/kokkos/benchmarks/atomic/main.cpp @@ -1,124 +1,120 @@ -#include -#include -#include +#include +#include +#include -template -double test_atomic(int L, int N, int M,int K,int R,Kokkos::View offsets) { - Kokkos::View output("Output",N); +template +double test_atomic(int L, int N, int M, int K, int R, + Kokkos::View offsets) { + Kokkos::View output("Output", N); Kokkos::Impl::Timer timer; - for(int r = 0; r -double test_no_atomic(int L, int N, int M,int K,int R,Kokkos::View offsets) { - Kokkos::View output("Output",N); +template +double test_no_atomic(int L, int N, int M, int K, int R, + Kokkos::View offsets) { + Kokkos::View output("Output", N); Kokkos::Impl::Timer timer; - for(int r = 0; r\n"); - printf("Example Input GPU:\n"); - printf(" Histogram : 1000000 1000 1 1000 1 10 1\n"); - printf(" MD Force : 100000 100000 100 1000 20 10 4\n"); - printf(" Matrix Assembly : 100000 1000000 50 1000 20 10 4\n"); - Kokkos::finalize(); - return 0; + Kokkos::initialize(argc, argv); + { + if (argc < 8) { + printf("Arguments: L N M D K R T\n"); + printf(" L: Number of iterations to run\n"); + printf(" N: Length of array to do atomics into\n"); + printf(" M: Number of atomics per iteration to do\n"); + printf(" D: Distance from index i to do atomics into (randomly)\n"); + printf(" K: Number of FMAD per atomic\n"); + printf(" R: Number of repeats of the experiments\n"); + printf(" T: Type of atomic\n"); + printf(" 1 - int\n"); + printf(" 2 - long\n"); + printf(" 3 - float\n"); + printf(" 4 - double\n"); + printf(" 5 - complex\n"); + printf("Example Input GPU:\n"); + printf(" Histogram : 1000000 1000 1 1000 1 10 1\n"); + printf(" MD Force : 100000 100000 100 1000 20 10 4\n"); + printf(" Matrix Assembly : 100000 1000000 50 1000 20 10 4\n"); + Kokkos::finalize(); + return 0; + } + + int L = atoi(argv[1]); + int N = atoi(argv[2]); + int M = atoi(argv[3]); + int D = atoi(argv[4]); + int K = atoi(argv[5]); + int R = atoi(argv[6]); + int type = atoi(argv[7]); + + Kokkos::View offsets("Offsets", L, M); + Kokkos::Random_XorShift64_Pool<> pool(12371); + Kokkos::fill_random(offsets, pool, D); + double time = 0; + if (type == 1) time = test_atomic(L, N, M, K, R, offsets); + if (type == 2) time = test_atomic(L, N, M, K, R, offsets); + if (type == 3) time = test_atomic(L, N, M, K, R, offsets); + if (type == 4) time = test_atomic(L, N, M, K, R, offsets); + if (type == 5) + time = test_atomic >(L, N, M, K, R, offsets); + + double time2 = 1; + if (type == 1) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 2) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 3) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 4) time2 = test_no_atomic(L, N, M, K, R, offsets); + if (type == 5) + time2 = test_no_atomic >(L, N, M, K, R, offsets); + + int size = 0; + if (type == 1) size = sizeof(int); + if (type == 2) size = sizeof(long); + if (type == 3) size = sizeof(float); + if (type == 4) size = sizeof(double); + if (type == 5) size = sizeof(Kokkos::complex); + + printf("%i\n", size); + printf( + "Time: %s %i %i %i %i %i %i (t_atomic: %e t_nonatomic: %e ratio: %lf " + ")( GUpdates/s: %lf GB/s: %lf )\n", + (type == 1) + ? "int" + : ((type == 2) + ? "long" + : ((type == 3) ? "float" + : ((type == 4) ? "double" : "complex"))), + L, N, M, D, K, R, time, time2, time / time2, 1.e-9 * L * R * M / time, + 1.0 * L * R * M * 2 * size / time / 1024 / 1024 / 1024); } - - - int L = atoi(argv[1]); - int N = atoi(argv[2]); - int M = atoi(argv[3]); - int D = atoi(argv[4]); - int K = atoi(argv[5]); - int R = atoi(argv[6]); - int type = atoi(argv[7]); - - Kokkos::View offsets("Offsets",L,M); - Kokkos::Random_XorShift64_Pool<> pool(12371); - Kokkos::fill_random(offsets,pool,D); - double time = 0; - if(type==1) - time = test_atomic(L,N,M,K,R,offsets); - if(type==2) - time = test_atomic(L,N,M,K,R,offsets); - if(type==3) - time = test_atomic(L,N,M,K,R,offsets); - if(type==4) - time = test_atomic(L,N,M,K,R,offsets); - if(type==5) - time = test_atomic >(L,N,M,K,R,offsets); - - double time2 = 1; - if(type==1) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==2) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==3) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==4) - time2 = test_no_atomic(L,N,M,K,R,offsets); - if(type==5) - time2 = test_no_atomic >(L,N,M,K,R,offsets); - - int size = 0; - if(type==1) size = sizeof(int); - if(type==2) size = sizeof(long); - if(type==3) size = sizeof(float); - if(type==4) size = sizeof(double); - if(type==5) size = sizeof(Kokkos::complex); - - printf("%i\n",size); - printf("Time: %s %i %i %i %i %i %i (t_atomic: %e t_nonatomic: %e ratio: %lf )( GUpdates/s: %lf GB/s: %lf )\n", - (type==1)?"int": ( - (type==2)?"long": ( - (type==3)?"float": ( - (type==4)?"double":"complex"))), - L,N,M,D,K,R,time,time2,time/time2, - 1.e-9*L*R*M/time, 1.0*L*R*M*2*size/time/1024/1024/1024); -} Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp index 59b4d50c44..62d7ef4a4c 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,59 +42,52 @@ //@HEADER */ -#include -#include +#include +#include -template +template struct Run { -static void run(int N, int K, int R, int F, int T, int S); + static void run(int N, int K, int R, int F, int T, int S); }; -template +template struct RunStride { -static void run_1(int N, int K, int R, int F, int T, int S); -static void run_2(int N, int K, int R, int F, int T, int S); -static void run_3(int N, int K, int R, int F, int T, int S); -static void run_4(int N, int K, int R, int F, int T, int S); -static void run_5(int N, int K, int R, int F, int T, int S); -static void run_6(int N, int K, int R, int F, int T, int S); -static void run_7(int N, int K, int R, int F, int T, int S); -static void run_8(int N, int K, int R, int F, int T, int S); -static void run(int N, int K, int R, int U, int F, int T, int S); + static void run_1(int N, int K, int R, int F, int T, int S); + static void run_2(int N, int K, int R, int F, int T, int S); + static void run_3(int N, int K, int R, int F, int T, int S); + static void run_4(int N, int K, int R, int F, int T, int S); + static void run_5(int N, int K, int R, int F, int T, int S); + static void run_6(int N, int K, int R, int F, int T, int S); + static void run_7(int N, int K, int R, int F, int T, int S); + static void run_8(int N, int K, int R, int F, int T, int S); + static void run(int N, int K, int R, int U, int F, int T, int S); }; #define STRIDE 1 -#include +#include #undef STRIDE #define STRIDE 2 -#include +#include #undef STRIDE #define STRIDE 4 -#include +#include #undef STRIDE #define STRIDE 8 -#include +#include #undef STRIDE #define STRIDE 16 -#include +#include #undef STRIDE #define STRIDE 32 -#include +#include #undef STRIDE -template +template void run_stride_unroll(int N, int K, int R, int D, int U, int F, int T, int S) { - if(D == 1) - RunStride::run(N,K,R,U,F,T,S); - if(D == 2) - RunStride::run(N,K,R,U,F,T,S); - if(D == 4) - RunStride::run(N,K,R,U,F,T,S); - if(D == 8) - RunStride::run(N,K,R,U,F,T,S); - if(D == 16) - RunStride::run(N,K,R,U,F,T,S); - if(D == 32) - RunStride::run(N,K,R,U,F,T,S); + if (D == 1) RunStride::run(N, K, R, U, F, T, S); + if (D == 2) RunStride::run(N, K, R, U, F, T, S); + if (D == 4) RunStride::run(N, K, R, U, F, T, S); + if (D == 8) RunStride::run(N, K, R, U, F, T, S); + if (D == 16) RunStride::run(N, K, R, U, F, T, S); + if (D == 32) RunStride::run(N, K, R, U, F, T, S); } - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp index 6509c654e7..64817fe9dc 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench_stride.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,84 +42,82 @@ //@HEADER */ - #define UNROLL 1 -#include +#include #undef UNROLL #define UNROLL 2 -#include +#include #undef UNROLL #define UNROLL 3 -#include +#include #undef UNROLL #define UNROLL 4 -#include +#include #undef UNROLL #define UNROLL 5 -#include +#include #undef UNROLL #define UNROLL 6 -#include +#include #undef UNROLL #define UNROLL 7 -#include +#include #undef UNROLL #define UNROLL 8 -#include +#include #undef UNROLL -template -struct RunStride { -static void run_1(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_2(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_3(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_4(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_5(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_6(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_7(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} -static void run_8(int N, int K, int R, int F, int T, int S) { - Run::run(N,K,R,F,T,S); -} +template +struct RunStride { + static void run_1(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_2(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_3(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_4(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_5(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_6(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_7(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } + static void run_8(int N, int K, int R, int F, int T, int S) { + Run::run(N, K, R, F, T, S); + } -static void run(int N, int K, int R, int U, int F, int T, int S) { - if(U==1) { - run_1(N,K,R,F,T,S); + static void run(int N, int K, int R, int U, int F, int T, int S) { + if (U == 1) { + run_1(N, K, R, F, T, S); + } + if (U == 2) { + run_2(N, K, R, F, T, S); + } + if (U == 3) { + run_3(N, K, R, F, T, S); + } + if (U == 4) { + run_4(N, K, R, F, T, S); + } + if (U == 5) { + run_5(N, K, R, F, T, S); + } + if (U == 6) { + run_6(N, K, R, F, T, S); + } + if (U == 7) { + run_7(N, K, R, F, T, S); + } + if (U == 8) { + run_8(N, K, R, F, T, S); + } } - if(U==2) { - run_2(N,K,R,F,T,S); - } - if(U==3) { - run_3(N,K,R,F,T,S); - } - if(U==4) { - run_4(N,K,R,F,T,S); - } - if(U==5) { - run_5(N,K,R,F,T,S); - } - if(U==6) { - run_6(N,K,R,F,T,S); - } - if(U==7) { - run_7(N,K,R,F,T,S); - } - if(U==8) { - run_8(N,K,R,F,T,S); - } -} }; - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp index c6651da1e7..00ce635a48 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,108 +42,110 @@ //@HEADER */ -template -struct Run { -static void run(int N, int K, int R, int F, int T, int S) { - Kokkos::View A("A",N,K); - Kokkos::View B("B",N,K); - Kokkos::View C("C",N,K); +template +struct Run { + static void run(int N, int K, int R, int F, int T, int S) { + Kokkos::View A("A", N, K); + Kokkos::View B("B", N, K); + Kokkos::View C("C", N, K); - Kokkos::deep_copy(A,Scalar(1.5)); - Kokkos::deep_copy(B,Scalar(2.5)); - Kokkos::deep_copy(C,Scalar(3.5)); + Kokkos::deep_copy(A, Scalar(1.5)); + Kokkos::deep_copy(B, Scalar(2.5)); + Kokkos::deep_copy(C, Scalar(3.5)); - Kokkos::Timer timer; - Kokkos::parallel_for("BenchmarkKernel",Kokkos::TeamPolicy<>(N,T).set_scratch_size(0,Kokkos::PerTeam(S)), - KOKKOS_LAMBDA ( const Kokkos::TeamPolicy<>::member_type& team) { - const int n = team.league_rank(); - for(int r=0; r1) - Scalar a2 = a1*1.3; + Kokkos::Timer timer; + Kokkos::parallel_for( + "BenchmarkKernel", + Kokkos::TeamPolicy<>(N, T).set_scratch_size(0, Kokkos::PerTeam(S)), + KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type& team) { + const int n = team.league_rank(); + for (int r = 0; r < R; r++) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, 0, K), [&](const int& i) { + Scalar a1 = A(n, i, 0); + const Scalar b = B(n, i, 0); +#if (UNROLL > 1) + Scalar a2 = a1 * 1.3; #endif -#if(UNROLL>2) - Scalar a3 = a2*1.1; +#if (UNROLL > 2) + Scalar a3 = a2 * 1.1; #endif -#if(UNROLL>3) - Scalar a4 = a3*1.1; +#if (UNROLL > 3) + Scalar a4 = a3 * 1.1; #endif -#if(UNROLL>4) - Scalar a5 = a4*1.3; +#if (UNROLL > 4) + Scalar a5 = a4 * 1.3; #endif -#if(UNROLL>5) - Scalar a6 = a5*1.1; +#if (UNROLL > 5) + Scalar a6 = a5 * 1.1; #endif -#if(UNROLL>6) - Scalar a7 = a6*1.1; +#if (UNROLL > 6) + Scalar a7 = a6 * 1.1; #endif -#if(UNROLL>7) - Scalar a8 = a7*1.1; +#if (UNROLL > 7) + Scalar a8 = a7 * 1.1; #endif + for (int f = 0; f < F; f++) { + a1 += b * a1; +#if (UNROLL > 1) + a2 += b * a2; +#endif +#if (UNROLL > 2) + a3 += b * a3; +#endif +#if (UNROLL > 3) + a4 += b * a4; +#endif +#if (UNROLL > 4) + a5 += b * a5; +#endif +#if (UNROLL > 5) + a6 += b * a6; +#endif +#if (UNROLL > 6) + a7 += b * a7; +#endif +#if (UNROLL > 7) + a8 += b * a8; +#endif + } +#if (UNROLL == 1) + C(n, i, 0) = a1; +#endif +#if (UNROLL == 2) + C(n, i, 0) = a1 + a2; +#endif +#if (UNROLL == 3) + C(n, i, 0) = a1 + a2 + a3; +#endif +#if (UNROLL == 4) + C(n, i, 0) = a1 + a2 + a3 + a4; +#endif +#if (UNROLL == 5) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5; +#endif +#if (UNROLL == 6) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6; +#endif +#if (UNROLL == 7) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6 + a7; +#endif +#if (UNROLL == 8) + C(n, i, 0) = a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8; +#endif + }); + } + }); + Kokkos::fence(); + double seconds = timer.seconds(); - for(int f = 0; f1) - a2 += b*a2; -#endif -#if(UNROLL>2) - a3 += b*a3; -#endif -#if(UNROLL>3) - a4 += b*a4; -#endif -#if(UNROLL>4) - a5 += b*a5; -#endif -#if(UNROLL>5) - a6 += b*a6; -#endif -#if(UNROLL>6) - a7 += b*a7; -#endif -#if(UNROLL>7) - a8 += b*a8; -#endif - - - } -#if(UNROLL==1) - C(n,i,0) = a1; -#endif -#if(UNROLL==2) - C(n,i,0) = a1+a2; -#endif -#if(UNROLL==3) - C(n,i,0) = a1+a2+a3; -#endif -#if(UNROLL==4) - C(n,i,0) = a1+a2+a3+a4; -#endif -#if(UNROLL==5) - C(n,i,0) = a1+a2+a3+a4+a5; -#endif -#if(UNROLL==6) - C(n,i,0) = a1+a2+a3+a4+a5+a6; -#endif -#if(UNROLL==7) - C(n,i,0) = a1+a2+a3+a4+a5+a6+a7; -#endif -#if(UNROLL==8) - C(n,i,0) = a1+a2+a3+a4+a5+a6+a7+a8; -#endif - - }); - } - }); - Kokkos::fence(); - double seconds = timer.seconds(); - - double bytes = 1.0*N*K*R*3*sizeof(Scalar); - double flops = 1.0*N*K*R*(F*2*UNROLL + 2*(UNROLL-1)); - printf("NKRUFTS: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: %lf\n",N,K,R,UNROLL,F,T,S,seconds,1.0*bytes/seconds/1024/1024/1024,1.e-9*flops/seconds); -} + double bytes = 1.0 * N * K * R * 3 * sizeof(Scalar); + double flops = 1.0 * N * K * R * (F * 2 * UNROLL + 2 * (UNROLL - 1)); + printf( + "NKRUFTS: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: " + "%lf\n", + N, K, R, UNROLL, F, T, S, seconds, + 1.0 * bytes / seconds / 1024 / 1024 / 1024, 1.e-9 * flops / seconds); + } }; - diff --git a/lib/kokkos/benchmarks/bytes_and_flops/main.cpp b/lib/kokkos/benchmarks/bytes_and_flops/main.cpp index 4f46b38717..c21a16200e 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/main.cpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,25 +42,27 @@ //@HEADER */ -#include -#include -#include -#include +#include +#include +#include +#include int main(int argc, char* argv[]) { Kokkos::initialize(); - - if(argc<10) { + if (argc < 10) { printf("Arguments: N K R D U F T S\n"); printf(" P: Precision (1==float, 2==double)\n"); printf(" N,K: dimensions of the 2D array to allocate\n"); printf(" R: how often to loop through the K dimension with each team\n"); printf(" D: distance between loaded elements (stride)\n"); printf(" U: how many independent flops to do per load\n"); - printf(" F: how many times to repeat the U unrolled operations before reading next element\n"); + printf( + " F: how many times to repeat the U unrolled operations before " + "reading next element\n"); printf(" T: team size\n"); - printf(" S: shared memory per team (used to control occupancy on GPUs)\n"); + printf( + " S: shared memory per team (used to control occupancy on GPUs)\n"); printf("Example Input GPU:\n"); printf(" Bandwidth Bound : 2 100000 1024 1 1 1 1 256 6000\n"); printf(" Cache Bound : 2 100000 1024 64 1 1 1 512 20000\n"); @@ -70,7 +73,6 @@ int main(int argc, char* argv[]) { return 0; } - int P = atoi(argv[1]); int N = atoi(argv[2]); int K = atoi(argv[3]); @@ -81,17 +83,25 @@ int main(int argc, char* argv[]) { int T = atoi(argv[8]); int S = atoi(argv[9]); - if(U>8) {printf("U must be 1-8\n"); return 0;} - if( (D!=1) && (D!=2) && (D!=4) && (D!=8) && (D!=16) && (D!=32)) {printf("D must be one of 1,2,4,8,16,32\n"); return 0;} - if( (P!=1) && (P!=2) ) {printf("P must be one of 1,2\n"); return 0;} - - if(P==1) { - run_stride_unroll(N,K,R,D,U,F,T,S); + if (U > 8) { + printf("U must be 1-8\n"); + return 0; } - if(P==2) { - run_stride_unroll(N,K,R,D,U,F,T,S); + if ((D != 1) && (D != 2) && (D != 4) && (D != 8) && (D != 16) && (D != 32)) { + printf("D must be one of 1,2,4,8,16,32\n"); + return 0; + } + if ((P != 1) && (P != 2)) { + printf("P must be one of 1,2\n"); + return 0; + } + + if (P == 1) { + run_stride_unroll(N, K, R, D, U, F, T, S); + } + if (P == 2) { + run_stride_unroll(N, K, R, D, U, F, T, S); } Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/gather/gather.hpp b/lib/kokkos/benchmarks/gather/gather.hpp index bbbd65850f..239614184b 100644 --- a/lib/kokkos/benchmarks/gather/gather.hpp +++ b/lib/kokkos/benchmarks/gather/gather.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,52 +42,44 @@ //@HEADER */ -template +template struct RunGather { static void run(int N, int K, int D, int R, int F); }; #define UNROLL 1 -#include +#include #undef UNROLL #define UNROLL 2 -#include +#include #undef UNROLL #define UNROLL 3 -#include +#include #undef UNROLL #define UNROLL 4 -#include +#include #undef UNROLL #define UNROLL 5 -#include +#include #undef UNROLL #define UNROLL 6 -#include +#include #undef UNROLL #define UNROLL 7 -#include +#include #undef UNROLL #define UNROLL 8 -#include +#include #undef UNROLL -template +template void run_gather_test(int N, int K, int D, int R, int U, int F) { - if(U == 1) - RunGather::run(N,K,D,R,F); - if(U == 2) - RunGather::run(N,K,D,R,F); - if(U == 3) - RunGather::run(N,K,D,R,F); - if(U == 4) - RunGather::run(N,K,D,R,F); - if(U == 5) - RunGather::run(N,K,D,R,F); - if(U == 6) - RunGather::run(N,K,D,R,F); - if(U == 7) - RunGather::run(N,K,D,R,F); - if(U == 8) - RunGather::run(N,K,D,R,F); + if (U == 1) RunGather::run(N, K, D, R, F); + if (U == 2) RunGather::run(N, K, D, R, F); + if (U == 3) RunGather::run(N, K, D, R, F); + if (U == 4) RunGather::run(N, K, D, R, F); + if (U == 5) RunGather::run(N, K, D, R, F); + if (U == 6) RunGather::run(N, K, D, R, F); + if (U == 7) RunGather::run(N, K, D, R, F); + if (U == 8) RunGather::run(N, K, D, R, F); } diff --git a/lib/kokkos/benchmarks/gather/gather_unroll.hpp b/lib/kokkos/benchmarks/gather/gather_unroll.hpp index 1d9c99adf9..4dc046f99c 100644 --- a/lib/kokkos/benchmarks/gather/gather_unroll.hpp +++ b/lib/kokkos/benchmarks/gather/gather_unroll.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,129 +42,132 @@ //@HEADER */ -#include -#include +#include +#include -template -struct RunGather { -static void run(int N, int K, int D, int R, int F) { - Kokkos::View connectivity("Connectivity",N,K); - Kokkos::View A_in("Input",N); - Kokkos::View B_in("Input",N); - Kokkos::View C("Output",N); +template +struct RunGather { + static void run(int N, int K, int D, int R, int F) { + Kokkos::View connectivity("Connectivity", N, K); + Kokkos::View A_in("Input", N); + Kokkos::View B_in("Input", N); + Kokkos::View C("Output", N); - Kokkos::Random_XorShift64_Pool<> rand_pool(12313); + Kokkos::Random_XorShift64_Pool<> rand_pool(12313); - Kokkos::deep_copy(A_in,1.5); - Kokkos::deep_copy(B_in,2.0); + Kokkos::deep_copy(A_in, 1.5); + Kokkos::deep_copy(B_in, 2.0); - Kokkos::View > A(A_in); - Kokkos::View > B(B_in); + Kokkos::View > A( + A_in); + Kokkos::View > B( + B_in); - Kokkos::parallel_for("InitKernel",N, - KOKKOS_LAMBDA (const int& i) { - auto rand_gen = rand_pool.get_state(); - for( int jj=0; jj 1) + Scalar a2 = a1 * Scalar(1.3); +#endif +#if (UNROLL > 2) + Scalar a3 = a2 * Scalar(1.1); +#endif +#if (UNROLL > 3) + Scalar a4 = a3 * Scalar(1.1); +#endif +#if (UNROLL > 4) + Scalar a5 = a4 * Scalar(1.3); +#endif +#if (UNROLL > 5) + Scalar a6 = a5 * Scalar(1.1); +#endif +#if (UNROLL > 6) + Scalar a7 = a6 * Scalar(1.1); +#endif +#if (UNROLL > 7) + Scalar a8 = a7 * Scalar(1.1); +#endif + + for (int f = 0; f < F; f++) { + a1 += b * a1; +#if (UNROLL > 1) + a2 += b * a2; +#endif +#if (UNROLL > 2) + a3 += b * a3; +#endif +#if (UNROLL > 3) + a4 += b * a4; +#endif +#if (UNROLL > 4) + a5 += b * a5; +#endif +#if (UNROLL > 5) + a6 += b * a6; +#endif +#if (UNROLL > 6) + a7 += b * a7; +#endif +#if (UNROLL > 7) + a8 += b * a8; +#endif + } +#if (UNROLL == 1) + c += a1; +#endif +#if (UNROLL == 2) + c += a1 + a2; +#endif +#if (UNROLL == 3) + c += a1 + a2 + a3; +#endif +#if (UNROLL == 4) + c += a1 + a2 + a3 + a4; +#endif +#if (UNROLL == 5) + c += a1 + a2 + a3 + a4 + a5; +#endif +#if (UNROLL == 6) + c += a1 + a2 + a3 + a4 + a5 + a6; +#endif +#if (UNROLL == 7) + c += a1 + a2 + a3 + a4 + a5 + a6 + a7; +#endif +#if (UNROLL == 8) + c += a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8; +#endif + } + C(i) = c; + }); + Kokkos::fence(); } - rand_pool.free_state(rand_gen); - }); - Kokkos::fence(); + double seconds = timer.seconds(); - - Kokkos::Timer timer; - for(int r = 0; r1) - Scalar a2 = a1*Scalar(1.3); -#endif -#if(UNROLL>2) - Scalar a3 = a2*Scalar(1.1); -#endif -#if(UNROLL>3) - Scalar a4 = a3*Scalar(1.1); -#endif -#if(UNROLL>4) - Scalar a5 = a4*Scalar(1.3); -#endif -#if(UNROLL>5) - Scalar a6 = a5*Scalar(1.1); -#endif -#if(UNROLL>6) - Scalar a7 = a6*Scalar(1.1); -#endif -#if(UNROLL>7) - Scalar a8 = a7*Scalar(1.1); -#endif - - - for(int f = 0; f1) - a2 += b*a2; -#endif -#if(UNROLL>2) - a3 += b*a3; -#endif -#if(UNROLL>3) - a4 += b*a4; -#endif -#if(UNROLL>4) - a5 += b*a5; -#endif -#if(UNROLL>5) - a6 += b*a6; -#endif -#if(UNROLL>6) - a7 += b*a7; -#endif -#if(UNROLL>7) - a8 += b*a8; -#endif - - - } -#if(UNROLL==1) - c += a1; -#endif -#if(UNROLL==2) - c += a1+a2; -#endif -#if(UNROLL==3) - c += a1+a2+a3; -#endif -#if(UNROLL==4) - c += a1+a2+a3+a4; -#endif -#if(UNROLL==5) - c += a1+a2+a3+a4+a5; -#endif -#if(UNROLL==6) - c += a1+a2+a3+a4+a5+a6; -#endif -#if(UNROLL==7) - c += a1+a2+a3+a4+a5+a6+a7; -#endif -#if(UNROLL==8) - c += a1+a2+a3+a4+a5+a6+a7+a8; -#endif - - } - C(i) = c ; - }); - Kokkos::fence(); + double bytes = 1.0 * N * K * R * (2 * sizeof(Scalar) + sizeof(int)) + + 1.0 * N * R * sizeof(Scalar); + double flops = 1.0 * N * K * R * (F * 2 * UNROLL + 2 * (UNROLL - 1)); + double gather_ops = 1.0 * N * K * R * 2; + printf( + "SNKDRUF: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: " + "%lf GGather/s: %lf\n", + sizeof(Scalar) / 4, N, K, D, R, UNROLL, F, seconds, + 1.0 * bytes / seconds / 1024 / 1024 / 1024, 1.e-9 * flops / seconds, + 1.e-9 * gather_ops / seconds); } - double seconds = timer.seconds(); - - double bytes = 1.0*N*K*R*(2*sizeof(Scalar)+sizeof(int)) + 1.0*N*R*sizeof(Scalar); - double flops = 1.0*N*K*R*(F*2*UNROLL + 2*(UNROLL-1)); - double gather_ops = 1.0*N*K*R*2; - printf("SNKDRUF: %i %i %i %i %i %i %i Time: %lfs Bandwidth: %lfGiB/s GFlop/s: %lf GGather/s: %lf\n",sizeof(Scalar)/4,N,K,D,R,UNROLL,F,seconds,1.0*bytes/seconds/1024/1024/1024,1.e-9*flops/seconds,1.e-9*gather_ops/seconds); -} }; diff --git a/lib/kokkos/benchmarks/gather/main.cpp b/lib/kokkos/benchmarks/gather/main.cpp index ca5238e7fd..6a2db3e024 100644 --- a/lib/kokkos/benchmarks/gather/main.cpp +++ b/lib/kokkos/benchmarks/gather/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -41,23 +42,26 @@ //@HEADER */ -#include -#include -#include -#include +#include +#include +#include +#include int main(int argc, char* argv[]) { - Kokkos::initialize(argc,argv); + Kokkos::initialize(argc, argv); - if(argc<8) { + if (argc < 8) { printf("Arguments: S N K D\n"); - printf(" S: Scalar Type Size (1==float, 2==double, 4=complex)\n"); + printf( + " S: Scalar Type Size (1==float, 2==double, 4=complex)\n"); printf(" N: Number of entities\n"); printf(" K: Number of things to gather per entity\n"); printf(" D: Max distance of gathered things of an entity\n"); printf(" R: how often to loop through the K dimension with each team\n"); printf(" U: how many independent flops to do per load\n"); - printf(" F: how many times to repeat the U unrolled operations before reading next element\n"); + printf( + " F: how many times to repeat the U unrolled operations before " + "reading next element\n"); printf("Example Input GPU:\n"); printf(" Bandwidth Bound : 2 10000000 1 1 10 1 1\n"); printf(" Cache Bound : 2 10000000 64 1 10 1 1\n"); @@ -68,7 +72,6 @@ int main(int argc, char* argv[]) { return 0; } - int S = atoi(argv[1]); int N = atoi(argv[2]); int K = atoi(argv[3]); @@ -77,17 +80,22 @@ int main(int argc, char* argv[]) { int U = atoi(argv[6]); int F = atoi(argv[7]); - if( (S!=1) && (S!=2) && (S!=4)) {printf("S must be one of 1,2,4\n"); return 0;} - if( N(N,K,D,R,U,F); + if ((S != 1) && (S != 2) && (S != 4)) { + printf("S must be one of 1,2,4\n"); + return 0; } - if(S==2) { - run_gather_test(N,K,D,R,U,F); + if (N < D) { + printf("N must be larger or equal to D\n"); + return 0; } - if(S==4) { - run_gather_test >(N,K,D,R,U,F); + if (S == 1) { + run_gather_test(N, K, D, R, U, F); + } + if (S == 2) { + run_gather_test(N, K, D, R, U, F); + } + if (S == 4) { + run_gather_test >(N, K, D, R, U, F); } Kokkos::finalize(); } - diff --git a/lib/kokkos/benchmarks/gups/gups-kokkos.cc b/lib/kokkos/benchmarks/gups/gups-kokkos.cc index 4602adda79..36fc36925b 100644 --- a/lib/kokkos/benchmarks/gups/gups-kokkos.cc +++ b/lib/kokkos/benchmarks/gups/gups-kokkos.cc @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -60,7 +61,7 @@ typedef int GUPSIndex; double now() { struct timeval now; - gettimeofday(&now, NULL); + gettimeofday(&now, nullptr); return (double) now.tv_sec + ((double) now.tv_usec * 1.0e-6); } diff --git a/lib/kokkos/benchmarks/policy_performance/main.cpp b/lib/kokkos/benchmarks/policy_performance/main.cpp index 2f5395734a..332e5574da 100644 --- a/lib/kokkos/benchmarks/policy_performance/main.cpp +++ b/lib/kokkos/benchmarks/policy_performance/main.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -44,67 +45,86 @@ #include #include "policy_perf_test.hpp" -int main(int argc, char* argv[] ) { - Kokkos::initialize(argc,argv); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); - if(argc<10) { + if (argc < 10) { printf(" Ten arguments are needed to run this program:\n"); - printf(" (1)team_range, (2)thread_range, (3)vector_range, (4)outer_repeat, (5)thread_repeat, (6)vector_repeat, (7)team_size, (8)vector_size, (9)schedule, (10)test_type\n"); + printf( + " (1)team_range, (2)thread_range, (3)vector_range, (4)outer_repeat, " + "(5)thread_repeat, (6)vector_repeat, (7)team_size, (8)vector_size, " + "(9)schedule, (10)test_type\n"); printf(" team_range: number of teams (league_size)\n"); printf(" thread_range: range for nested TeamThreadRange parallel_*\n"); printf(" vector_range: range for nested ThreadVectorRange parallel_*\n"); printf(" outer_repeat: number of repeats for outer parallel_* call\n"); - printf(" thread_repeat: number of repeats for TeamThreadRange parallel_* call\n"); - printf(" vector_repeat: number of repeats for ThreadVectorRange parallel_* call\n"); + printf( + " thread_repeat: number of repeats for TeamThreadRange parallel_* " + "call\n"); + printf( + " vector_repeat: number of repeats for ThreadVectorRange parallel_* " + "call\n"); printf(" team_size: number of team members (team_size)\n"); printf(" vector_size: desired vectorization (if possible)\n"); printf(" schedule: 1 == Static 2 == Dynamic\n"); - printf(" test_type: 3-digit code XYZ for testing (nested) parallel_*\n"); - printf(" code key: XYZ X in {1,2,3,4,5}, Y in {0,1,2}, Z in {0,1,2}\n"); + printf( + " test_type: 3-digit code XYZ for testing (nested) parallel_*\n"); + printf( + " code key: XYZ X in {1,2,3,4,5}, Y in {0,1,2}, Z in " + "{0,1,2}\n"); printf(" TeamPolicy:\n"); - printf(" X: 0 = none (never used, makes no sense); 1 = parallel_for; 2 = parallel_reduce\n"); - printf(" Y: 0 = none; 1 = parallel_for; 2 = parallel_reduce\n"); - printf(" Z: 0 = none; 1 = parallel_for; 2 = parallel_reduce\n"); + printf( + " X: 0 = none (never used, makes no sense); 1 = " + "parallel_for; 2 = parallel_reduce\n"); + printf( + " Y: 0 = none; 1 = parallel_for; 2 = " + "parallel_reduce\n"); + printf( + " Z: 0 = none; 1 = parallel_for; 2 = " + "parallel_reduce\n"); printf(" RangePolicy:\n"); - printf(" X: 3 = parallel_for; 4 = parallel_reduce; 5 = parallel_scan\n"); + printf( + " X: 3 = parallel_for; 4 = parallel_reduce; 5 = " + "parallel_scan\n"); printf(" Y: 0 = none\n"); printf(" Z: 0 = none\n"); printf(" Example Input:\n"); - printf(" 100000 32 32 100 100 100 8 1 1 100\n"); + printf(" 100000 32 32 100 100 100 8 1 1 100\n"); Kokkos::finalize(); return 0; } - int team_range = atoi(argv[1]); + int team_range = atoi(argv[1]); int thread_range = atoi(argv[2]); int vector_range = atoi(argv[3]); - int outer_repeat = atoi(argv[4]); + int outer_repeat = atoi(argv[4]); int thread_repeat = atoi(argv[5]); int vector_repeat = atoi(argv[6]); - int team_size = atoi(argv[7]); + int team_size = atoi(argv[7]); int vector_size = atoi(argv[8]); - int schedule = atoi(argv[9]); - int test_type = atoi(argv[10]); + int schedule = atoi(argv[9]); + int test_type = atoi(argv[10]); - int disable_verbose_output = 0; - if ( argc > 11 ) { + int disable_verbose_output = 0; + if (argc > 11) { disable_verbose_output = atoi(argv[11]); } - if ( schedule != 1 && schedule != 2 ) { + if (schedule != 1 && schedule != 2) { printf("schedule: %d\n", schedule); printf("Options for schedule are: 1 == Static 2 == Dynamic\n"); Kokkos::finalize(); return -1; } - if ( test_type != 100 && test_type != 110 && test_type != 111 && test_type != 112 && test_type != 120 && test_type != 121 && test_type != 122 - && test_type != 200 && test_type != 210 && test_type != 211 && test_type != 212 && test_type != 220 && test_type != 221 && test_type != 222 - && test_type != 300 && test_type != 400 && test_type != 500 - ) - { + if (test_type != 100 && test_type != 110 && test_type != 111 && + test_type != 112 && test_type != 120 && test_type != 121 && + test_type != 122 && test_type != 200 && test_type != 210 && + test_type != 211 && test_type != 212 && test_type != 220 && + test_type != 221 && test_type != 222 && test_type != 300 && + test_type != 400 && test_type != 500) { printf("Incorrect test_type option\n"); Kokkos::finalize(); return -2; @@ -112,56 +132,85 @@ int main(int argc, char* argv[] ) { double result = 0.0; - Kokkos::parallel_reduce( "parallel_reduce warmup", Kokkos::TeamPolicy<>(10,1), - KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type team, double& lval) { - lval += 1; - }, result); + Kokkos::parallel_reduce( + "parallel_reduce warmup", Kokkos::TeamPolicy<>(10, 1), + KOKKOS_LAMBDA(const Kokkos::TeamPolicy<>::member_type team, + double& lval) { lval += 1; }, + result); - typedef Kokkos::View view_type_1d; - typedef Kokkos::View view_type_2d; + typedef Kokkos::View view_type_1d; + typedef Kokkos::View view_type_2d; typedef Kokkos::View view_type_3d; // Allocate view without initializing - // Call a 'warmup' test with 1 repeat - this will initialize the corresponding view appropriately for test and should obey first-touch etc - // Second call to test is the one we actually care about and time - view_type_1d v_1( Kokkos::ViewAllocateWithoutInitializing("v_1"), team_range*team_size); - view_type_2d v_2( Kokkos::ViewAllocateWithoutInitializing("v_2"), team_range*team_size, thread_range); - view_type_3d v_3( Kokkos::ViewAllocateWithoutInitializing("v_3"), team_range*team_size, thread_range, vector_range); + // Call a 'warmup' test with 1 repeat - this will initialize the corresponding + // view appropriately for test and should obey first-touch etc Second call to + // test is the one we actually care about and time + view_type_1d v_1(Kokkos::ViewAllocateWithoutInitializing("v_1"), + team_range * team_size); + view_type_2d v_2(Kokkos::ViewAllocateWithoutInitializing("v_2"), + team_range * team_size, thread_range); + view_type_3d v_3(Kokkos::ViewAllocateWithoutInitializing("v_3"), + team_range * team_size, thread_range, vector_range); double result_computed = 0.0; - double result_expect = 0.0; - double time = 0.0; + double result_expect = 0.0; + double time = 0.0; - if(schedule==1) { - if ( test_type != 500 ) { + if (schedule == 1) { + if (test_type != 500) { // warmup - no repeat of loops - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - } - else { + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, test_type, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); + } else { // parallel_scan: initialize 1d view for parallel_scan - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,100,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, 100, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); } } - if(schedule==2) { - if ( test_type != 500 ) { + if (schedule == 2) { + if (test_type != 500) { // warmup - no repeat of loops - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); - } - else { + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, test_type, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); + } else { // parallel_scan: initialize 1d view for parallel_scan - test_policy,int>(team_range,thread_range,vector_range,1,1,1,team_size,vector_size,100,v_1,v_2,v_3,result_computed,result_expect,time); - test_policy,int>(team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,test_type,v_1,v_2,v_3,result_computed,result_expect,time); + test_policy, int>( + team_range, thread_range, vector_range, 1, 1, 1, team_size, + vector_size, 100, v_1, v_2, v_3, result_computed, result_expect, + time); + test_policy, int>( + team_range, thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, test_type, v_1, v_2, v_3, + result_computed, result_expect, time); } } - if ( disable_verbose_output == 0 ) { - printf("%7i %4i %2i %9i %4i %4i %4i %2i %1i %3i %e %e %lf\n",team_range,thread_range,vector_range,outer_repeat,thread_repeat,vector_repeat,team_size,vector_size,schedule,test_type,result_computed,result_expect,time); - } - else { - printf("%lf\n",time); + if (disable_verbose_output == 0) { + printf("%7i %4i %2i %9i %4i %4i %4i %2i %1i %3i %e %e %lf\n", team_range, + thread_range, vector_range, outer_repeat, thread_repeat, + vector_repeat, team_size, vector_size, schedule, test_type, + result_computed, result_expect, time); + } else { + printf("%lf\n", time); } Kokkos::finalize(); diff --git a/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp b/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp index 1ab437928d..7a1500891f 100644 --- a/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp +++ b/lib/kokkos/benchmarks/policy_performance/policy_perf_test.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -43,297 +44,375 @@ #include -template < class ViewType > +template struct ParallelScanFunctor { using value_type = double; ViewType v; - ParallelScanFunctor( const ViewType & v_ ) - : v(v_) - {} + ParallelScanFunctor(const ViewType& v_) : v(v_) {} KOKKOS_INLINE_FUNCTION - void operator()( const int idx, value_type& val, const bool& final ) const - { - // inclusive scan - val += v(idx); - if ( final ) { - v(idx) = val; - } + void operator()(const int idx, value_type& val, const bool& final) const { + // inclusive scan + val += v(idx); + if (final) { + v(idx) = val; } + } }; -template +template void test_policy(int team_range, int thread_range, int vector_range, - int outer_repeat, int thread_repeat, int inner_repeat, - int team_size, int vector_size, int test_type, - ViewType1 &v1, ViewType2 &v2, ViewType3 &v3, - double &result, double &result_expect, double &time) { - - typedef Kokkos::TeamPolicy t_policy; + int outer_repeat, int thread_repeat, int inner_repeat, + int team_size, int vector_size, int test_type, ViewType1& v1, + ViewType2& v2, ViewType3& v3, double& result, + double& result_expect, double& time) { + typedef Kokkos::TeamPolicy t_policy; typedef typename t_policy::member_type t_team; Kokkos::Timer timer; - for(int orep = 0; orep(v1) + Kokkos::parallel_scan("500 outer scan", team_size * team_range, + ParallelScanFunctor(v1) #if 0 // This does not compile with pre Cuda 8.0 - see Github Issue #913 for explanation KOKKOS_LAMBDA (const int idx, double& val, const bool& final) { @@ -345,11 +424,12 @@ void test_policy(int team_range, int thread_range, int vector_range, } #endif ); - // result = v1( team_size*team_range - 1 ); // won't work with Cuda - need to copy result back to host to print - // result_expect = 0.5*(team_size*team_range)*(team_size*team_range-1); + // result = v1( team_size*team_range - 1 ); // won't work with Cuda - need + // to copy result back to host to print result_expect = + // 0.5*(team_size*team_range)*(team_size*team_range-1); } - } // end outer for loop + } // end outer for loop time = timer.seconds(); -} //end test_policy +} // end test_policy diff --git a/lib/kokkos/benchmarks/stream/stream-kokkos.cc b/lib/kokkos/benchmarks/stream/stream-kokkos.cc index 370995432e..8d604079d4 100644 --- a/lib/kokkos/benchmarks/stream/stream-kokkos.cc +++ b/lib/kokkos/benchmarks/stream/stream-kokkos.cc @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -63,7 +64,7 @@ typedef int StreamIndex; double now() { struct timeval now; - gettimeofday(&now, NULL); + gettimeofday(&now, nullptr); return (double) now.tv_sec + ((double) now.tv_usec * 1.0e-6); } diff --git a/lib/kokkos/bin/nvcc_wrapper b/lib/kokkos/bin/nvcc_wrapper index 94bc72854e..8a23d0d620 100755 --- a/lib/kokkos/bin/nvcc_wrapper +++ b/lib/kokkos/bin/nvcc_wrapper @@ -85,11 +85,11 @@ first_xcompiler_arg=1 temp_dir=${TMPDIR:-/tmp} -# Check if we have an optimization argument already -optimization_applied=0 +# optimization flag added as a command-line argument +optimization_flag="" -# Check if we have -std=c++X or --std=c++X already -stdcxx_applied=0 +# std standard flag added as a command-line argument +std_flag="" # Run nvcc a second time to generate dependencies if needed depfile_separate=0 @@ -99,6 +99,10 @@ depfile_target_arg="" # Option to remove duplicate libraries and object files remove_duplicate_link_files=0 +function warn_std_flag() { + echo "nvcc_wrapper - *warning* you have set multiple standard flags (-std=c++1* or --std=c++1*), only the last is used because nvcc can only accept a single std setting" +} + #echo "Arguments: $# $@" while [ $# -gt 0 ] @@ -130,12 +134,16 @@ do ;; # Ensure we only have one optimization flag because NVCC doesn't allow muliple -O*) - if [ $optimization_applied -eq 1 ]; then - echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the first is used because nvcc can only accept a single optimization setting." - else - shared_args="$shared_args $1" - optimization_applied=1 + if [ -n "$optimization_flag" ]; then + echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-O*), only the last is used because nvcc can only accept a single optimization setting." + shared_args=${shared_args/ $optimization_flag/} fi + if [ "$1" = "-O" ]; then + optimization_flag="-O2" + else + optimization_flag=$1 + fi + shared_args="$shared_args $optimization_flag" ;; #Handle shared args (valid for both nvcc and the host compiler) -D*) @@ -171,7 +179,7 @@ do shift ;; #Handle known nvcc args - --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|--resource-usage|-Xptxas*) + --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|--resource-usage|-Xptxas*|--fmad*) cuda_args="$cuda_args $1" ;; #Handle more known nvcc args @@ -179,21 +187,43 @@ do cuda_args="$cuda_args $1" ;; #Handle known nvcc args that have an argument - -rdc|-maxrregcount|--default-stream) + -rdc|-maxrregcount|--default-stream|-Xnvlink|--fmad) cuda_args="$cuda_args $1 $2" shift ;; -rdc=*|-maxrregcount*|--maxrregcount*) cuda_args="$cuda_args $1" ;; - #Handle c++11 - --std=c++11|-std=c++11|--std=c++14|-std=c++14|--std=c++1y|-std=c++1y|--std=c++17|-std=c++17|--std=c++1z|-std=c++1z) - if [ $stdcxx_applied -eq 1 ]; then - echo "nvcc_wrapper - *warning* you have set multiple optimization flags (-std=c++1* or --std=c++1*), only the first is used because nvcc can only accept a single std setting" - else - shared_args="$shared_args $1" - stdcxx_applied=1 + #Handle unsupported standard flags + --std=c++1y|-std=c++1y|--std=c++1z|-std=c++1z|--std=gnu++1y|-std=gnu++1y|--std=gnu++1z|-std=gnu++1z|--std=c++2a|-std=c++2a|--std=c++17|-std=c++17) + fallback_std_flag="-std=c++14" + # this is hopefully just occurring in a downstream project during CMake feature tests + # we really have no choice here but to accept the flag and change to an accepted C++ standard + echo "nvcc_wrapper does not accept standard flags $1 since partial standard flags and standards after C++14 are not supported. nvcc_wrapper will use $fallback_std_flag instead. It is undefined behavior to use this flag. This should only be occurring during CMake configuration." + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} fi + std_flag=$fallback_std_flag + shared_args="$shared_args $std_flag" + ;; + -std=gnu*) + corrected_std_flag=${1/gnu/c} + echo "nvcc_wrapper has been given GNU extension standard flag $1 - reverting flag to $corrected_std_flag" + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} + fi + std_flag=$corrected_std_flag + shared_args="$shared_args $std_flag" + ;; + --std=c++11|-std=c++11|--std=c++14|-std=c++14) + if [ -n "$std_flag" ]; then + warn_std_flag + shared_args=${shared_args/ $std_flag/} + fi + std_flag=$1 + shared_args="$shared_args $std_flag" ;; #strip of -std=c++98 due to nvcc warnings and Tribits will place both -std=c++11 and -std=c++98 @@ -308,16 +338,6 @@ do shift done -#Check if nvcc exists -if [ $host_only -ne 1 ]; then - var=$(which nvcc ) - if [ $? -gt 0 ]; then - echo "Could not find nvcc in PATH" - exit $? - fi -fi - - # Only print host compiler version if [ $get_host_version -eq 1 ]; then $host_compiler --version @@ -372,6 +392,9 @@ if [ $first_xcompiler_arg -eq 0 ]; then nvcc_command="$nvcc_command -Xcompiler $xcompiler_args" fi +#Replace all commas in xcompiler_args with a space for the host only command +xcompiler_args=${xcompiler_args//,/" "} + #Compose host only command host_command="$host_compiler $shared_args $host_only_args $compile_arg $output_arg $xcompiler_args $host_linker_args $shared_versioned_libraries_host" diff --git a/lib/kokkos/cmake/Dependencies.cmake b/lib/kokkos/cmake/Dependencies.cmake index 8c51eab4d7..c0be9f5641 100644 --- a/lib/kokkos/cmake/Dependencies.cmake +++ b/lib/kokkos/cmake/Dependencies.cmake @@ -6,5 +6,4 @@ TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( Core core PS REQUIRED Containers containers PS OPTIONAL Algorithms algorithms PS OPTIONAL - Example example EX OPTIONAL ) diff --git a/lib/kokkos/cmake/KokkosConfig.cmake.in b/lib/kokkos/cmake/KokkosConfig.cmake.in index fc099a494c..6f4607687e 100644 --- a/lib/kokkos/cmake/KokkosConfig.cmake.in +++ b/lib/kokkos/cmake/KokkosConfig.cmake.in @@ -1,18 +1,14 @@ -# - Config file for the Kokkos package -# It defines the following variables -# Kokkos_INCLUDE_DIRS - include directories for Kokkos -# Kokkos_LIBRARIES - libraries to link against - # Compute paths +@PACKAGE_INIT@ + +#Find dependencies +INCLUDE(CMakeFindDependencyMacro) + +#This needs to go above the KokkosTargets in case +#the Kokkos targets depend in some way on the TPL imports +@KOKKOS_TPL_EXPORTS@ + GET_FILENAME_COMPONENT(Kokkos_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) -SET(Kokkos_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") - -# Our library dependencies (contains definitions for IMPORTED targets) -IF(NOT TARGET kokkos AND NOT Kokkos_BINARY_DIR) - INCLUDE("${Kokkos_CMAKE_DIR}/KokkosTargets.cmake") -ENDIF() - -# These are IMPORTED targets created by KokkosTargets.cmake -SET(Kokkos_LIBRARY_DIRS @INSTALL_LIB_DIR@) -SET(Kokkos_LIBRARIES @Kokkos_LIBRARIES_NAMES@) -SET(Kokkos_TPL_LIBRARIES @KOKKOS_LIBS@) +INCLUDE("${Kokkos_CMAKE_DIR}/KokkosTargets.cmake") +INCLUDE("${Kokkos_CMAKE_DIR}/KokkosConfigCommon.cmake") +UNSET(Kokkos_CMAKE_DIR) diff --git a/lib/kokkos/cmake/KokkosConfigCommon.cmake.in b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in new file mode 100644 index 0000000000..8c663d01c1 --- /dev/null +++ b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in @@ -0,0 +1,90 @@ +SET(Kokkos_DEVICES @KOKKOS_ENABLED_DEVICES@) +SET(Kokkos_OPTIONS @KOKKOS_ENABLED_OPTIONS@) +SET(Kokkos_TPLS @KOKKOS_ENABLED_TPLS@) +SET(Kokkos_ARCH @KOKKOS_ENABLED_ARCH_LIST@) + +# These are needed by KokkosKernels +FOREACH(DEV ${Kokkos_DEVICES}) + SET(Kokkos_ENABLE_${DEV} ON) +ENDFOREACH() + +IF(NOT Kokkos_FIND_QUIETLY) + MESSAGE(STATUS "Enabled Kokkos devices: ${Kokkos_DEVICES}") +ENDIF() + +IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + #If we are building CUDA, we have tricked CMake because we declare a CXX project + #If the default C++ standard for a given compiler matches the requested + #standard, then CMake just omits the -std flag in later versions of CMake + #This breaks CUDA compilation (CUDA compiler can have a different default + #-std then the underlying host compiler by itself). Setting this variable + #forces CMake to always add the -std flag even if it thinks it doesn't need it + SET(CMAKE_CXX_STANDARD_DEFAULT 98 CACHE INTERNAL "" FORCE) +ENDIF() + +SET(KOKKOS_USE_CXX_EXTENSIONS @KOKKOS_USE_CXX_EXTENSIONS@) +IF (NOT DEFINED CMAKE_CXX_EXTENSIONS OR CMAKE_CXX_EXTENSIONS) + IF (NOT KOKKOS_USE_CXX_EXTENSIONS) + MESSAGE(WARNING "The installed Kokkos configuration does not support CXX extensions. Forcing -DCMAKE_CXX_EXTENSIONS=Off") + SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "" FORCE) + ENDIF() +ENDIF() + +include(FindPackageHandleStandardArgs) + +# This function makes sure that Kokkos was built with the requested backends +# and target architectures and generates a fatal error if it was not. +# +# kokkos_check( +# [DEVICES ...] # Set of backends (e.g. "OpenMP" and/or "Cuda") +# [ARCH ...] # Target architectures (e.g. "Power9" and/or "Volta70") +# [OPTIONS ...] # Optional settings (e.g. "PROFILING") +# [TPLS ...] # Third party libraries +# [RETURN_VALUE ] # Set a variable that indicates the result of the +# # check instead of a fatal error +# ) +function(kokkos_check) + set(ALLOWED_ARGS DEVICES ARCH OPTIONS TPLS) + cmake_parse_arguments(KOKKOS_CHECK "" "RETURN_VALUE" "${ALLOWED_ARGS}" ${ARGN}) + foreach(_arg ${KOKKOS_CHECK_UNPARSED_ARGUMENTS}) + message(SEND_ERROR "Argument '${_arg}' passed to kokkos_check() was not recognized") + endforeach() + # Get the list of keywords that were actually passed to the function. + set(REQUESTED_ARGS) + foreach(arg ${ALLOWED_ARGS}) + if(KOKKOS_CHECK_${arg}) + list(APPEND REQUESTED_ARGS ${arg}) + endif() + endforeach() + set(KOKKOS_CHECK_SUCCESS TRUE) + foreach(arg ${REQUESTED_ARGS}) + # Define variables named after the required arguments that are provided by + # the Kokkos install. + foreach(requested ${KOKKOS_CHECK_${arg}}) + foreach(provided ${Kokkos_${arg}}) + STRING(TOUPPER ${requested} REQUESTED_UC) + STRING(TOUPPER ${provided} PROVIDED_UC) + if(PROVIDED_UC STREQUAL REQUESTED_UC) + string(REPLACE ";" " " ${requested} "${KOKKOS_CHECK_${arg}}") + endif() + endforeach() + endforeach() + # Somewhat divert the CMake function below from its original purpose and + # use it to check that there are variables defined for all required + # arguments. Success or failure messages will be displayed but we are + # responsible for signaling failure and skip the build system generation. + if (KOKKOS_CHECK_RETURN_VALUE) + set(Kokkos_${arg}_FIND_QUIETLY ON) + endif() + find_package_handle_standard_args("Kokkos_${arg}" DEFAULT_MSG + ${KOKKOS_CHECK_${arg}}) + if(NOT Kokkos_${arg}_FOUND) + set(KOKKOS_CHECK_SUCCESS FALSE) + endif() + endforeach() + if(NOT KOKKOS_CHECK_SUCCESS AND NOT KOKKOS_CHECK_RETURN_VALUE) + message(FATAL_ERROR "Kokkos does NOT provide all backends and/or architectures requested") + else() + set(${KOKKOS_CHECK_RETURN_VALUE} ${KOKKOS_CHECK_SUCCESS} PARENT_SCOPE) + endif() +endfunction() diff --git a/lib/kokkos/cmake/KokkosCore_config.h.in b/lib/kokkos/cmake/KokkosCore_config.h.in new file mode 100644 index 0000000000..1d0b58fe02 --- /dev/null +++ b/lib/kokkos/cmake/KokkosCore_config.h.in @@ -0,0 +1,98 @@ + +#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H) +#error "Do not include KokkosCore_config.h directly; include Kokkos_Macros.hpp instead." +#else +#define KOKKOS_CORE_CONFIG_H +#endif + +// KOKKOS_VERSION % 100 is the patch level +// KOKKOS_VERSION / 100 % 100 is the minor version +// KOKKOS_VERSION / 10000 is the major version +#cmakedefine KOKKOS_VERSION @KOKKOS_VERSION@ + + +/* Execution Spaces */ +#cmakedefine KOKKOS_ENABLE_SERIAL +#cmakedefine KOKKOS_ENABLE_OPENMP +#cmakedefine KOKKOS_ENABLE_OPENMPTARGET +#cmakedefine KOKKOS_ENABLE_THREADS +#cmakedefine KOKKOS_ENABLE_CUDA +#cmakedefine KOKKOS_ENABLE_HIP +#cmakedefine KOKKOS_ENABLE_HPX +#cmakedefine KOKKOS_ENABLE_MEMKIND +#cmakedefine KOKKOS_ENABLE_LIBRT + +#ifndef __CUDA_ARCH__ +#cmakedefine KOKKOS_ENABLE_TM +#cmakedefine KOKKOS_USE_ISA_X86_64 +#cmakedefine KOKKOS_USE_ISA_KNC +#cmakedefine KOKKOS_USE_ISA_POWERPCLE +#cmakedefine KOKKOS_USE_ISA_POWERPCBE +#endif + +/* General Settings */ +#cmakedefine KOKKOS_ENABLE_CXX11 +#cmakedefine KOKKOS_ENABLE_CXX14 +#cmakedefine KOKKOS_ENABLE_CXX17 +#cmakedefine KOKKOS_ENABLE_CXX20 + +#cmakedefine KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE +#cmakedefine KOKKOS_ENABLE_CUDA_UVM +#cmakedefine KOKKOS_ENABLE_CUDA_LAMBDA +#cmakedefine KOKKOS_ENABLE_CUDA_CONSTEXPR +#cmakedefine KOKKOS_ENABLE_CUDA_LDG_INTRINSIC +#cmakedefine KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE +#cmakedefine KOKKOS_ENABLE_HPX_ASYNC_DISPATCH +#cmakedefine KOKKOS_ENABLE_DEBUG +#cmakedefine KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK +#cmakedefine KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK +#cmakedefine KOKKOS_ENABLE_COMPILER_WARNINGS +#cmakedefine KOKKOS_ENABLE_PROFILING +#cmakedefine KOKKOS_ENABLE_PROFILING_LOAD_PRINT +#cmakedefine KOKKOS_ENABLE_DEPRECATED_CODE +#cmakedefine KOKKOS_ENABLE_ETI +#cmakedefine KOKKOS_ENABLE_LARGE_MEM_TESTS +#cmakedefine KOKKOS_ENABLE_DUALVIEW_MODIFY_CHECK +#cmakedefine KOKKOS_ENABLE_COMPLEX_ALIGN +#cmakedefine KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION + +/* TPL Settings */ +#cmakedefine KOKKOS_ENABLE_HWLOC +#cmakedefine KOKKOS_USE_LIBRT +#cmakedefine KOKKOS_ENABLE_HWBSPACE + +#cmakedefine KOKKOS_IMPL_CUDA_CLANG_WORKAROUND + +#cmakedefine KOKKOS_COMPILER_CUDA_VERSION @KOKKOS_COMPILER_CUDA_VERSION@ + +#cmakedefine KOKKOS_ARCH_SSE42 +#cmakedefine KOKKOS_ARCH_ARMV80 +#cmakedefine KOKKOS_ARCH_ARMV8_THUNDERX +#cmakedefine KOKKOS_ARCH_ARMV81 +#cmakedefine KOKKOS_ARCH_ARMV8_THUNDERX2 +#cmakedefine KOKKOS_ARCH_AMD_AVX2 +#cmakedefine KOKKOS_ARCH_AVX +#cmakedefine KOKKOS_ARCH_AVX2 +#cmakedefine KOKKOS_ARCH_AVX512XEON +#cmakedefine KOKKOS_ARCH_KNC +#cmakedefine KOKKOS_ARCH_AVX512MIC +#cmakedefine KOKKOS_ARCH_POWER7 +#cmakedefine KOKKOS_ARCH_POWER8 +#cmakedefine KOKKOS_ARCH_POWER9 +#cmakedefine KOKKOS_ARCH_KEPLER +#cmakedefine KOKKOS_ARCH_KEPLER30 +#cmakedefine KOKKOS_ARCH_KEPLER32 +#cmakedefine KOKKOS_ARCH_KEPLER35 +#cmakedefine KOKKOS_ARCH_KEPLER37 +#cmakedefine KOKKOS_ARCH_MAXWELL +#cmakedefine KOKKOS_ARCH_MAXWELL50 +#cmakedefine KOKKOS_ARCH_MAXWELL52 +#cmakedefine KOKKOS_ARCH_MAXWELL53 +#cmakedefine KOKKOS_ARCH_PASCAL +#cmakedefine KOKKOS_ARCH_PASCAL60 +#cmakedefine KOKKOS_ARCH_PASCAL61 +#cmakedefine KOKKOS_ARCH_VOLTA +#cmakedefine KOKKOS_ARCH_VOLTA70 +#cmakedefine KOKKOS_ARCH_VOLTA72 +#cmakedefine KOKKOS_ARCH_TURING75 +#cmakedefine KOKKOS_ARCH_AMD_EPYC diff --git a/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in b/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in new file mode 100644 index 0000000000..626ef5a8eb --- /dev/null +++ b/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in @@ -0,0 +1,17 @@ +IF (NOT TARGET Kokkos::kokkos) + # Compute the installation prefix relative to this file. + get_filename_component(KOKKOS_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) + get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) + get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) + get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) + if(KOKKOS_IMPORT_PREFIX STREQUAL "/") + set(KOKKOS_IMPORT_PREFIX "") + endif() + add_library(Kokkos::kokkos INTERFACE IMPORTED) + set_target_properties(Kokkos::kokkos PROPERTIES + INTERFACE_LINK_LIBRARIES "@Kokkos_LIBRARIES@;@KOKKOS_LINK_OPTIONS@" + INTERFACE_COMPILE_FEATURES "@KOKKOS_CXX_STANDARD_FEATURE@" + INTERFACE_COMPILE_OPTIONS "@KOKKOS_ALL_COMPILE_OPTIONS@" + INTERFACE_INCLUDE_DIRECTORIES "${KOKKOS_IMPORT_PREFIX}/include" + ) +ENDIF() diff --git a/lib/kokkos/cmake/Makefile.generate_cmake_settings b/lib/kokkos/cmake/Makefile.generate_cmake_settings deleted file mode 100644 index da076b23db..0000000000 --- a/lib/kokkos/cmake/Makefile.generate_cmake_settings +++ /dev/null @@ -1,8 +0,0 @@ -ifndef KOKKOS_PATH - MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) - KOKKOS_PATH = $(subst Makefile,,$(MAKEFILE_PATH)).. -endif - -include $(KOKKOS_PATH)/Makefile.kokkos -include $(KOKKOS_PATH)/core/src/Makefile.generate_header_lists -include $(KOKKOS_PATH)/core/src/Makefile.generate_build_files diff --git a/lib/kokkos/cmake/Modules/FindHWLOC.cmake b/lib/kokkos/cmake/Modules/FindHWLOC.cmake deleted file mode 100644 index 60df8084d8..0000000000 --- a/lib/kokkos/cmake/Modules/FindHWLOC.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindHWLOC -# ---------- -# -# Try to find HWLOC, based on KOKKOS_HWLOC_DIR -# -# The following variables are defined: -# -# HWLOC_FOUND - System has HWLOC -# HWLOC_INCLUDE_DIR - HWLOC include directory -# HWLOC_LIBRARIES - Libraries needed to use HWLOC - -find_path(HWLOC_INCLUDE_DIR hwloc.h PATHS "${KOKKOS_HWLOC_DIR}/include") -find_library(HWLOC_LIBRARIES hwloc PATHS "${KOKKOS_HWLOC_DIR}/lib") - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(HWLOC DEFAULT_MSG - HWLOC_INCLUDE_DIR HWLOC_LIBRARIES) - -mark_as_advanced(HWLOC_INCLUDE_DIR HWLOC_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindMemkind.cmake b/lib/kokkos/cmake/Modules/FindMemkind.cmake deleted file mode 100644 index 245fb44c19..0000000000 --- a/lib/kokkos/cmake/Modules/FindMemkind.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindMemkind -# ---------- -# -# Try to find Memkind. -# -# The following variables are defined: -# -# MEMKIND_FOUND - System has Memkind -# MEMKIND_INCLUDE_DIR - Memkind include directory -# MEMKIND_LIBRARIES - Libraries needed to use Memkind - -find_path(MEMKIND_INCLUDE_DIR memkind.h) -find_library(MEMKIND_LIBRARIES memkind) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Memkind DEFAULT_MSG - MEMKIND_INCLUDE_DIR MEMKIND_LIBRARIES) - -mark_as_advanced(MEMKIND_INCLUDE_DIR MEMKIND_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindQthreads.cmake b/lib/kokkos/cmake/Modules/FindQthreads.cmake deleted file mode 100644 index a254b0e996..0000000000 --- a/lib/kokkos/cmake/Modules/FindQthreads.cmake +++ /dev/null @@ -1,20 +0,0 @@ -#.rst: -# FindQthreads -# ---------- -# -# Try to find Qthreads. -# -# The following variables are defined: -# -# QTHREADS_FOUND - System has Qthreads -# QTHREADS_INCLUDE_DIR - Qthreads include directory -# QTHREADS_LIBRARIES - Libraries needed to use Qthreads - -find_path(QTHREADS_INCLUDE_DIR qthread.h) -find_library(QTHREADS_LIBRARIES qthread) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Qthreads DEFAULT_MSG - QTHREADS_INCLUDE_DIR QTHREADS_LIBRARIES) - -mark_as_advanced(QTHREADS_INCLUDE_DIR QTHREADS_LIBRARIES) diff --git a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake new file mode 100644 index 0000000000..1b36ab819f --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake @@ -0,0 +1,17 @@ + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + # Note: "stubs" suffix allows CMake to find the dummy + # libcuda.so provided by the NVIDIA CUDA Toolkit for + # cross-compiling CUDA on a host without a GPU. + KOKKOS_FIND_IMPORTED(CUDA INTERFACE + LIBRARIES cudart cuda + LIBRARY_PATHS ENV LD_LIBRARY_PATH ENV CUDA_PATH /usr/local/cuda + LIBRARY_SUFFIXES lib lib64 lib/stubs lib64/stubs + ALLOW_SYSTEM_PATH_FALLBACK + ) +ELSE() + KOKKOS_CREATE_IMPORTED_TPL(CUDA INTERFACE + LINK_LIBRARIES cuda + ) +ENDIF() + diff --git a/lib/kokkos/cmake/Modules/FindTPLHPX.cmake b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake new file mode 100644 index 0000000000..c8b3bc4c9b --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake @@ -0,0 +1,15 @@ + +FIND_PACKAGE(HPX REQUIRED) +#as of right now, HPX doesn't export correctly +#so let's convert it to an interface target +KOKKOS_CREATE_IMPORTED_TPL(HPX INTERFACE + LINK_LIBRARIES ${HPX_LIBRARIES} + INCLUDES ${HPX_INCLUDE_DIRS} +) +#this is a bit funky since this is a CMake target +#but HPX doesn't export itself correctly +KOKKOS_EXPORT_CMAKE_TPL(HPX) + +#I would prefer all of this gets replaced with +#KOKKOS_IMPORT_CMAKE_TPL(HPX) + diff --git a/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake new file mode 100644 index 0000000000..cf763b7e5b --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(HWLOC HEADER hwloc.h LIBRARY hwloc) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake new file mode 100644 index 0000000000..5fc6a69303 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBDL HEADER dlfcn.h LIBRARY dl) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake new file mode 100644 index 0000000000..811db5851b --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBNUMA.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBNUMA HEADER numa.h LIBRARY numa) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake new file mode 100644 index 0000000000..e75da56b5b --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLLIBRT.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(LIBRT HEADER time.h LIBRARY rt) diff --git a/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake b/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake new file mode 100644 index 0000000000..20aaff2295 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLMEMKIND.cmake @@ -0,0 +1 @@ +KOKKOS_FIND_IMPORTED(MEMKIND HEADER memkind.h LIBRARY memkind) diff --git a/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake new file mode 100644 index 0000000000..1d154e29af --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake @@ -0,0 +1,20 @@ + +TRY_COMPILE(KOKKOS_HAS_PTHREAD_ARG + ${KOKKOS_TOP_BUILD_DIR}/tpl_tests + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/pthread.cpp + LINK_LIBRARIES -pthread + COMPILE_DEFINITIONS -pthread +) +# The test no longer requires C++11 +# if we did needed C++ standard support, then we should add option +# ${CMAKE_CXX${KOKKOS_CXX_STANDARD}_STANDARD_COMPILE_OPTION} + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(PTHREAD DEFAULT_MSG KOKKOS_HAS_PTHREAD_ARG) +#Only create the TPL if we succeed +IF (KOKKOS_HAS_PTHREAD_ARG) + KOKKOS_CREATE_IMPORTED_TPL(PTHREAD + INTERFACE #this is not a real library with a real location + COMPILE_OPTIONS -pthread + LINK_OPTIONS -pthread) +ENDIF() diff --git a/lib/kokkos/cmake/README.md b/lib/kokkos/cmake/README.md new file mode 100644 index 0000000000..6d0cc2daf1 --- /dev/null +++ b/lib/kokkos/cmake/README.md @@ -0,0 +1,332 @@ +![Kokkos](https://avatars2.githubusercontent.com/u/10199860?s=200&v=4) + +# Developing Kokkos + +This document contains a build system overview for developers with information on adding new CMake options that could influence +* Header configuration macros +* Optional features +* Third-partly libraries +* Compiler and linker flags +For build system details for users, refer to the [build instructions](../BUILD.md). + +## Build System + +Kokkos uses CMake to configure, build, and install. +Rather than being a completely straightforward use of modern CMake, +Kokkos has several extra complications, primarily due to: +* Kokkos must support linking to an installed version or in-tree builds as a subdirectory of a larger project. +* Kokkos must configure a special compiler `nvcc_wrapper` that allows `nvcc` to accept all C++ flags (which `nvcc` currently does not). +* Kokkos must work as a part of TriBITS, a CMake library providing a particular build idiom for Trilinos. +* Kokkos has many pre-existing users. We need to be careful about breaking previous versions or generating meaningful error messags if we do break backwards compatibility. + +If you are looking at the build system code wondering why certain decisions were made: we have had to balance many competing requirements and certain technical debt. Everything in the build system was done for a reason, trying to adhere as closely as possible to modern CMake best practices while meeting all pre-existing. customer requirements. + +### Modern CMake Philosophy + +Modern CMake relies on understanding the principle of *building* and *using* a code project. +What preprocessor, compiler, and linker flags do I need to *build* my project? +What flags does a downstream project that links to me need to *use* my project? +In CMake terms, flags that are only needed for building are `PRIVATE`. +Only Kokkos needs these flags, not a package that depends on Kokkos. +Flags that must be used in a downstream project are `PUBLIC`. +Kokkos must tell other projects to use them. + +In Kokkos, almost everything is a public flag since Kokkos is driven by headers and Kokkos is in charge of optimizing your code to achieve performance portability! +Include paths, C++ standard flags, architecture-specific optimizations, or OpenMP and CUDA flags are all examples of flags that Kokkos configures and adds to your project. + +Modern CMake now automatically propagates flags through the `target_link_libraries` command. +Suppose you have a library `stencil` that needs to build with Kokkos. +Consider the following CMake code: + +```` +find_package(Kokkos) +add_library(stencil stencil.cpp) +target_link_libraries(stencil Kokkos::kokkos) +```` + +This locates the Kokkos package, adds your library, and tells CMake to link Kokkos to your library. +All public build flags get added automatically through the `target_link_libraries` command. +There is nothing to do. You can be happily oblivious to how Kokkos was configured. +Everything should just work. + +As a Kokkos developer who wants to add new public compiler flags, how do you ensure that CMake does this properly? Modern CMake works through targets and properties. +Each target has a set of standard properties: +* `INTERFACE_COMPILE_OPTIONS` contains all the compiler options that Kokkos should add to downstream projects +* `INTERFACE_INCLUDE_DIRECTORIES` contains all the directories downstream projects must include from Kokkos +* `INTERFACE_COMPILE_DEFINITIONS` contains the list of preprocessor `-D` flags +* `INTERFACE_LINK_LIBRARIES` contains all the libraries downstream projects need to link +* `INTERFACE_COMPILE_FEATURES` essentially adds compiler flags, but with extra complications. Features names are specific to CMake. More later. + +CMake makes it easy to append to these properties using: +* `target_compile_options(kokkos PUBLIC -fmyflag)` +* `target_include_directories(kokkos PUBLIC mySpecialFolder)` +* `target_compile_definitions(kokkos PUBLIC -DmySpecialFlag=0)` +* `target_link_libraries(kokkos PUBLIC mySpecialLibrary)` +* `target_compile_features(kokkos PUBLIC mySpecialFeature)` +Note that all of these use `PUBLIC`! Almost every Kokkos flag is not private to Kokkos, but must also be used by downstream projects. + + +### Compiler Features and Compiler Options +Compiler options are flags like `-fopenmp` that do not need to be "resolved." +The flag is either on or off. +Compiler features are more fine-grained and require conflicting requests to be resolved. +Suppose I have +```` +add_library(A a.cpp) +target_compile_features(A PUBLIC cxx_std_11) +```` +then another target +```` +add_library(B b.cpp) +target_compile_features(B PUBLIC cxx_std_14) +target_link_libraries(A B) +```` +I have requested two different features. +CMake understands the requests and knows that `cxx_std_11` is a subset of `cxx_std_14`. +CMake then picks C++14 for library `B`. +CMake would not have been able to do feature resolution if we had directly done: +```` +target_compile_options(A PUBLIC -std=c++11) +```` + +### Adding Kokkos Options +After configuring for the first time, +CMake creates a cache of configure variables in `CMakeCache.txt`. +Reconfiguring in the folder "restarts" from those variables. +All flags passed as `-DKokkos_SOME_OPTION=X` to `cmake` become variables in the cache. +All Kokkos options begin with camel case `Kokkos_` followed by an upper case option name. + +CMake best practice is to avoid cache variables, if possible. +In essence, you want the minimal amount of state cached between configurations. +And never, ever have behavior influenced by multiple cache variables. +If you want to change the Kokkos configuration, have a single unique variable that needs to be changed. +Never require two cache variables to be changed. + +Kokkos provides a function `KOKKOS_OPTION` for defining valid cache-level variables, +proofreading them, and defining local project variables. +The most common variables are called `Kokkos_ENABLE_X`, +for which a helper function `KOKKOS_ENABLE_OPTION` is provided, e.g. +```` +KOKKOS_ENABLE_OPTION(TESTS OFF "Whether to build tests") +```` +The function checks if `-DKokkos_ENABLE_TESTS` was given, +whether it was given with the wrong case, e.g. `-DKokkos_Enable_Tests`, +and then defines a regular (non-cache) variable `KOKKOS_ENABLE_TESTS` to `ON` or `OFF` +depending on the given default and whether the option was specified. + +### Defining Kokkos Config Macros + +Sometimes you may want to add `#define Kokkos_X` macros to the config header. +This is straightforward with CMake. +Suppose you want to define an optional macro `KOKKOS_SUPER_SCIENCE`. +Simply go into `KokkosCore_config.h.in` and add +```` +#cmakedefine KOKKOS_SUPER_SCIENCE +```` +I can either add +```` +KOKKOS_OPTION(SUPER_SCIENCE ON "Whether to do some super science") +```` +to directly set the variable as a command-line `-D` option. +Alternatively, based on other logic, I could add to a `CMakeLists.txt` +```` +SET(KOKKOS_SUPER_SCIENCE ON) +```` +If not set as a command-line option (cache variable), you must make sure the variable is visible in the top-level scope. +If set in a function, you would need: +```` +SET(KOKKOS_SUPER_SCIENCE ON PARENT_SCOPE) +```` + +### Third-Party Libraries +In much the same way that compiler flags transitively propagate to dependent projects, +modern CMake allows us to propagate dependent libraries. +If Kokkos depends on, e.g. `hwloc` the downstream project will also need to link `hwloc`. +There are three stages in adding a new third-party library (TPL): +* Finding: find the desired library on the system and verify the installation is correct +* Importing: create a CMake target, if necessary, that is compatible with `target_link_libraries`. This is mostly relevant for TPLs not installed with CMake. +* Exporting: make the desired library visible to downstream projects + +TPLs are somewhat complicated by whether the library was installed with CMake or some other build system. +If CMake, our lives are greatly simplified. We simply use `find_package` to locate the installed CMake project then call `target_link_libraries(kokkoscore PUBLIC/PRIVATE TPL)`. For libaries not installed with CMake, the process is a bit more complex. +It is up to the Kokkos developers to "convert" the library into a CMake target as if it had been installed as a valid modern CMake target with properties. +There are helper functions for simplifying the process of importing TPLs in Kokkos, but we walk through the process in detail to clearly illustrate the steps involved. + +#### TPL Search Order + +There are several options for where CMake could try to find a TPL. +If there are multiple installations of the same TPL on the system, +the search order is critical for making sure the correct TPL is found. +There are 3 possibilities that could be used: + +1. Default system paths like /usr +1. User-provided paths through options `_ROOT` and `Kokkos__DIR` +1. Additional paths not in the CMake default list or provided by the user that Kokkos decides to add. For example, Kokkos may query `nvcc` or `LD_LIBRARY_PATH` for where to find CUDA libraries. + +The following is the search order that Kokkos follows. Note: This differs from the default search order used by CMake `find_library` and `find_header`. CMake prefers default system paths over user-provided paths. +For Kokkos (and package managers in general), it is better to prefer user-provided paths since this usually indicates a specific version we want. + +1. `_ROOT` command line option +1. `_ROOT` environment variable +1. `Kokkos__DIR` command line option +1. Paths added by Kokkos CMake logic +1. Default system paths (if allowed) + +Default system paths are allowed in two cases. First, none of the other options are given so the only place to look is system paths. Second, if explicitly given permission, configure will look in system paths. +The rationale for this logic is that if you specify a custom location, you usually *only* want to look in that location. +If you do not find the TPL where you expect it, you should error out rather than grab another random match. + + +#### Finding TPLs + +If finding a TPL that is not a modern CMake project, refer to the `FindHWLOC.cmake` file in `cmake/Modules` for an example. +You will usually need to verify expected headers with `find_path` +```` +find_path(TPL_INCLUDE_DIR mytpl.h PATHS "${KOKKOS_MYTPL_DIR}/include") +```` +This insures that the library header is in the expected include directory and defines the variable `TPL_INCLUDE_DIR` with a valid path if successful. +Similarly, you can verify a library +```` +find_library(TPL_LIBRARY mytpl PATHS "${KOKKOS_MYTPL_DIR/lib") +```` +that then defines the variable `TPL_LIBRARY` with a valid path if successful. +CMake provides a utility for checking if the `find_path` and `find_library` calls were successful that emulates the behavior of `find_package` for a CMake target. +```` +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MYTPL DEFAULT_MSG + MYTPL_INCLUDE_DIR MYTPL_LIBRARY) +```` +If the find failed, CMake will print standard error messages explaining the failure. + +#### Importing TPLs + +The installed TPL must be adapted into a CMake target. +CMake allows libraries to be added that are built externally as follows: +```` +add_library(Kokkos::mytpl UNKNOWN IMPORTED) +```` +Importantly, we use a `Kokkos::` namespace to avoid name conflicts and identify this specifically as the version imported by Kokkos. +Because we are importing a non-CMake target, we must populate all the target properties that would have been automatically populated for a CMake target. +```` +set_target_properties(Kokkos::mytpl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MYTPL_INCLUDE_DIR}" + IMPORTED_LOCATION "${MYTPL_LIBRARY}" +) +```` + +#### Exporting TPLs + +Kokkos may now depend on the target `Kokkos::mytpl` as a `PUBLIC` library (remember building and using). +This means that downstream projects must also know about `Kokkos::myptl` - so Kokkos must export them. +In the `KokkosConfig.cmake.in` file, we need to add code like the following: +```` +set(MYTPL_LIBRARY @MYTPL_LIBRARY@) +set(MYTPL_INCLUDE_DIR @MYTPL_INCLUDE_DIR@) +add_library(Kokkos::mytpl UNKNOWN IMPORTED) +set_target_properties(Kokkos::mytpl PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MYTPL_INCLUDE_DIR}" + IMPORTED_LOCATION "${MYTPL_LIBRARY}" +) +```` +If this looks familiar, that's because it is exactly the same code as above for importing the TPL. +Exporting a TPL really just means importing the TPL when Kokkos is loaded by an external project. +We will describe helper functions that simplify this process. + +#### Interface TPLs + +If a TPL is just a library and set of headers, we can make a simple `IMPORTED` target. +However, a TPL is actually completely flexible and need not be limited to just headers and libraries. +TPLs can configure compiler flags, linker flags, or multiple different libraries. +For this, we use a special type of CMake target: `INTERFACE` libraries. +These libraries don't build anything. +They simply populate properties that will configure flags for dependent targets. +We consider the example: +```` +add_library(PTHREAD INTERFACE) +target_compile_options(PTHREAD PUBLIC -pthread) +```` +Kokkos uses the compiler flag `-pthread` to define compiler macros for re-entrant functions rather than treating it simply as a library with header `pthread.h` and library `-lpthread`. +Any property can be configured, e.g. +```` +target_link_libraries(MYTPL ...) +```` +In contrast to imported TPLs which require direct modification of `KokkosConfig.cmake.in`, +we can use CMake's built-in export functions: +```` +INSTALL( + TARGETS MYTPL + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) +```` +These interface targets will be automatically populated in the config file. + +#### Linking the TPL +After finishing the import process, it still remains to link the imported target as needed. +For example, +```` +target_link_libraries(kokkoscore PUBLIC Kokkos::HWLOC) +```` +The complexity of which includes, options, and libraries the TPL requires +should be encapsulated in the CMake target. + +#### TPL Helper Functions +##### KOKKOS_IMPORT_TPL +This function can be invoked as, e.g. +```` +KOKKOS_IMPORT_TPL(HWLOC) +```` +This function checks if the TPL was enabled by a `-DKokkos_ENABLE_HWLOC=On` flag. +If so, it calls `find_package(TPLHWLOC)`. +This invokes the file `FindTPLHWLOC.cmake` which should be contained in the `cmake/Modules` folder. +If successful, another function `KOKKOS_EXPORT_CMAKE_TPL` gets invoked. +This automatically adds all the necessary import commands to `KokkosConfig.cmake`. + +##### KOKKOS_FIND_IMPORTED +Inside a `FindTPLX.cmake` file, the simplest way to import a library is to call, e.g. +```` +KOKKOS_FIND_IMPORTED(HWLOC LIBRARY hwloc HEADER hwloc.h) +```` +This finds the location of the library and header and creates an imported target `Kokkos::HWLOC` +that can be linked against. +The library/header find can be guided with `-DHWLOC_ROOT=` or `-DKokkos_HWLOC_DIR=` during CMake configure. +These both specify the install prefix. + +##### KOKKOS_LINK_TPL +This function checks if the TPL has been enabled. +If so, it links a given library against the imported (or interface) TPL target. + +##### KOKKOS_CREATE_IMPORTED_TPL +This helper function is best understood by reading the actual code. +This function takes arguments specifying the properties and creates the actual TPL target. +The most important thing to understand for this function is whether you call this function with the optional `INTERFACE` keyword. +This tells the project to either create the target as an imported target or interface target, as discussed above. + +##### KOKKOS_EXPORT_CMAKE_TPL +Even if the TPL just loads a valid CMake target, we still must "export" it into the config file. +When Kokkos is loaded by a downstream project, this TPL must be loaded. +Calling this function simply appends text recording the location where the TPL was found +and adding a `find_dependency(...)` call that will reload the CMake target. + +### The Great TriBITS Compromise + +TriBITS was a masterpiece of CMake version 2 before the modern CMake idioms of building and using. +TriBITS greatly limited verbosity of CMake files, handled complicated dependency trees between packages, and handled automatically setting up include and linker paths for dependent libraries. + +Kokkos is now used by numerous projects that don't (and won't) depend on TriBITS for their build systems. +Kokkos has to work outside of TriBITS and provide a standard CMake 3+ build system. +At the same time, Kokkos is used by numerous projects that depend on TriBITS and don't (and won't) switch to a standard CMake 3+ build system. + +Instead of calling functions `TRIBITS_X(...)`, the CMake calls wrapper functions `KOKKOS_X(...)`. +If TriBITS is available (as in Trilinos), `KOKKOS_X` will just be a thin wrapper around `TRIBITS_X`. +If TriBITS is not available, Kokkos maps `KOKKOS_X` calls to native CMake that complies with CMake 3 idioms. +For the time being, this seems the most sensible way to handle the competing requirements of a standalone modern CMake and TriBITS build system. + +##### [LICENSE](https://github.com/kokkos/kokkos/blob/devel/LICENSE) + +[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Under the terms of Contract DE-NA0003525 with NTESS, +the U.S. Government retains certain rights in this software. diff --git a/lib/kokkos/cmake/compile_tests/clang_omp.cpp b/lib/kokkos/cmake/compile_tests/clang_omp.cpp new file mode 100644 index 0000000000..60a5c52282 --- /dev/null +++ b/lib/kokkos/cmake/compile_tests/clang_omp.cpp @@ -0,0 +1,9 @@ +#include + +int main(int, char**) { + int thr = omp_get_num_threads(); + if (thr > 0) + return thr; + else + return 0; +} diff --git a/lib/kokkos/cmake/compile_tests/pthread.cpp b/lib/kokkos/cmake/compile_tests/pthread.cpp new file mode 100644 index 0000000000..92310da029 --- /dev/null +++ b/lib/kokkos/cmake/compile_tests/pthread.cpp @@ -0,0 +1,14 @@ +#include + +void* kokkos_test(void* args) { return args; } + +int main(void) { + pthread_t thread; + /* Use NULL to avoid C++11. Some compilers + do not have C++11 by default. Forcing C++11 + in the compile tests can be done, but is unnecessary + */ + pthread_create(&thread, NULL, kokkos_test, NULL); + pthread_join(thread, NULL); + return 0; +} diff --git a/lib/kokkos/cmake/cray.cmake b/lib/kokkos/cmake/cray.cmake new file mode 100644 index 0000000000..08912f5130 --- /dev/null +++ b/lib/kokkos/cmake/cray.cmake @@ -0,0 +1,9 @@ + + +function(kokkos_set_cray_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + SET(KOKKOS_CXX_STANDARD_FLAG "-hstd=c++${FULL_LC_STANDARD}", PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "-hstd=c++${INT_LC_STANDARD}" PARENT_SCOPE) +endfunction() + diff --git a/lib/kokkos/cmake/deps/CUDA.cmake b/lib/kokkos/cmake/deps/CUDA.cmake index 801c20067b..4876bca259 100644 --- a/lib/kokkos/cmake/deps/CUDA.cmake +++ b/lib/kokkos/cmake/deps/CUDA.cmake @@ -73,7 +73,7 @@ IF(NOT _CUDA_FAILURE) GLOBAL_SET(TPL_CUDA_LIBRARY_DIRS) GLOBAL_SET(TPL_CUDA_INCLUDE_DIRS ${CUDA_TOOLKIT_INCLUDE}) GLOBAL_SET(TPL_CUDA_LIBRARIES ${CUDA_CUDART_LIBRARY} ${CUDA_cublas_LIBRARY} ${CUDA_cufft_LIBRARY}) - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) ELSE() SET(TPL_ENABLE_CUDA OFF) ENDIF() diff --git a/lib/kokkos/cmake/deps/CUSPARSE.cmake b/lib/kokkos/cmake/deps/CUSPARSE.cmake index 6f26d857c0..b2420d1168 100644 --- a/lib/kokkos/cmake/deps/CUSPARSE.cmake +++ b/lib/kokkos/cmake/deps/CUSPARSE.cmake @@ -59,6 +59,6 @@ # GLOBAL_SET(TPL_CUSPARSE_LIBRARY_DIRS) # GLOBAL_SET(TPL_CUSPARSE_INCLUDE_DIRS ${TPL_CUDA_INCLUDE_DIRS}) # GLOBAL_SET(TPL_CUSPARSE_LIBRARIES ${CUDA_cusparse_LIBRARY}) -# TIBITS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) +# KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(CUSPARSE) #ENDIF() diff --git a/lib/kokkos/cmake/deps/HWLOC.cmake b/lib/kokkos/cmake/deps/HWLOC.cmake index 275abd3a5d..ed89c8c1e5 100644 --- a/lib/kokkos/cmake/deps/HWLOC.cmake +++ b/lib/kokkos/cmake/deps/HWLOC.cmake @@ -64,7 +64,7 @@ # Version: 1.3 # -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC +KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc" ) diff --git a/lib/kokkos/cmake/deps/Pthread.cmake b/lib/kokkos/cmake/deps/Pthread.cmake index 46d0a939ca..5f835fc300 100644 --- a/lib/kokkos/cmake/deps/Pthread.cmake +++ b/lib/kokkos/cmake/deps/Pthread.cmake @@ -74,9 +74,9 @@ IF(USE_THREADS) SET(TPL_Pthread_INCLUDE_DIRS "") SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") SET(TPL_Pthread_LIBRARY_DIRS "") - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(Pthread) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(Pthread) ELSE() - TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread + KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread ) diff --git a/lib/kokkos/cmake/deps/QTHREADS.cmake b/lib/kokkos/cmake/deps/QTHREADS.cmake deleted file mode 100644 index c312f2590b..0000000000 --- a/lib/kokkos/cmake/deps/QTHREADS.cmake +++ /dev/null @@ -1,69 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trilinos: An Object-Oriented Solver Framework -# Copyright (2001) Sandia Corporation -# -# -# Copyright (2001) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000, there is a non-exclusive license for use of this -# work by or on behalf of the U.S. Government. Export of this program -# may require a license from the United States Government. -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# NOTICE: The United States Government is granted for itself and others -# acting on its behalf a paid-up, nonexclusive, irrevocable worldwide -# license in this data to reproduce, prepare derivative works, and -# perform publicly and display publicly. Beginning five (5) years from -# July 25, 2001, the United States Government is granted for itself and -# others acting on its behalf a paid-up, nonexclusive, irrevocable -# worldwide license in this data to reproduce, prepare derivative works, -# distribute copies to the public, perform publicly and display -# publicly, and to permit others to do so. -# -# NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT -# OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES -# ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR -# RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY -# INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS -# THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. -# -# ************************************************************************ -# @HEADER - - -#----------------------------------------------------------------------------- -# Hardware locality detection and control library. -# -# Acquisition information: -# Date checked: July 2014 -# Checked by: H. Carter Edwards -# Source: https://code.google.com/p/qthreads -# - -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( QTHREADS - REQUIRED_HEADERS qthread.h - REQUIRED_LIBS_NAMES "qthread" - ) diff --git a/lib/kokkos/cmake/fake_tribits.cmake b/lib/kokkos/cmake/fake_tribits.cmake new file mode 100644 index 0000000000..acee4a249d --- /dev/null +++ b/lib/kokkos/cmake/fake_tribits.cmake @@ -0,0 +1,331 @@ +#These are tribits wrappers used by all projects in the Kokkos ecosystem + +INCLUDE(CMakeParseArguments) +INCLUDE(CTest) + +cmake_policy(SET CMP0054 NEW) + +FUNCTION(ASSERT_DEFINED VARS) + FOREACH(VAR ${VARS}) + IF(NOT DEFINED ${VAR}) + MESSAGE(SEND_ERROR "Error, the variable ${VAR} is not defined!") + ENDIF() + ENDFOREACH() +ENDFUNCTION() + +MACRO(KOKKOS_ADD_OPTION_AND_DEFINE USER_OPTION_NAME MACRO_DEFINE_NAME DOCSTRING DEFAULT_VALUE ) +SET( ${USER_OPTION_NAME} "${DEFAULT_VALUE}" CACHE BOOL "${DOCSTRING}" ) +IF(NOT ${MACRO_DEFINE_NAME} STREQUAL "") + IF(${USER_OPTION_NAME}) + GLOBAL_SET(${MACRO_DEFINE_NAME} ON) + ELSE() + GLOBAL_SET(${MACRO_DEFINE_NAME} OFF) + ENDIF() +ENDIF() +ENDMACRO() + +MACRO(GLOBAL_OVERWRITE VARNAME VALUE TYPE) + SET(${VARNAME} ${VALUE} CACHE ${TYPE} "" FORCE) +ENDMACRO() + +IF (NOT KOKKOS_HAS_TRILINOS) +MACRO(APPEND_GLOB VAR) + FILE(GLOB LOCAL_TMP_VAR ${ARGN}) + LIST(APPEND ${VAR} ${LOCAL_TMP_VAR}) +ENDMACRO() + +MACRO(GLOBAL_SET VARNAME) + SET(${VARNAME} ${ARGN} CACHE INTERNAL "" FORCE) +ENDMACRO() + +FUNCTION(VERIFY_EMPTY CONTEXT) +if(${ARGN}) +MESSAGE(FATAL_ERROR "Kokkos does not support all of Tribits. Unhandled arguments in ${CONTEXT}:\n${ARGN}") +endif() +ENDFUNCTION() + +MACRO(PREPEND_GLOBAL_SET VARNAME) + ASSERT_DEFINED(${VARNAME}) + GLOBAL_SET(${VARNAME} ${ARGN} ${${VARNAME}}) +ENDMACRO() + +MACRO(PREPEND_TARGET_SET VARNAME TARGET_NAME TYPE) + IF(TYPE STREQUAL "REQUIRED") + SET(REQUIRED TRUE) + ELSE() + SET(REQUIRED FALSE) + ENDIF() + IF(TARGET ${TARGET_NAME}) + PREPEND_GLOBAL_SET(${VARNAME} ${TARGET_NAME}) + ELSE() + IF(REQUIRED) + MESSAGE(FATAL_ERROR "Missing dependency ${TARGET_NAME}") + ENDIF() + ENDIF() +ENDMACRO() +endif() + + +FUNCTION(KOKKOS_CONFIGURE_FILE PACKAGE_NAME_CONFIG_FILE) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME_CONFIG_FILE}) + else() + # Configure the file + CONFIGURE_FILE( + ${PACKAGE_SOURCE_DIR}/cmake/${PACKAGE_NAME_CONFIG_FILE}.in + ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME_CONFIG_FILE} + ) + endif() +ENDFUNCTION() + +MACRO(ADD_INTERFACE_LIBRARY LIB_NAME) + FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") + ADD_LIBRARY(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + SET_TARGET_PROPERTIES(${LIB_NAME} PROPERTIES INTERFACE TRUE) +ENDMACRO() + +FUNCTION(KOKKOS_ADD_TEST) + if (KOKKOS_HAS_TRILINOS) + CMAKE_PARSE_ARGUMENTS(TEST + "" + "EXE;NAME" + "" + ${ARGN}) + IF(TEST_EXE) + SET(EXE_ROOT ${TEST_EXE}) + ELSE() + SET(EXE_ROOT ${TEST_NAME}) + ENDIF() + + TRIBITS_ADD_TEST( + ${EXE_ROOT} + NAME ${TEST_NAME} + COMM serial mpi + NUM_MPI_PROCS 1 + ${TEST_UNPARSED_ARGUMENTS} + ) + else() + CMAKE_PARSE_ARGUMENTS(TEST + "WILL_FAIL" + "FAIL_REGULAR_EXPRESSION;PASS_REGULAR_EXPRESSION;EXE;NAME" + "CATEGORIES;CMD_ARGS" + ${ARGN}) + # To match Tribits, we should always be receiving + # the root names of exes/libs + IF(TEST_EXE) + SET(EXE_ROOT ${TEST_EXE}) + ELSE() + SET(EXE_ROOT ${TEST_NAME}) + ENDIF() + # Prepend package name to the test name + # These should be the full target name + SET(TEST_NAME ${PACKAGE_NAME}_${TEST_NAME}) + SET(EXE ${PACKAGE_NAME}_${EXE_ROOT}) + IF(WIN32) + ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} ${TEST_CMD_ARGS}) + ELSE() + ADD_TEST(NAME ${TEST_NAME} COMMAND ${EXE} ${TEST_CMD_ARGS}) + ENDIF() + IF(TEST_WILL_FAIL) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) + ENDIF() + IF(TEST_FAIL_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) + ENDIF() + IF(TEST_PASS_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) + ENDIF() + VERIFY_EMPTY(KOKKOS_ADD_TEST ${TEST_UNPARSED_ARGUMENTS}) + endif() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_ADVANCED_TEST) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_ADVANCED_TEST(${ARGN}) + else() + # TODO Write this + endif() +ENDFUNCTION() + +MACRO(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) + ADD_INTERFACE_LIBRARY(TPL_LIB_${TPL_NAME}) + TARGET_LINK_LIBRARIES(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) + TARGET_INCLUDE_DIRECTORIES(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) +ENDMACRO() + +FUNCTION(KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES(${TPL_NAME} ${ARGN}) + else() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "REQUIRED_HEADERS;REQUIRED_LIBS_NAMES" + ${ARGN}) + + SET(_${TPL_NAME}_ENABLE_SUCCESS TRUE) + IF (PARSE_REQUIRED_LIBS_NAMES) + FIND_LIBRARY(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) + IF(NOT TPL_${TPL_NAME}_LIBRARIES) + SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + ENDIF() + ENDIF() + IF (PARSE_REQUIRED_HEADERS) + FIND_PATH(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) + IF(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) + SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + ENDIF() + ENDIF() + IF (_${TPL_NAME}_ENABLE_SUCCESS) + KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(${TPL_NAME}) + ENDIF() + VERIFY_EMPTY(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) + endif() +ENDFUNCTION() + +MACRO(KOKKOS_TARGET_COMPILE_OPTIONS TARGET) +if(KOKKOS_HAS_TRILINOS) + TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) +else() + TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) +endif() +ENDMACRO() + + +MACRO(KOKKOS_EXCLUDE_AUTOTOOLS_FILES) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_EXCLUDE_AUTOTOOLS_FILES() + else() + #do nothing + endif() +ENDMACRO() + +FUNCTION(KOKKOS_LIB_TYPE LIB RET) +GET_TARGET_PROPERTY(PROP ${LIB} TYPE) +IF (${PROP} STREQUAL "INTERFACE_LIBRARY") + SET(${RET} "INTERFACE" PARENT_SCOPE) +ELSE() + SET(${RET} "PUBLIC" PARENT_SCOPE) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_TARGET_INCLUDE_DIRECTORIES TARGET) +IF(KOKKOS_HAS_TRILINOS) + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + #don't trust tribits to do this correctly - but need to add package name + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) +ELSEIF(TARGET ${TARGET}) + #the target actually exists - this means we are doing separate libs + #or this a test library + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) +ELSE() + GET_PROPERTY(LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + IF (${TARGET} IN_LIST LIBS) + SET_PROPERTY(GLOBAL APPEND PROPERTY KOKKOS_LIBRARY_INCLUDES ${ARGN}) + ELSE() + MESSAGE(FATAL_ERROR "Trying to set include directories on unknown target ${TARGET}") + ENDIF() +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LINK_INTERNAL_LIBRARY TARGET DEPLIB) +IF(KOKKOS_HAS_TRILINOS) + #do nothing +ELSE() + SET(options INTERFACE) + SET(oneValueArgs) + SET(multiValueArgs) + CMAKE_PARSE_ARGUMENTS(PARSE + "INTERFACE" + "" + "" + ${ARGN}) + SET(LINK_TYPE) + IF(PARSE_INTERFACE) + SET(LINK_TYPE INTERFACE) + ELSE() + SET(LINK_TYPE PUBLIC) + ENDIF() + TARGET_LINK_LIBRARIES(${TARGET} ${LINK_TYPE} ${DEPLIB}) + VERIFY_EMPTY(KOKKOS_LINK_INTERNAL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_TEST_LIBRARY NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${NAME} ${ARGN} TESTONLY + ADDED_LIB_TARGET_NAME_OUT ${NAME} + ) +ELSE() + SET(oneValueArgs) + SET(multiValueArgs HEADERS SOURCES) + + CMAKE_PARSE_ARGUMENTS(PARSE + "STATIC;SHARED" + "" + "HEADERS;SOURCES" + ${ARGN}) + + IF(PARSE_HEADERS) + LIST(REMOVE_DUPLICATES PARSE_HEADERS) + ENDIF() + IF(PARSE_SOURCES) + LIST(REMOVE_DUPLICATES PARSE_SOURCES) + ENDIF() + ADD_LIBRARY(${NAME} ${PARSE_SOURCES}) +ENDIF() +ENDFUNCTION() + + +FUNCTION(KOKKOS_TARGET_COMPILE_DEFINITIONS) + IF (KOKKOS_HAS_TRILINOS) + TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) + ELSE() + TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_INCLUDE_DIRECTORIES) +IF(KOKKOS_HAS_TRILINOS) + TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) +ELSE() + CMAKE_PARSE_ARGUMENTS( + INC + "REQUIRED_DURING_INSTALLATION_TESTING" + "" + "" + ${ARGN} + ) + INCLUDE_DIRECTORIES(${INC_UNPARSED_ARGUMENTS}) +ENDIF() +ENDFUNCTION() + + +MACRO(KOKKOS_ADD_COMPILE_OPTIONS) +ADD_COMPILE_OPTIONS(${ARGN}) +ENDMACRO() + +MACRO(PRINTALL match) +get_cmake_property(_variableNames VARIABLES) +list (SORT _variableNames) +foreach (_variableName ${_variableNames}) + if("${_variableName}" MATCHES "${match}") + message(STATUS "${_variableName}=${${_variableName}}") + endif() +endforeach() +ENDMACRO() + +MACRO(SET_GLOBAL_REPLACE SUBSTR VARNAME) + STRING(REPLACE ${SUBSTR} ${${VARNAME}} TEMP) + GLOBAL_SET(${VARNAME} ${TEMP}) +ENDMACRO() + +FUNCTION(GLOBAL_APPEND VARNAME) + #We make this a function since we are setting variables + #and want to use scope to avoid overwriting local variables + SET(TEMP ${${VARNAME}}) + LIST(APPEND TEMP ${ARGN}) + GLOBAL_SET(${VARNAME} ${TEMP}) +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/gnu.cmake b/lib/kokkos/cmake/gnu.cmake new file mode 100644 index 0000000000..aa11fe87b1 --- /dev/null +++ b/lib/kokkos/cmake/gnu.cmake @@ -0,0 +1,23 @@ + +FUNCTION(kokkos_set_gnu_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + # The following three blocks of code were copied from + # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. + IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + SET(_std -Qstd) + SET(_ext c++) + ELSE() + SET(_std -std) + SET(_ext gnu++) + ENDIF() + + IF (CMAKE_CXX_EXTENSIONS) + SET(KOKKOS_CXX_STANDARD_FLAG "-std=gnu++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=gnu++${INT_LC_STANDARD}" PARENT_SCOPE) + ELSE() + SET(KOKKOS_CXX_STANDARD_FLAG "-std=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=c++${INT_LC_STANDARD}" PARENT_SCOPE) + ENDIF() +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/intel.cmake b/lib/kokkos/cmake/intel.cmake new file mode 100644 index 0000000000..f36f01d8ca --- /dev/null +++ b/lib/kokkos/cmake/intel.cmake @@ -0,0 +1,30 @@ + +FUNCTION(kokkos_set_intel_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + # The following three blocks of code were copied from + # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. + IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + SET(_std -Qstd) + SET(_ext c++) + ELSE() + SET(_std -std) + SET(_ext gnu++) + ENDIF() + + IF(NOT KOKKOS_CXX_STANDARD STREQUAL 11 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) + #There is no gnu++14 value supported; figure out what to do. + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "${_std}=c++${INT_LC_STANDARD}" PARENT_SCOPE) + ELSEIF(KOKKOS_CXX_STANDARD STREQUAL 11 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0) + IF (CMAKE_CXX_EXTENSIONS) + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=${_ext}c++11" PARENT_SCOPE) + ELSE() + SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++11" PARENT_SCOPE) + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "Intel compiler version too low - need 13.0 for C++11 and 15.0 for C++14") + ENDIF() + +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/kokkos_arch.cmake b/lib/kokkos/cmake/kokkos_arch.cmake new file mode 100644 index 0000000000..d73a353981 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_arch.cmake @@ -0,0 +1,456 @@ + +FUNCTION(KOKKOS_ARCH_OPTION SUFFIX DEV_TYPE DESCRIPTION) + #all optimizations off by default + KOKKOS_OPTION(ARCH_${SUFFIX} OFF BOOL "Optimize for ${DESCRIPTION} (${DEV_TYPE})") + IF (KOKKOS_ARCH_${SUFFIX}) + LIST(APPEND KOKKOS_ENABLED_ARCH_LIST ${SUFFIX}) + SET(KOKKOS_ENABLED_ARCH_LIST ${KOKKOS_ENABLED_ARCH_LIST} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ARCH_${SUFFIX} ${KOKKOS_ARCH_${SUFFIX}} PARENT_SCOPE) +ENDFUNCTION() + + +# Make sure devices and compiler ID are done +KOKKOS_CFG_DEPENDS(ARCH COMPILER_ID) +KOKKOS_CFG_DEPENDS(ARCH DEVICES) +KOKKOS_CFG_DEPENDS(ARCH OPTIONS) + + +#------------------------------------------------------------------------------- +# List of possible host architectures. +#------------------------------------------------------------------------------- +SET(KOKKOS_ARCH_LIST) + + +KOKKOS_DEPRECATED_LIST(ARCH ARCH) +KOKKOS_ARCH_OPTION(AMDAVX HOST "AMD chip") +KOKKOS_ARCH_OPTION(ARMV80 HOST "ARMv8.0 Compatible CPU") +KOKKOS_ARCH_OPTION(ARMV81 HOST "ARMv8.1 Compatible CPU") +KOKKOS_ARCH_OPTION(ARMV8_THUNDERX HOST "ARMv8 Cavium ThunderX CPU") +KOKKOS_ARCH_OPTION(ARMV8_THUNDERX2 HOST "ARMv8 Cavium ThunderX2 CPU") +KOKKOS_ARCH_OPTION(WSM HOST "Intel Westmere CPU") +KOKKOS_ARCH_OPTION(SNB HOST "Intel Sandy/Ivy Bridge CPUs") +KOKKOS_ARCH_OPTION(HSW HOST "Intel Haswell CPUs") +KOKKOS_ARCH_OPTION(BDW HOST "Intel Broadwell Xeon E-class CPUs") +KOKKOS_ARCH_OPTION(SKX HOST "Intel Sky Lake Xeon E-class HPC CPUs (AVX512)") +KOKKOS_ARCH_OPTION(KNC HOST "Intel Knights Corner Xeon Phi") +KOKKOS_ARCH_OPTION(KNL HOST "Intel Knights Landing Xeon Phi") +KOKKOS_ARCH_OPTION(BGQ HOST "IBM Blue Gene Q") +KOKKOS_ARCH_OPTION(POWER7 HOST "IBM POWER7 CPUs") +KOKKOS_ARCH_OPTION(POWER8 HOST "IBM POWER8 CPUs") +KOKKOS_ARCH_OPTION(POWER9 HOST "IBM POWER9 CPUs") +KOKKOS_ARCH_OPTION(KEPLER30 GPU "NVIDIA Kepler generation CC 3.0") +KOKKOS_ARCH_OPTION(KEPLER32 GPU "NVIDIA Kepler generation CC 3.2") +KOKKOS_ARCH_OPTION(KEPLER35 GPU "NVIDIA Kepler generation CC 3.5") +KOKKOS_ARCH_OPTION(KEPLER37 GPU "NVIDIA Kepler generation CC 3.7") +KOKKOS_ARCH_OPTION(MAXWELL50 GPU "NVIDIA Maxwell generation CC 5.0") +KOKKOS_ARCH_OPTION(MAXWELL52 GPU "NVIDIA Maxwell generation CC 5.2") +KOKKOS_ARCH_OPTION(MAXWELL53 GPU "NVIDIA Maxwell generation CC 5.3") +KOKKOS_ARCH_OPTION(PASCAL60 GPU "NVIDIA Pascal generation CC 6.0") +KOKKOS_ARCH_OPTION(PASCAL61 GPU "NVIDIA Pascal generation CC 6.1") +KOKKOS_ARCH_OPTION(VOLTA70 GPU "NVIDIA Volta generation CC 7.0") +KOKKOS_ARCH_OPTION(VOLTA72 GPU "NVIDIA Volta generation CC 7.2") +KOKKOS_ARCH_OPTION(TURING75 GPU "NVIDIA Turing generation CC 7.5") +KOKKOS_ARCH_OPTION(EPYC HOST "AMD Epyc architecture") +KOKKOS_ARCH_OPTION(VEGA900 GPU "AMD GPU MI25 GFX900") +KOKKOS_ARCH_OPTION(VEGA906 GPU "AMD GPU MI50/MI60 GFX906") + +IF (KOKKOS_ENABLE_CUDA) + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_KEPLER30 OR KOKKOS_ARCH_KEPLER32 OR KOKKOS_ARCH_KEPLER35 OR KOKKOS_ARCH_KEPLER37) + SET(KOKKOS_ARCH_KEPLER ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_MAXWELL50 OR KOKKOS_ARCH_MAXWELL52 OR KOKKOS_ARCH_MAXWELL53) + SET(KOKKOS_ARCH_MAXWELL ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_PASCAL60 OR KOKKOS_ARCH_PASCAL61) + SET(KOKKOS_ARCH_PASCAL ON) + ENDIF() + + #Regardless of version, make sure we define the general architecture name + IF (KOKKOS_ARCH_VOLTA70 OR KOKKOS_ARCH_VOLTA72) + SET(KOKKOS_ARCH_VOLTA ON) + ENDIF() +ENDIF() + + + +IF(KOKKOS_ENABLE_COMPILER_WARNINGS) + SET(COMMON_WARNINGS + "-Wall" "-Wunused-parameter" "-Wshadow" "-pedantic" + "-Wsign-compare" "-Wtype-limits" "-Wuninitialized") + + SET(GNU_WARNINGS "-Wempty-body" "-Wclobbered" "-Wignored-qualifiers" + ${COMMON_WARNINGS}) + + COMPILER_SPECIFIC_FLAGS( + PGI NO-VALUE-SPECIFIED + GNU ${GNU_WARNINGS} + DEFAULT ${COMMON_WARNINGS} + ) +ENDIF() + + +#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- +#clear anything that might be in the cache +GLOBAL_SET(KOKKOS_CUDA_OPTIONS) +# Construct the Makefile options +IF (KOKKOS_ENABLE_CUDA_LAMBDA) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-expt-extended-lambda") + ENDIF() +ENDIF() + +IF (KOKKOS_ENABLE_CUDA_CONSTEXPR) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-expt-relaxed-constexpr") + ENDIF() +ENDIF() + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + SET(CUDA_ARCH_FLAG "--cuda-gpu-arch") + SET(AMDGPU_ARCH_FLAG "--amdgpu-target") + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -x cuda) + IF (KOKKOS_ENABLE_CUDA) + SET(KOKKOS_IMPL_CUDA_CLANG_WORKAROUND ON CACHE BOOL "enable CUDA Clang workarounds" FORCE) + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + SET(CUDA_ARCH_FLAG "-arch") +ENDIF() + +IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + STRING(TOUPPER "${CMAKE_BUILD_TYPE}" _UPPERCASE_CMAKE_BUILD_TYPE) + IF (KOKKOS_ENABLE_DEBUG OR _UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -lineinfo) + ENDIF() + UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) + IF (KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9.0 AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 10.0) + GLOBAL_APPEND(KOKKOS_CUDAFE_OPTIONS --diag_suppress=esa_on_defaulted_function_ignored) + ENDIF() +ENDIF() + +IF (KOKKOS_ARCH_ARMV80) + COMPILER_SPECIFIC_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8-a + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV81) + COMPILER_SPECIFIC_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8.1-a + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV8_THUNDERX) + SET(KOKKOS_ARCH_ARMV80 ON) #Not a cache variable + COMPILER_SPECIFIC_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -march=armv8-a -mtune=thunderx + ) +ENDIF() + +IF (KOKKOS_ARCH_ARMV8_THUNDERX2) + SET(KOKKOS_ARCH_ARMV81 ON) #Not a cache variable + COMPILER_SPECIFIC_FLAGS( + Cray NO-VALUE-SPECIFIED + PGI NO-VALUE-SPECIFIED + DEFAULT -mcpu=thunderx2t99 -mtune=thunderx2t99 + ) +ENDIF() + +IF (KOKKOS_ARCH_EPYC) + COMPILER_SPECIFIC_FLAGS( + Intel -mavx2 + DEFAULT -march=znver1 -mtune=znver1 + ) + SET(KOKKOS_ARCH_AMD_EPYC ON) + SET(KOKKOS_ARCH_AMD_AVX2 ON) +ENDIF() + +IF (KOKKOS_ARCH_WSM) + COMPILER_SPECIFIC_FLAGS( + Intel -xSSE4.2 + PGI -tp=nehalem + Cray NO-VALUE-SPECIFIED + DEFAULT -msse4.2 + ) + SET(KOKKOS_ARCH_SSE42 ON) +ENDIF() + +IF (KOKKOS_ARCH_SNB OR KOKKOS_ARCH_AMDAVX) + SET(KOKKOS_ARCH_AVX ON) + COMPILER_SPECIFIC_FLAGS( + Intel -mavx + PGI -tp=sandybridge + Cray NO-VALUE-SPECIFIED + DEFAULT -mavx + ) +ENDIF() + +IF (KOKKOS_ARCH_HSW) + SET(KOKKOS_ARCH_AVX2 ON) + COMPILER_SPECIFIC_FLAGS( + Intel -xCORE-AVX2 + PGI -tp=haswell + Cray NO-VALUE-SPECIFIED + DEFAULT -march=core-avx2 -mtune=core-avx2 + ) +ENDIF() + +IF (KOKKOS_ARCH_BDW) + SET(KOKKOS_ARCH_AVX2 ON) + COMPILER_SPECIFIC_FLAGS( + Intel -xCORE-AVX2 + PGI -tp=haswell + Cray NO-VALUE-SPECIFIED + DEFAULT -march=core-avx2 -mtune=core-avx2 -mrtm + ) +ENDIF() + +IF (KOKKOS_ARCH_EPYC) + SET(KOKKOS_ARCH_AMD_AVX2 ON) + COMPILER_SPECIFIC_FLAGS( + Intel -mvax2 + DEFAULT -march=znver1 -mtune=znver1 + ) +ENDIF() + +IF (KOKKOS_ARCH_KNL) + #avx512-mic + SET(KOKKOS_ARCH_AVX512MIC ON) #not a cache variable + COMPILER_SPECIFIC_FLAGS( + Intel -xMIC-AVX512 + PGI NO-VALUE-SPECIFIED + Cray NO-VALUE-SPECIFIED + DEFAULT -march=knl -mtune=knl + ) +ENDIF() + +IF (KOKKOS_ARCH_KNC) + SET(KOKKOS_USE_ISA_KNC ON) + COMPILER_SPECIFIC_FLAGS( + DEFAULT -mmic + ) +ENDIF() + +IF (KOKKOS_ARCH_SKX) + #avx512-xeon + SET(KOKKOS_ARCH_AVX512XEON ON) + COMPILER_SPECIFIC_FLAGS( + Intel -xCORE-AVX512 + PGI NO-VALUE-SPECIFIED + Cray NO-VALUE-SPECIFIED + DEFAULT -march=skylake-avx512 -mtune=skylake-avx512 -mrtm + ) +ENDIF() + +IF (KOKKOS_ARCH_WSM OR KOKKOS_ARCH_SNB OR KOKKOS_ARCH_HSW OR KOKKOS_ARCH_BDW OR KOKKOS_ARCH_KNL OR KOKKOS_ARCH_SKX OR KOKKOS_ARCH_EPYC) + SET(KOKKOS_USE_ISA_X86_64 ON) +ENDIF() + +IF (KOKKOS_ARCH_BDW OR KOKKOS_ARCH_SKX) + SET(KOKKOS_ENABLE_TM ON) #not a cache variable +ENDIF() + +IF (KOKKOS_ARCH_POWER7) + COMPILER_SPECIFIC_FLAGS( + PGI NO-VALUE-SPECIFIED + DEFAULT -mcpu=power7 -mtune=power7 + ) + SET(KOKKOS_USE_ISA_POWERPCBE ON) +ENDIF() + +IF (KOKKOS_ARCH_POWER8) + COMPILER_SPECIFIC_FLAGS( + PGI NO-VALUE-SPECIFIED + NVIDIA NO-VALUE-SPECIFIED + DEFAULT -mcpu=power8 -mtune=power8 + ) +ENDIF() + +IF (KOKKOS_ARCH_POWER9) + COMPILER_SPECIFIC_FLAGS( + PGI NO-VALUE-SPECIFIED + NVIDIA NO-VALUE-SPECIFIED + DEFAULT -mcpu=power9 -mtune=power9 + ) +ENDIF() + +IF (KOKKOS_ARCH_POWER8 OR KOKKOS_ARCH_POWER9) + SET(KOKKOS_USE_ISA_POWERPCLE ON) +ENDIF() + +IF (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + COMPILER_SPECIFIC_FLAGS( + Clang -fcuda-rdc + NVIDIA --relocatable-device-code=true + ) +ENDIF() + +#Right now we cannot get the compiler ID when cross-compiling, so just check +#that HIP is enabled +IF (Kokkos_ENABLE_HIP) + IF (Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + COMPILER_SPECIFIC_FLAGS( + DEFAULT -fgpu-rdc + ) + ELSE() + COMPILER_SPECIFIC_FLAGS( + DEFAULT -fno-gpu-rdc + ) + ENDIF() +ENDIF() + + +SET(CUDA_ARCH_ALREADY_SPECIFIED "") +FUNCTION(CHECK_CUDA_ARCH ARCH FLAG) + IF(KOKKOS_ARCH_${ARCH}) + IF(CUDA_ARCH_ALREADY_SPECIFIED) + MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${CUDA_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") + ENDIF() + SET(CUDA_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) + IF (NOT KOKKOS_ENABLE_CUDA AND NOT KOKKOS_ENABLE_OPENMPTARGET) + MESSAGE(WARNING "Given CUDA arch ${ARCH}, but Kokkos_ENABLE_CUDA and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") + UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) + ELSE() + SET(KOKKOS_CUDA_ARCH_FLAG ${FLAG} PARENT_SCOPE) + GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + IF(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + ENDIF() + ENDIF() + ENDIF() +ENDFUNCTION() + + +#These will define KOKKOS_CUDA_ARCH_FLAG +#to the corresponding flag name if ON +CHECK_CUDA_ARCH(KEPLER30 sm_30) +CHECK_CUDA_ARCH(KEPLER32 sm_32) +CHECK_CUDA_ARCH(KEPLER35 sm_35) +CHECK_CUDA_ARCH(KEPLER37 sm_37) +CHECK_CUDA_ARCH(MAXWELL50 sm_50) +CHECK_CUDA_ARCH(MAXWELL52 sm_52) +CHECK_CUDA_ARCH(MAXWELL53 sm_53) +CHECK_CUDA_ARCH(PASCAL60 sm_60) +CHECK_CUDA_ARCH(PASCAL61 sm_61) +CHECK_CUDA_ARCH(VOLTA70 sm_70) +CHECK_CUDA_ARCH(VOLTA72 sm_72) +CHECK_CUDA_ARCH(TURING75 sm_75) + +SET(AMDGPU_ARCH_ALREADY_SPECIFIED "") +FUNCTION(CHECK_AMDGPU_ARCH ARCH FLAG) + IF(KOKKOS_ARCH_${ARCH}) + IF(AMDGPU_ARCH_ALREADY_SPECIFIED) + MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${AMDGPU_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") + ENDIF() + SET(AMDGPU_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) + IF (NOT KOKKOS_ENABLE_HIP AND NOT KOKKOS_ENABLE_OPENMPTARGET) + MESSAGE(WARNING "Given HIP arch ${ARCH}, but Kokkos_ENABLE_AMDGPU and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") + UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) + ELSE() + SET(KOKKOS_AMDGPU_ARCH_FLAG ${FLAG} PARENT_SCOPE) + GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") + IF(KOKKOS_ENABLE_HIP) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") + ENDIF() + ENDIF() + ENDIF() +ENDFUNCTION() + +#These will define KOKKOS_AMDGPU_ARCH_FLAG +#to the corresponding flag name if ON +CHECK_AMDGPU_ARCH(VEGA900 gfx900) # Radeon Instinct MI25 +CHECK_AMDGPU_ARCH(VEGA906 gfx906) # Radeon Instinct MI50 and MI60 + +IF (KOKKOS_ENABLE_OPENMPTARGET) + SET(CLANG_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) + IF (CLANG_CUDA_ARCH) + COMPILER_SPECIFIC_FLAGS( + Clang -Xopenmp-target -march=${CLANG_CUDA_ARCH} -fopenmp-targets=nvptx64-nvidia-cuda + XL -qtgtarch=${KOKKOS_CUDA_ARCH_FLAG} + ) + ENDIF() + SET(CLANG_AMDGPU_ARCH ${KOKKOS_AMDGPU_ARCH_FLAG}) + IF (CLANG_AMDGPU_ARCH) + COMPILER_SPECIFIC_FLAGS( + Clang -Xopenmp-target=amdgcn-amd-amdhsa -march=${CLANG_AMDGPU_ARCH} -fopenmp-targets=amdgcn-amd-amdhsa + ) + ENDIF() +ENDIF() + +IF(KOKKOS_ENABLE_CUDA AND NOT CUDA_ARCH_ALREADY_SPECIFIED) + MESSAGE(SEND_ERROR "CUDA enabled but no NVIDIA GPU architecture currently enabled. Please give one -DKokkos_ARCH_{..}=ON' to enable an NVIDIA GPU architecture.") +ENDIF() + +#CMake verbose is kind of pointless +#Let's just always print things +MESSAGE(STATUS "Execution Spaces:") + +FOREACH (_BACKEND CUDA OPENMPTARGET HIP) + IF(KOKKOS_ENABLE_${_BACKEND}) + IF(_DEVICE_PARALLEL) + MESSAGE(FATAL_ERROR "Multiple device parallel execution spaces are not allowed! " + "Trying to enable execution space ${_BACKEND}, " + "but execution space ${_DEVICE_PARALLEL} is already enabled. " + "Remove the CMakeCache.txt file and re-configure.") + ENDIF() + SET(_DEVICE_PARALLEL ${_BACKEND}) + ENDIF() +ENDFOREACH() +IF(NOT _DEVICE_PARALLEL) + SET(_DEVICE_PARALLEL "NONE") +ENDIF() +MESSAGE(STATUS " Device Parallel: ${_DEVICE_PARALLEL}") +UNSET(_DEVICE_PARALLEL) + + +FOREACH (_BACKEND OPENMP PTHREAD HPX) + IF(KOKKOS_ENABLE_${_BACKEND}) + IF(_HOST_PARALLEL) + MESSAGE(FATAL_ERROR "Multiple host parallel execution spaces are not allowed! " + "Trying to enable execution space ${_BACKEND}, " + "but execution space ${_HOST_PARALLEL} is already enabled. " + "Remove the CMakeCache.txt file and re-configure.") + ENDIF() + SET(_HOST_PARALLEL ${_BACKEND}) + ENDIF() +ENDFOREACH() + +IF(NOT _HOST_PARALLEL AND NOT KOKKOS_ENABLE_SERIAL) + MESSAGE(FATAL_ERROR "At least one host execution space must be enabled, " + "but no host parallel execution space was requested " + "and Kokkos_ENABLE_SERIAL=OFF.") +ENDIF() + +IF(NOT _HOST_PARALLEL) + SET(_HOST_PARALLEL "NONE") +ENDIF() +MESSAGE(STATUS " Host Parallel: ${_HOST_PARALLEL}") +UNSET(_HOST_PARALLEL) + +IF(KOKKOS_ENABLE_PTHREAD) + SET(KOKKOS_ENABLE_THREADS ON) +ENDIF() + +IF(KOKKOS_ENABLE_SERIAL) + MESSAGE(STATUS " Host Serial: SERIAL") +ELSE() + MESSAGE(STATUS " Host Serial: NONE") +ENDIF() + +MESSAGE(STATUS "") +MESSAGE(STATUS "Architectures:") +FOREACH(Arch ${KOKKOS_ENABLED_ARCH_LIST}) + MESSAGE(STATUS " ${Arch}") +ENDFOREACH() + diff --git a/lib/kokkos/cmake/kokkos_build.cmake b/lib/kokkos/cmake/kokkos_build.cmake deleted file mode 100644 index f9b995baae..0000000000 --- a/lib/kokkos/cmake/kokkos_build.cmake +++ /dev/null @@ -1,261 +0,0 @@ -############################ Detect if submodule ############################### -# -# With thanks to StackOverflow: -# http://stackoverflow.com/questions/25199677/how-to-detect-if-current-scope-has-a-parent-in-cmake -# -get_directory_property(HAS_PARENT PARENT_DIRECTORY) -if(HAS_PARENT) - message(STATUS "Submodule build") - SET(KOKKOS_HEADER_DIR "include/kokkos") -else() - message(STATUS "Standalone build") - SET(KOKKOS_HEADER_DIR "include") -endif() - -################################ Handle the actual build ####################### - -SET(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") -SET(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") -SET(INSTALL_INCLUDE_DIR ${KOKKOS_HEADER_DIR} CACHE PATH - "Installation directory for header files") -IF(WIN32 AND NOT CYGWIN) - SET(DEF_INSTALL_CMAKE_DIR CMake) -ELSE() - SET(DEF_INSTALL_CMAKE_DIR lib/CMake/Kokkos) -ENDIF() - -SET(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH - "Installation directory for CMake files") - -# Make relative paths absolute (needed later on) -FOREACH(p LIB BIN INCLUDE CMAKE) - SET(var INSTALL_${p}_DIR) - IF(NOT IS_ABSOLUTE "${${var}}") - SET(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") - ENDIF() -ENDFOREACH() - -# set up include-directories -SET (Kokkos_INCLUDE_DIRS - ${Kokkos_SOURCE_DIR}/core/src - ${Kokkos_SOURCE_DIR}/containers/src - ${Kokkos_SOURCE_DIR}/algorithms/src - ${Kokkos_BINARY_DIR} # to find KokkosCore_config.h - ${KOKKOS_INCLUDE_DIRS} -) - -# pass include dirs back to parent scope -if(HAS_PARENT) -SET(Kokkos_INCLUDE_DIRS_RET ${Kokkos_INCLUDE_DIRS} PARENT_SCOPE) -else() -SET(Kokkos_INCLUDE_DIRS_RET ${Kokkos_INCLUDE_DIRS}) -endif() - -INCLUDE_DIRECTORIES(${Kokkos_INCLUDE_DIRS}) - -IF(KOKKOS_SEPARATE_LIBS) - # Sources come from makefile-generated kokkos_generated_settings.cmake file - # Separate libs need to separate the sources - set_kokkos_srcs(KOKKOS_SRC ${KOKKOS_SRC}) - - # kokkoscore - ADD_LIBRARY( - kokkoscore - ${KOKKOS_CORE_SRCS} - ) - - target_compile_options( - kokkoscore - PUBLIC $<$:${KOKKOS_CXX_FLAGS}> - ) - - target_include_directories( - kokkoscore - PUBLIC - ${KOKKOS_TPL_INCLUDE_DIRS} - ) - - foreach(lib IN LISTS KOKKOS_TPL_LIBRARY_NAMES) - if (("${lib}" STREQUAL "cuda") AND (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")) - set(LIB_cuda "-lcuda") - elseif ("${lib}" STREQUAL "hpx") - find_package(HPX REQUIRED) - if(${HPX_FOUND}) - target_link_libraries(kokkoscore PUBLIC ${HPX_LIBRARIES}) - target_link_libraries(kokkoscontainers PUBLIC ${HPX_LIBRARIES}) - target_link_libraries(kokkosalgorithms PUBLIC ${HPX_LIBRARIES}) - target_include_directories(kokkoscore PUBLIC ${HPX_INCLUDE_DIRS}) - target_include_directories(kokkoscontainers PUBLIC ${HPX_INCLUDE_DIRS}) - target_include_directories(kokkosalgorithms PUBLIC ${HPX_INCLUDE_DIRS}) - else() - message(ERROR "HPX not found. Check the value of HPX_DIR (= ${HPX_DIR}) or CMAKE_PREFIX_PATH (= ${CMAKE_PREFIX_PATH}).") - endif() - else() - find_library(LIB_${lib} ${lib} PATHS ${KOKKOS_TPL_LIBRARY_DIRS}) - endif() - target_link_libraries(kokkoscore PUBLIC ${LIB_${lib}}) - endforeach() - - target_link_libraries(kokkoscore PUBLIC "${KOKKOS_LINK_FLAGS}") - - # Install the kokkoscore library - INSTALL (TARGETS kokkoscore - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin - ) - - # kokkoscontainers - if (DEFINED KOKKOS_CONTAINERS_SRCS) - ADD_LIBRARY( - kokkoscontainers - ${KOKKOS_CONTAINERS_SRCS} - ) - endif() - - TARGET_LINK_LIBRARIES( - kokkoscontainers - kokkoscore - ) - - # Install the kokkocontainers library - INSTALL (TARGETS kokkoscontainers - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - # kokkosalgorithms - Build as interface library since no source files. - ADD_LIBRARY( - kokkosalgorithms - INTERFACE - ) - - target_include_directories( - kokkosalgorithms - INTERFACE ${Kokkos_SOURCE_DIR}/algorithms/src - ) - - TARGET_LINK_LIBRARIES( - kokkosalgorithms - INTERFACE kokkoscore - ) - - # Install the kokkoalgorithms library - INSTALL (TARGETS kokkosalgorithms - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - SET (Kokkos_LIBRARIES_NAMES kokkoscore kokkoscontainers kokkosalgorithms) - -ELSE() - # kokkos - ADD_LIBRARY( - kokkos - ${KOKKOS_CORE_SRCS} - ${KOKKOS_CONTAINERS_SRCS} - ) - - target_compile_options( - kokkos - PUBLIC $<$:${KOKKOS_CXX_FLAGS}> - ) - - target_include_directories( - kokkos - PUBLIC - ${KOKKOS_TPL_INCLUDE_DIRS} - ) - - foreach(lib IN LISTS KOKKOS_TPL_LIBRARY_NAMES) - if (("${lib}" STREQUAL "cuda") AND (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")) - set(LIB_cuda "-lcuda") - elseif ("${lib}" STREQUAL "hpx") - find_package(HPX REQUIRED) - if(${HPX_FOUND}) - target_link_libraries(kokkos PUBLIC ${HPX_LIBRARIES}) - target_include_directories(kokkos PUBLIC ${HPX_INCLUDE_DIRS}) - else() - message(ERROR "HPX not found. Check the value of HPX_DIR (= ${HPX_DIR}) or CMAKE_PREFIX_PATH (= ${CMAKE_PREFIX_PATH}).") - endif() - else() - find_library(LIB_${lib} ${lib} PATHS ${KOKKOS_TPL_LIBRARY_DIRS}) - endif() - target_link_libraries(kokkos PUBLIC ${LIB_${lib}}) - endforeach() - - target_link_libraries(kokkos PUBLIC "${KOKKOS_LINK_FLAGS}") - - # Install the kokkos library - INSTALL (TARGETS kokkos - EXPORT KokkosTargets - ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib - RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) - - - SET (Kokkos_LIBRARIES_NAMES kokkos) - -endif() # KOKKOS_SEPARATE_LIBS - -# Install the kokkos headers -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/core/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/containers/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) -INSTALL (DIRECTORY - EXPORT KokkosTargets - ${Kokkos_SOURCE_DIR}/algorithms/src/ - DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" -) - -INSTALL (FILES - ${Kokkos_BINARY_DIR}/KokkosCore_config.h - DESTINATION ${KOKKOS_HEADER_DIR} -) - -# Add all targets to the build-tree export set -export(TARGETS ${Kokkos_LIBRARIES_NAMES} - FILE "${Kokkos_BINARY_DIR}/KokkosTargets.cmake") - -# Export the package for use from the build-tree -# (this registers the build-tree with a global CMake-registry) -export(PACKAGE Kokkos) - -# Create the KokkosConfig.cmake and KokkosConfigVersion files -file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}" - "${INSTALL_INCLUDE_DIR}") -# ... for the build tree -set(CONF_INCLUDE_DIRS "${Kokkos_SOURCE_DIR}" "${Kokkos_BINARY_DIR}") -configure_file(${Kokkos_SOURCE_DIR}/cmake/KokkosConfig.cmake.in - "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" @ONLY) -# ... for the install tree -set(CONF_INCLUDE_DIRS "\${Kokkos_CMAKE_DIR}/${REL_INCLUDE_DIR}") -configure_file(${Kokkos_SOURCE_DIR}/cmake/KokkosConfig.cmake.in - "${Kokkos_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/KokkosConfig.cmake" @ONLY) - -# Install the KokkosConfig.cmake and KokkosConfigVersion.cmake -install(FILES - "${Kokkos_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/KokkosConfig.cmake" - DESTINATION "${INSTALL_CMAKE_DIR}") - -#This seems not to do anything? -#message(STATUS "KokkosTargets: " ${KokkosTargets}) -# Install the export set for use with the install-tree -INSTALL(EXPORT KokkosTargets DESTINATION - "${INSTALL_CMAKE_DIR}") - -# build and install pkgconfig file -CONFIGURE_FILE(core/src/kokkos.pc.in kokkos.pc @ONLY) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/kokkos.pc DESTINATION lib/pkgconfig) diff --git a/lib/kokkos/cmake/kokkos_check_env.cmake b/lib/kokkos/cmake/kokkos_check_env.cmake new file mode 100644 index 0000000000..a455a403b9 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_check_env.cmake @@ -0,0 +1,12 @@ +SET(CRAYPE_VERSION $ENV{CRAYPE_VERSION}) +IF (CRAYPE_VERSION) + SET(KOKKOS_IS_CRAYPE TRUE) + SET(CRAYPE_LINK_TYPE $ENV{CRAYPE_LINK_TYPE}) + IF (CRAYPE_LINK_TYPE) + IF (NOT CRAYPE_LINK_TYPE STREQUAL "dynamic") + MESSAGE(WARNING "CRAYPE_LINK_TYPE is set to ${CRAYPE_LINK_TYPE}. Linking is likely to fail unless this is set to 'dynamic'") + ENDIF() + ELSE() + MESSAGE(WARNING "CRAYPE_LINK_TYPE is not set. Linking is likely to fail unless this is set to 'dynamic'") + ENDIF() +ENDIF() diff --git a/lib/kokkos/cmake/kokkos_compiler_id.cmake b/lib/kokkos/cmake/kokkos_compiler_id.cmake new file mode 100644 index 0000000000..cd5e7c9e4e --- /dev/null +++ b/lib/kokkos/cmake/kokkos_compiler_id.cmake @@ -0,0 +1,96 @@ +KOKKOS_CFG_DEPENDS(COMPILER_ID NONE) + +SET(KOKKOS_CXX_COMPILER ${CMAKE_CXX_COMPILER}) +SET(KOKKOS_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) +SET(KOKKOS_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) + +# Check if the compiler is nvcc (which really means nvcc_wrapper). +EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep nvcc + COMMAND wc -l + OUTPUT_VARIABLE INTERNAL_HAVE_COMPILER_NVCC + OUTPUT_STRIP_TRAILING_WHITESPACE) + + +STRING(REGEX REPLACE "^ +" "" + INTERNAL_HAVE_COMPILER_NVCC "${INTERNAL_HAVE_COMPILER_NVCC}") + + +IF(INTERNAL_HAVE_COMPILER_NVCC) + # SET the compiler id to nvcc. We use the value used by CMake 3.8. + SET(KOKKOS_CXX_COMPILER_ID NVIDIA CACHE STRING INTERNAL FORCE) + + # SET nvcc's compiler version. + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep release + OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+$" + TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) +ENDIF() + + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + # The Cray compiler reports as Clang to most versions of CMake + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep Cray + COMMAND wc -l + OUTPUT_VARIABLE INTERNAL_HAVE_CRAY_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE) + IF (INTERNAL_HAVE_CRAY_COMPILER) #not actually Clang + SET(KOKKOS_CLANG_IS_CRAY TRUE) + ENDIF() +ENDIF() + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray OR KOKKOS_CLANG_IS_CRAY) + # SET Cray's compiler version. + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version + OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" + TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + IF (KOKKOS_CLANG_IS_CRAY) + SET(KOKKOS_CLANG_CRAY_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION}) + ELSE() + SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) + ENDIF() +ENDIF() + +# Enforce the minimum compilers supported by Kokkos. +SET(KOKKOS_MESSAGE_TEXT "Compiler not supported by Kokkos. Required compiler versions:") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang 3.5.2 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC 4.8.4 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel 15.0.2 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC 9.0.69 or higher") +SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n PGI 17.1 or higher\n") + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 3.5.2) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 9.0.69) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() + SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Kokkos turns off CXX extensions" FORCE) +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL PGI) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 17.1) + MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + ENDIF() +ENDIF() + +STRING(REPLACE "." ";" VERSION_LIST ${KOKKOS_CXX_COMPILER_VERSION}) +LIST(GET VERSION_LIST 0 KOKKOS_COMPILER_VERSION_MAJOR) +LIST(GET VERSION_LIST 1 KOKKOS_COMPILER_VERSION_MINOR) +LIST(GET VERSION_LIST 2 KOKKOS_COMPILER_VERSION_PATCH) diff --git a/lib/kokkos/cmake/kokkos_corner_cases.cmake b/lib/kokkos/cmake/kokkos_corner_cases.cmake new file mode 100644 index 0000000000..e30be3c841 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_corner_cases.cmake @@ -0,0 +1,59 @@ +IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND KOKKOS_ENABLE_OPENMP AND NOT KOKKOS_CLANG_IS_CRAY) + # The clang "version" doesn't actually tell you what runtimes and tools + # were built into Clang. We should therefore make sure that libomp + # was actually built into Clang. Otherwise the user will get nonsensical + # errors when they try to build. + + #Try compile is the height of CMake nonsense + #I can't just give it compiler and link flags + #I have to hackily pretend that compiler flags are compiler definitions + #and that linker flags are libraries + #also - this is easier to use than CMakeCheckCXXSourceCompiles + TRY_COMPILE(CLANG_HAS_OMP + ${KOKKOS_TOP_BUILD_DIR}/corner_cases + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/clang_omp.cpp + COMPILE_DEFINITIONS -fopenmp=libomp + LINK_LIBRARIES -fopenmp=libomp + ) + IF (NOT CLANG_HAS_OMP) + UNSET(CLANG_HAS_OMP CACHE) #make sure CMake always re-runs this + MESSAGE(FATAL_ERROR "Clang failed OpenMP check. You have requested -DKokkos_ENABLE_OPENMP=ON, but the Clang compiler does not appear to have been built with OpenMP support") + ENDIF() + UNSET(CLANG_HAS_OMP CACHE) #make sure CMake always re-runs this +ENDIF() + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL AppleClang AND KOKKOS_ENABLE_OPENMP) + # The clang "version" doesn't actually tell you what runtimes and tools + # were built into Clang. We should therefore make sure that libomp + # was actually built into Clang. Otherwise the user will get nonsensical + # errors when they try to build. + + #Try compile is the height of CMake nonsense + #I can't just give it compiler and link flags + #I have to hackily pretend that compiler flags are compiler definitions + #and that linker flags are libraries + #also - this is easier to use than CMakeCheckCXXSourceCompiles + TRY_COMPILE(APPLECLANG_HAS_OMP + ${KOKKOS_TOP_BUILD_DIR}/corner_cases + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/clang_omp.cpp + COMPILE_DEFINITIONS -Xpreprocessor -fopenmp + LINK_LIBRARIES -lomp + ) + IF (NOT APPLECLANG_HAS_OMP) + UNSET(APPLECLANG_HAS_OMP CACHE) #make sure CMake always re-runs this + MESSAGE(FATAL_ERROR "AppleClang failed OpenMP check. You have requested -DKokkos_ENABLE_OPENMP=ON, but the AppleClang compiler does not appear to have been built with OpenMP support") + ENDIF() + UNSET(APPLECLANG_HAS_OMP CACHE) #make sure CMake always re-runs this +ENDIF() + + +IF (KOKKOS_CXX_STANDARD STREQUAL 17) + IF (KOKKOS_CXX_COMPILER_ID STREQUAL GNU AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 7) + MESSAGE(FATAL_ERROR "You have requested c++17 support for GCC ${KOKKOS_CXX_COMPILER_VERSION}. Although CMake has allowed this and GCC accepts -std=c++1z/c++17, GCC <= 6 does not properly support *this capture. Please reduce the C++ standard to 14 or upgrade the compiler if you do need 17 support") + ENDIF() + + IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + MESSAGE(FATAL_ERROR "You have requested c++17 support for NVCC. Please reduce the C++ standard to 14. No versions of NVCC currently support 17.") + ENDIF() +ENDIF() + diff --git a/lib/kokkos/cmake/kokkos_enable_devices.cmake b/lib/kokkos/cmake/kokkos_enable_devices.cmake new file mode 100644 index 0000000000..7b50cfe458 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_enable_devices.cmake @@ -0,0 +1,98 @@ + +FUNCTION(KOKKOS_DEVICE_OPTION SUFFIX DEFAULT DEV_TYPE DOCSTRING) + KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + STRING(TOUPPER ${SUFFIX} UC_NAME) + IF (KOKKOS_ENABLE_${UC_NAME}) + LIST(APPEND KOKKOS_ENABLED_DEVICES ${SUFFIX}) + #I hate that CMake makes me do this + SET(KOKKOS_ENABLED_DEVICES ${KOKKOS_ENABLED_DEVICES} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) + IF (KOKKOS_ENABLE_${UC_NAME} AND DEV_TYPE STREQUAL "HOST") + SET(KOKKOS_HAS_HOST ON PARENT_SCOPE) + ENDIF() +ENDFUNCTION() + +KOKKOS_CFG_DEPENDS(DEVICES NONE) + +# Put a check in just in case people are using this option +KOKKOS_DEPRECATED_LIST(DEVICES ENABLE) + + +KOKKOS_DEVICE_OPTION(PTHREAD OFF HOST "Whether to build Pthread backend") +IF (KOKKOS_ENABLE_PTHREAD) + #patch the naming here + SET(KOKKOS_ENABLE_THREADS ON) +ENDIF() + +IF(Trilinos_ENABLE_Kokkos AND Trilinos_ENABLE_OpenMP) + SET(OMP_DEFAULT ON) +ELSE() + SET(OMP_DEFAULT OFF) +ENDIF() +KOKKOS_DEVICE_OPTION(OPENMP ${OMP_DEFAULT} HOST "Whether to build OpenMP backend") +IF(KOKKOS_ENABLE_OPENMP) + SET(ClangOpenMPFlag -fopenmp=libomp) + IF(KOKKOS_CLANG_IS_CRAY) + SET(ClangOpenMPFlag -fopenmp) + ENDIF() + COMPILER_SPECIFIC_FLAGS( + Clang ${ClangOpenMPFlag} + AppleClang -Xpreprocessor -fopenmp + PGI -mp + NVIDIA -Xcompiler -fopenmp + Cray NO-VALUE-SPECIFIED + XL -qsmp=omp + DEFAULT -fopenmp + ) + COMPILER_SPECIFIC_LIBS( + AppleClang -lomp + ) +ENDIF() + +KOKKOS_DEVICE_OPTION(OPENMPTARGET OFF DEVICE "Whether to build the OpenMP target backend") +IF (KOKKOS_ENABLE_OPENMPTARGET) + COMPILER_SPECIFIC_FLAGS( + Clang -fopenmp -fopenmp=libomp + XL -qsmp=omp -qoffload -qnoeh + DEFAULT -fopenmp + ) + COMPILER_SPECIFIC_DEFS( + XL KOKKOS_IBM_XL_OMP45_WORKAROUND + Clang KOKKOS_WORKAROUND_OPENMPTARGET_CLANG + ) +# Are there compilers which identify as Clang and need this library? +# COMPILER_SPECIFIC_LIBS( +# Clang -lopenmptarget +# ) +ENDIF() + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) + SET(CUDA_DEFAULT ON) +ELSE() + SET(CUDA_DEFAULT OFF) +ENDIF() +KOKKOS_DEVICE_OPTION(CUDA ${CUDA_DEFAULT} DEVICE "Whether to build CUDA backend") + +IF (KOKKOS_ENABLE_CUDA) + GLOBAL_SET(KOKKOS_DONT_ALLOW_EXTENSIONS "CUDA enabled") +ENDIF() + +# We want this to default to OFF for cache reasons, but if no +# host space is given, then activate serial +IF (KOKKOS_HAS_TRILINOS) + #However, Trilinos always wants Serial ON + SET(SERIAL_DEFAULT ON) +ELSEIF (KOKKOS_HAS_HOST) + SET(SERIAL_DEFAULT OFF) +ELSE() + SET(SERIAL_DEFAULT ON) + IF (NOT DEFINED Kokkos_ENABLE_SERIAL) + MESSAGE(STATUS "SERIAL backend is being turned on to ensure there is at least one Host space. To change this, you must enable another host execution space and configure with -DKokkos_ENABLE_SERIAL=OFF or change CMakeCache.txt") + ENDIF() +ENDIF() +KOKKOS_DEVICE_OPTION(SERIAL ${SERIAL_DEFAULT} HOST "Whether to build serial backend") + +KOKKOS_DEVICE_OPTION(HPX OFF HOST "Whether to build HPX backend (experimental)") + +KOKKOS_DEVICE_OPTION(HIP OFF DEVICE "Whether to build HIP backend") diff --git a/lib/kokkos/cmake/kokkos_enable_options.cmake b/lib/kokkos/cmake/kokkos_enable_options.cmake new file mode 100644 index 0000000000..4560c3df8f --- /dev/null +++ b/lib/kokkos/cmake/kokkos_enable_options.cmake @@ -0,0 +1,112 @@ +########################## NOTES ############################################### +# List the options for configuring kokkos using CMake method of doing it. +# These options then get mapped onto KOKKOS_SETTINGS environment variable by +# kokkos_settings.cmake. It is separate to allow other packages to override +# these variables (e.g., TriBITS). + +########################## AVAILABLE OPTIONS ################################### +# Use lists for documentation, verification, and programming convenience + + +FUNCTION(KOKKOS_ENABLE_OPTION SUFFIX DEFAULT DOCSTRING) + KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + STRING(TOUPPER ${SUFFIX} UC_NAME) + IF (KOKKOS_ENABLE_${UC_NAME}) + LIST(APPEND KOKKOS_ENABLED_OPTIONS ${UC_NAME}) + #I hate that CMake makes me do this + SET(KOKKOS_ENABLED_OPTIONS ${KOKKOS_ENABLED_OPTIONS} PARENT_SCOPE) + ENDIF() + SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) +ENDFUNCTION() + +# Certain defaults will depend on knowing the enabled devices +KOKKOS_CFG_DEPENDS(OPTIONS DEVICES) +KOKKOS_CFG_DEPENDS(OPTIONS COMPILER_ID) + +# Put a check in just in case people are using this option +KOKKOS_DEPRECATED_LIST(OPTIONS ENABLE) + +KOKKOS_ENABLE_OPTION(CUDA_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for CUDA") +KOKKOS_ENABLE_OPTION(CUDA_UVM OFF "Whether to use unified memory (UM) for CUDA by default") +KOKKOS_ENABLE_OPTION(CUDA_LDG_INTRINSIC OFF "Whether to use CUDA LDG intrinsics") +KOKKOS_ENABLE_OPTION(HIP_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for HIP") +KOKKOS_ENABLE_OPTION(HPX_ASYNC_DISPATCH OFF "Whether HPX supports asynchronous dispatch") +KOKKOS_ENABLE_OPTION(TESTS OFF "Whether to build the unit tests") +KOKKOS_ENABLE_OPTION(EXAMPLES OFF "Whether to build the examples") +STRING(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_CMAKE_BUILD_TYPE) +IF(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + KOKKOS_ENABLE_OPTION(DEBUG ON "Whether to activate extra debug features - may increase compile times") + KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK ON "Debug check on dual views") +ELSE() + KOKKOS_ENABLE_OPTION(DEBUG OFF "Whether to activate extra debug features - may increase compile times") + KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK OFF "Debug check on dual views") +ENDIF() +UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) +KOKKOS_ENABLE_OPTION(LARGE_MEM_TESTS OFF "Whether to perform extra large memory tests") +KOKKOS_ENABLE_OPTION(DEBUG_BOUNDS_CHECK OFF "Whether to use bounds checking - will increase runtime") +KOKKOS_ENABLE_OPTION(COMPILER_WARNINGS OFF "Whether to print all compiler warnings") +KOKKOS_ENABLE_OPTION(PROFILING ON "Whether to create bindings for profiling tools") +KOKKOS_ENABLE_OPTION(PROFILING_LOAD_PRINT OFF "Whether to print information about which profiling tools got loaded") +KOKKOS_ENABLE_OPTION(AGGRESSIVE_VECTORIZATION OFF "Whether to aggressively vectorize loops") +KOKKOS_ENABLE_OPTION(DEPRECATED_CODE OFF "Whether to enable deprecated code") + +IF (KOKKOS_ENABLE_CUDA) + SET(KOKKOS_COMPILER_CUDA_VERSION "${KOKKOS_COMPILER_VERSION_MAJOR}${KOKKOS_COMPILER_VERSION_MINOR}") +ENDIF() + +IF (Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) + SET(CUDA_LAMBDA_DEFAULT ON) +ELSEIF (KOKKOS_ENABLE_CUDA AND (KOKKOS_CXX_COMPILER_ID STREQUAL Clang)) + SET(CUDA_LAMBDA_DEFAULT ON) +ELSE() + SET(CUDA_LAMBDA_DEFAULT OFF) +ENDIF() +KOKKOS_ENABLE_OPTION(CUDA_LAMBDA ${CUDA_LAMBDA_DEFAULT} "Whether to activate experimental lambda features") +IF (Trilinos_ENABLE_Kokkos) + SET(COMPLEX_ALIGN_DEFAULT OFF) +ELSE() + SET(COMPLEX_ALIGN_DEFAULT ON) +ENDIF() +KOKKOS_ENABLE_OPTION(COMPLEX_ALIGN ${COMPLEX_ALIGN_DEFAULT} "Whether to align Kokkos::complex to 2*alignof(RealType)") + + +IF (KOKKOS_ENABLE_CUDA AND (KOKKOS_CXX_COMPILER_ID STREQUAL Clang)) + SET(CUDA_CONSTEXPR_DEFAULT ON) +ELSE() + SET(CUDA_CONSTEXPR_DEFAULT OFF) +ENDIF() +KOKKOS_ENABLE_OPTION(CUDA_CONSTEXPR ${CUDA_CONSTEXPR_DEFAULT} "Whether to activate experimental relaxed constexpr functions") + +FUNCTION(check_device_specific_options) + CMAKE_PARSE_ARGUMENTS(SOME "" "DEVICE" "OPTIONS" ${ARGN}) + IF(NOT KOKKOS_ENABLE_${SOME_DEVICE}) + FOREACH(OPTION ${SOME_OPTIONS}) + IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.14) + IF(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) + MESSAGE(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") + ENDIF() + ENDIF() + IF(KOKKOS_ENABLE_${OPTION}) + MESSAGE(WARNING "Kokkos_ENABLE_${OPTION} is ON but ${SOME_DEVICE} backend is not enabled. Option will be ignored.") + UNSET(KOKKOS_ENABLE_${OPTION} PARENT_SCOPE) + ENDIF() + ENDFOREACH() + ENDIF() +ENDFUNCTION() + +CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE CUDA OPTIONS CUDA_UVM CUDA_RELOCATABLE_DEVICE_CODE CUDA_LAMBDA CUDA_CONSTEXPR CUDA_LDG_INTRINSIC) +CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE HIP OPTIONS HIP_RELOCATABLE_DEVICE_CODE) +CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE HPX OPTIONS HPX_ASYNC_DISPATCH) + +# Needed due to change from deprecated name to new header define name +IF (KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) + SET(KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION ON) +ENDIF() + +# This is known to occur with Clang 9. We would need to use nvcc as the linker +# http://lists.llvm.org/pipermail/cfe-dev/2018-June/058296.html +# TODO: Through great effort we can use a different linker by hacking +# CMAKE_CXX_LINK_EXECUTABLE in a future release +IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE AND KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + MESSAGE(FATAL_ERROR "Relocatable device code is currently not supported with Clang - must use nvcc_wrapper or turn off RDC") +ENDIF() diff --git a/lib/kokkos/cmake/kokkos_functions.cmake b/lib/kokkos/cmake/kokkos_functions.cmake index 616618753b..fd04966baf 100644 --- a/lib/kokkos/cmake/kokkos_functions.cmake +++ b/lib/kokkos/cmake/kokkos_functions.cmake @@ -1,345 +1,801 @@ ################################### FUNCTIONS ################################## # List of functions -# set_kokkos_cxx_compiler -# set_kokkos_cxx_standard -# set_kokkos_srcs +# kokkos_option + +# Validate options are given with correct case and define an internal +# upper-case version for use within -#------------------------------------------------------------------------------- -# function(set_kokkos_cxx_compiler) -# Sets the following compiler variables that are analogous to the CMAKE_* -# versions. We add the ability to detect NVCC (really nvcc_wrapper). -# KOKKOS_CXX_COMPILER -# KOKKOS_CXX_COMPILER_ID -# KOKKOS_CXX_COMPILER_VERSION # -# Inputs: -# KOKKOS_ENABLE_CUDA -# CMAKE_CXX_COMPILER -# CMAKE_CXX_COMPILER_ID -# CMAKE_CXX_COMPILER_VERSION # -# Also verifies the compiler version meets the minimum required by Kokkos. -function(set_kokkos_cxx_compiler) - # Since CMake doesn't recognize the nvcc compiler until 3.8, we use our own - # version of the CMake variables and detect nvcc ourselves. Initially set to - # the CMake variable values. - set(INTERNAL_CXX_COMPILER ${CMAKE_CXX_COMPILER}) - set(INTERNAL_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) - set(INTERNAL_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) - - # Check if the compiler is nvcc (which really means nvcc_wrapper). - execute_process(COMMAND ${INTERNAL_CXX_COMPILER} --version - COMMAND grep nvcc - COMMAND wc -l - OUTPUT_VARIABLE INTERNAL_HAVE_COMPILER_NVCC - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REGEX REPLACE "^ +" "" - INTERNAL_HAVE_COMPILER_NVCC ${INTERNAL_HAVE_COMPILER_NVCC}) - - if(INTERNAL_HAVE_COMPILER_NVCC) - # Set the compiler id to nvcc. We use the value used by CMake 3.8. - set(INTERNAL_CXX_COMPILER_ID NVIDIA) - - # Set nvcc's compiler version. - execute_process(COMMAND ${INTERNAL_CXX_COMPILER} --version - COMMAND grep release - OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+$" - INTERNAL_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) - endif() - - # Enforce the minimum compilers supported by Kokkos. - set(KOKKOS_MESSAGE_TEXT "Compiler not supported by Kokkos. Required compiler versions:") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang 3.5.2 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC 4.8.4 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel 15.0.2 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC 7.0.28 or higher") - set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n PGI 17.1 or higher\n") - - if(INTERNAL_CXX_COMPILER_ID STREQUAL Clang) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 3.5.2) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL GNU) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 4.8.4) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL Intel) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 7.0.28) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - elseif(INTERNAL_CXX_COMPILER_ID STREQUAL PGI) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 17.1) - message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - endif() - endif() - - # Enforce that extensions are turned off for nvcc_wrapper. - if(INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - message(FATAL_ERROR "NVCC doesn't support C++ extensions. Set CMAKE_CXX_EXTENSIONS to OFF in your CMakeLists.txt.") - endif() - endif() - - if(KOKKOS_ENABLE_CUDA) - # Enforce that the compiler can compile CUDA code. - if(INTERNAL_CXX_COMPILER_ID STREQUAL Clang) - if(INTERNAL_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) - message(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") - endif() - elseif(NOT INTERNAL_CXX_COMPILER_ID STREQUAL NVIDIA) - message(FATAL_ERROR "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang, but compiler ID was ${INTERNAL_CXX_COMPILER_ID}") - endif() - endif() - - set(KOKKOS_CXX_COMPILER ${INTERNAL_CXX_COMPILER} PARENT_SCOPE) - set(KOKKOS_CXX_COMPILER_ID ${INTERNAL_CXX_COMPILER_ID} PARENT_SCOPE) - set(KOKKOS_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION} PARENT_SCOPE) -endfunction() - -#------------------------------------------------------------------------------- -# function(set_kokkos_cxx_standard) -# Transitively enforces that the appropriate CXX standard compile flags (C++11 -# or above) are added to targets that use the Kokkos library. Compile features -# are used if possible. Otherwise, the appropriate flags are added to -# KOKKOS_CXX_FLAGS. Values set by the user to CMAKE_CXX_STANDARD and -# CMAKE_CXX_EXTENSIONS are honored. +# @FUNCTION: kokkos_deprecated_list # -# Outputs: -# KOKKOS_CXX11_FEATURES -# KOKKOS_CXX_FLAGS -# -# Inputs: -# KOKKOS_CXX_COMPILER -# KOKKOS_CXX_COMPILER_ID -# KOKKOS_CXX_COMPILER_VERSION -# -function(set_kokkos_cxx_standard) - # The following table lists the versions of CMake that supports CXX_STANDARD - # and the CXX compile features for different compilers. The versions are - # based on CMake documentation, looking at CMake code, and verifying by - # testing with specific CMake versions. - # - # COMPILER CXX_STANDARD Compile Features - # --------------------------------------------------------------- - # Clang 3.1 3.1 - # GNU 3.1 3.2 - # AppleClang 3.2 3.2 - # Intel 3.6 3.6 - # Cray No No - # PGI No No - # XL No No - # - # For compiling CUDA code using nvcc_wrapper, we will use the host compiler's - # flags for turning on C++11. Since for compiler ID and versioning purposes - # CMake recognizes the host compiler when calling nvcc_wrapper, this just - # works. Both NVCC and nvcc_wrapper only recognize '-std=c++11' which means - # that we can only use host compilers for CUDA builds that use those flags. - # It also means that extensions (gnu++11) can't be turned on for CUDA builds. +# Function that checks if a deprecated list option like Kokkos_ARCH was given. +# This prints an error and prevents configure from completing. +# It attempts to print a helpful message about updating the options for the new CMake. +# Kokkos_${SUFFIX} is the name of the option (like Kokkos_ARCH) being checked. +# Kokkos_${PREFIX}_X is the name of new option to be defined from a list X,Y,Z,... +FUNCTION(kokkos_deprecated_list SUFFIX PREFIX) + SET(CAMEL_NAME Kokkos_${SUFFIX}) + STRING(TOUPPER ${CAMEL_NAME} UC_NAME) - # Check if we can use compile features. - if(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) - if(NOT CMAKE_VERSION VERSION_LESS 3.1) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang OR CMAKE_CXX_COMPILER_ID STREQUAL GNU) - if(NOT CMAKE_VERSION VERSION_LESS 3.2) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Intel) - if(NOT CMAKE_VERSION VERSION_LESS 3.6) - set(INTERNAL_USE_COMPILE_FEATURES ON) - endif() - endif() - endif() + #I don't love doing it this way but better to be safe + FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) + STRING(TOUPPER ${opt} OPT_UC) + IF ("${OPT_UC}" STREQUAL "${UC_NAME}") + STRING(REPLACE "," ";" optlist "${${opt}}") + SET(ERROR_MSG "Given deprecated option list ${opt}. This must now be given as separate -D options, which assuming you spelled options correctly would be:") + FOREACH(entry ${optlist}) + STRING(TOUPPER ${entry} ENTRY_UC) + STRING(APPEND ERROR_MSG "\n -DKokkos_${PREFIX}_${ENTRY_UC}=ON") + ENDFOREACH() + STRING(APPEND ERROR_MSG "\nRemove CMakeCache.txt and re-run. For a list of valid options, refer to BUILD.md or even look at CMakeCache.txt (before deleting it).") + IF (KOKKOS_HAS_TRILINOS) + MESSAGE(WARNING ${ERROR_MSG}) + FOREACH(entry ${optlist}) + STRING(TOUPPER ${entry} ENTRY_UC) + SET(${CAMEL_NAME}_${ENTRY_UC} ON CACHE BOOL "Deprecated Trilinos translation") + ENDFOREACH() + UNSET(${opt} CACHE) + ELSE() + MESSAGE(SEND_ERROR ${ERROR_MSG}) + ENDIF() + ENDIF() + ENDFOREACH() +ENDFUNCTION() - if(INTERNAL_USE_COMPILE_FEATURES) - # Use the compile features aspect of CMake to transitively cause C++ flags - # to populate to user code. +FUNCTION(kokkos_option CAMEL_SUFFIX DEFAULT TYPE DOCSTRING) + SET(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) + STRING(TOUPPER ${CAMEL_NAME} UC_NAME) - # I'm using a hack by requiring features that I know force the lowest version - # of the compilers we want to support. Clang 3.3 and later support all of - # the C++11 standard. With CMake 3.8 and higher, we could switch to using - # cxx_std_11. - set(KOKKOS_CXX11_FEATURES - cxx_nonstatic_member_init # Forces GCC 4.7 or later and Intel 14.0 or later. - PARENT_SCOPE - ) - else() - # CXX compile features are not yet implemented for this combination of - # compiler and version of CMake. + # Make sure this appears in the cache with the appropriate DOCSTRING + SET(${CAMEL_NAME} ${DEFAULT} CACHE ${TYPE} ${DOCSTRING}) - if(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) - # Versions of CMAKE before 3.2 don't support CXX_STANDARD or C++ compile - # features for the AppleClang compiler. Set compiler flags transitively - # here such that they trickle down to a call to target_compile_options(). + #I don't love doing it this way because it's N^2 in number options, but cest la vie + FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) + STRING(TOUPPER ${opt} OPT_UC) + IF ("${OPT_UC}" STREQUAL "${UC_NAME}") + IF (NOT "${opt}" STREQUAL "${CAMEL_NAME}") + IF (KOKKOS_HAS_TRILINOS) + #Allow this for now if Trilinos... we need to bootstrap our way to integration + MESSAGE(WARNING "Deprecated option ${opt} found - please change spelling to ${CAMEL_NAME}") + SET(${CAMEL_NAME} "${${opt}}" CACHE ${TYPE} ${DOCSTRING} FORCE) + UNSET(${opt} CACHE) + ELSE() + MESSAGE(FATAL_ERROR "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies.") + ENDIF() + ENDIF() + ENDIF() + ENDFOREACH() - # The following two blocks of code were copied from - # /Modules/Compiler/AppleClang-CXX.cmake from CMake 3.7.2 and then - # modified. - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.0) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-std=gnu++11") - endif() + #okay, great, we passed the validation test - use the default + IF (DEFINED ${CAMEL_NAME}) + SET(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) + ELSE() + SET(${UC_NAME} ${DEFAULT} PARENT_SCOPE) + ENDIF() - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.1) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++14") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=gnu++14") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.1) - # AppleClang 5.0 knows this flag, but does not set a __cplusplus macro - # greater than 201103L. - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++1y") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=gnu++1y") - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Intel) - # Versions of CMAKE before 3.6 don't support CXX_STANDARD or C++ compile - # features for the Intel compiler. Set compiler flags transitively here - # such that they trickle down to a call to target_compile_options(). +ENDFUNCTION() - # The following three blocks of code were copied from - # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. - if("x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") - set(_std -Qstd) - set(_ext c++) - else() - set(_std -std) - set(_ext gnu++) - endif() +FUNCTION(kokkos_append_config_line LINE) + GLOBAL_APPEND(KOKKOS_TPL_EXPORTS "${LINE}") +ENDFUNCTION() - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.2) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "${_std}=c++14") - # TODO: There is no gnu++14 value supported; figure out what to do. - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "${_std}=c++14") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0.0) - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "${_std}=c++1y") - # TODO: There is no gnu++14 value supported; figure out what to do. - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "${_std}=c++1y") - endif() +MACRO(kokkos_export_cmake_tpl NAME) + #CMake TPLs are located with a call to find_package + #find_package locates XConfig.cmake files through + #X_DIR or X_ROOT variables set prior to calling find_package - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "${_std}=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "${_std}=${_ext}11") - elseif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12.1) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "${_std}=c++0x") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "${_std}=${_ext}0x") - endif() - elseif(CMAKE_CXX_COMPILER_ID STREQUAL Cray) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the Cray - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-hstd=c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-hstd=c++11") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL PGI) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the PGI - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "--c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "--c++11") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL XL) - # CMAKE doesn't support CXX_STANDARD or C++ compile features for the XL - # compiler. Set compiler options transitively here such that they trickle - # down to a call to target_compile_options(). - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX14_STANDARD_COMPILE_OPTION "-std=c++11") - set(INTERNAL_CXX14_EXTENSION_COMPILE_OPTION "-std=c++11") - else() - # Assume GNU. CMAKE_CXX_STANDARD is handled correctly by CMake 3.1 and - # above for this compiler. If the user explicitly requests a C++ - # standard, CMake takes care of it. If not, transitively require C++11. - if(NOT CMAKE_CXX_STANDARD) - set(INTERNAL_CXX11_STANDARD_COMPILE_OPTION ${CMAKE_CXX11_STANDARD_COMPILE_OPTION}) - set(INTERNAL_CXX11_EXTENSION_COMPILE_OPTION ${CMAKE_CXX11_EXTENSION_COMPILE_OPTION}) - endif() - endif() + #If Kokkos was configured to find the TPL through a _DIR variable + #make sure thar DIR variable is available to downstream packages + IF (DEFINED ${NAME}_DIR) + #The downstream project may override the TPL location that Kokkos used + #Check if the downstream project chose its own TPL location + #If not, make the Kokkos found location available + KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_DIR)") + KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_DIR ${${NAME}_DIR})") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") + ENDIF() - # Set the C++ standard info for Kokkos respecting user set values for - # CMAKE_CXX_STANDARD and CMAKE_CXX_EXTENSIONS. - # Only use cxx extension if explicitly requested - if(CMAKE_CXX_STANDARD EQUAL 14) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX14_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX14_STANDARD_COMPILE_OPTION}) - endif() - elseif(CMAKE_CXX_STANDARD EQUAL 11) - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_STANDARD_COMPILE_OPTION}) - endif() - else() - # The user didn't explicitly request a standard, transitively require - # C++11 respecting CMAKE_CXX_EXTENSIONS. - if(DEFINED CMAKE_CXX_EXTENSIONS AND CMAKE_CXX_EXTENSIONS STREQUAL ON) - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_EXTENSION_COMPILE_OPTION}) - else() - set(INTERNAL_CXX_FLAGS ${INTERNAL_CXX11_STANDARD_COMPILE_OPTION}) - endif() - endif() + IF (DEFINED ${NAME}_ROOT) + #The downstream project may override the TPL location that Kokkos used + #Check if the downstream project chose its own TPL location + #If not, make the Kokkos found location available + KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_ROOT)") + KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_ROOT ${${NAME}_ROOT})") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") + ENDIF() + KOKKOS_APPEND_CONFIG_LINE("FIND_DEPENDENCY(${NAME})") +ENDMACRO() - set(KOKKOS_CXX_FLAGS ${INTERNAL_CXX_FLAGS} PARENT_SCOPE) - endif() -endfunction() - - -#------------------------------------------------------------------------------- -# function(set_kokkos_sources) -# Takes a list of sources for kokkos (e.g., KOKKOS_SRC from Makefile.kokkos and -# put it into kokkos_generated_settings.cmake) and sorts the files into the subpackages or -# separate_libraries. This is core and containers (algorithms is pure header -# files). -# -# Inputs: -# KOKKOS_SRC -# -# Outputs: -# KOKKOS_CORE_SRCS -# KOKKOS_CONTAINERS_SRCS -# -function(set_kokkos_srcs) - set(opts ) # no-value args - set(oneValArgs ) - set(multValArgs KOKKOS_SRC) # e.g., lists - cmake_parse_arguments(IN "${opts}" "${oneValArgs}" "${multValArgs}" ${ARGN}) - - foreach(sfile ${IN_KOKKOS_SRC}) - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "" stripfile "${sfile}") - string(REPLACE "/" ";" striplist "${stripfile}") - list(GET striplist 0 firstdir) - if(${firstdir} STREQUAL "core") - list(APPEND KOKKOS_CORE_SRCS ${sfile}) - else() - list(APPEND KOKKOS_CONTAINERS_SRCS ${sfile}) - endif() - endforeach() - set(KOKKOS_CORE_SRCS ${KOKKOS_CORE_SRCS} PARENT_SCOPE) - set(KOKKOS_CONTAINERS_SRCS ${KOKKOS_CONTAINERS_SRCS} PARENT_SCOPE) - return() -endfunction() - -# Setting a default value if it is not already set -macro(set_kokkos_default_default VARIABLE DEFAULT) - IF( "${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}" STREQUAL "" ) - IF( "${KOKKOS_ENABLE_${VARIABLE}}" STREQUAL "" ) - set(KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT ${DEFAULT}) - # MESSAGE(WARNING "Set: KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT to ${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}") +MACRO(kokkos_export_imported_tpl NAME) + IF (NOT KOKKOS_HAS_TRILINOS) + GET_TARGET_PROPERTY(LIB_TYPE ${NAME} TYPE) + IF (${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + # This is not an imported target + # This an interface library that we created + INSTALL( + TARGETS ${NAME} + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) ELSE() - set(KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT ${KOKKOS_ENABLE_${VARIABLE}}) - # MESSAGE(WARNING "Set: KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT to ${KOKKOS_INTERNAL_ENABLE_${VARIABLE}_DEFAULT}") + #make sure this also gets "exported" in the config file + KOKKOS_APPEND_CONFIG_LINE("IF(NOT TARGET ${NAME})") + KOKKOS_APPEND_CONFIG_LINE("ADD_LIBRARY(${NAME} UNKNOWN IMPORTED)") + KOKKOS_APPEND_CONFIG_LINE("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") + + GET_TARGET_PROPERTY(TPL_LIBRARY ${NAME} IMPORTED_LOCATION) + IF(TPL_LIBRARY) + KOKKOS_APPEND_CONFIG_LINE("IMPORTED_LOCATION ${TPL_LIBRARY}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_INCLUDES ${NAME} INTERFACE_INCLUDE_DIRECTORIES) + IF(TPL_INCLUDES) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_INCLUDE_DIRECTORIES ${TPL_INCLUDES}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_COMPILE_OPTIONS ${NAME} INTERFACE_COMPILE_OPTIONS) + IF(TPL_COMPILE_OPTIONS) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_COMPILE_OPTIONS ${TPL_COMPILE_OPTIONS}") + ENDIF() + + SET(TPL_LINK_OPTIONS) + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") + GET_TARGET_PROPERTY(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) + ENDIF() + IF(TPL_LINK_OPTIONS) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_OPTIONS ${TPL_LINK_OPTIONS}") + ENDIF() + + GET_TARGET_PROPERTY(TPL_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) + IF(TPL_LINK_LIBRARIES) + KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_LIBRARIES ${TPL_LINK_LIBRARIES}") + ENDIF() + KOKKOS_APPEND_CONFIG_LINE(")") + KOKKOS_APPEND_CONFIG_LINE("ENDIF()") ENDIF() ENDIF() - UNSET(KOKKOS_ENABLE_${VARIABLE} CACHE) -endmacro() +ENDMACRO() + + +# +# @MACRO: KOKKOS_IMPORT_TPL() +# +# Function that checks if a third-party library (TPL) has been enabled and calls `find_package` +# to create an imported target encapsulating all the flags and libraries +# needed to use the TPL +# +# Usage:: +# +# KOKKOS_IMPORT_TPL( +# +# NO_EXPORT +# INTERFACE +# +# ``NO_EXPORT`` +# +# If specified, this TPL will not be added to KokkosConfig.cmake as an export +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +MACRO(kokkos_import_tpl NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "NO_EXPORT;INTERFACE" + "" + "" + ${ARGN}) + IF (TPL_INTERFACE) + SET(TPL_IMPORTED_NAME ${NAME}) + ELSE() + SET(TPL_IMPORTED_NAME Kokkos::${NAME}) + ENDIF() + + # Even though this policy gets set in the top-level CMakeLists.txt, + # I have still been getting errors about ROOT variables being ignored + # I'm not sure if this is a scope issue - but make sure + # the policy is set before we do any find_package calls + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") + CMAKE_POLICY(SET CMP0074 NEW) + ENDIF() + + IF (KOKKOS_ENABLE_${NAME}) + #Tack on a TPL here to make sure we avoid using anyone else's find + FIND_PACKAGE(TPL${NAME} REQUIRED MODULE) + IF(NOT TARGET ${TPL_IMPORTED_NAME}) + MESSAGE(FATAL_ERROR "Find module succeeded for ${NAME}, but did not produce valid target ${TPL_IMPORTED_NAME}") + ENDIF() + IF(NOT TPL_NO_EXPORT) + KOKKOS_EXPORT_IMPORTED_TPL(${TPL_IMPORTED_NAME}) + ENDIF() + LIST(APPEND KOKKOS_ENABLED_TPLS ${NAME}) + ENDIF() +ENDMACRO(kokkos_import_tpl) + +MACRO(kokkos_import_cmake_tpl MODULE_NAME) + kokkos_import_tpl(${MODULE_NAME} ${ARGN} NO_EXPORT) + CMAKE_PARSE_ARGUMENTS(TPL + "NO_EXPORT" + "OPTION_NAME" + "" + ${ARGN}) + + IF (NOT TPL_OPTION_NAME) + SET(TPL_OPTION_NAME ${MODULE_NAME}) + ENDIF() + + IF (NOT TPL_NO_EXPORT) + KOKKOS_EXPORT_CMAKE_TPL(${MODULE_NAME}) + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_CREATE_IMPORTED_TPL() +# +# Function that creates an imported target encapsulating all the flags +# and libraries needed to use the TPL +# +# Usage:: +# +# KOKKOS_CREATE_IMPORTED_TPL( +# +# INTERFACE +# LIBRARY +# LINK_LIBRARIES ... +# COMPILE_OPTIONS ... +# LINK_OPTIONS ... +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +# +# ``LIBRARY `` +# +# If specified, this gives the IMPORTED_LOCATION of the library. +# +# ``LINK_LIBRARIES ...`` +# +# If specified, this gives a list of dependent libraries that also +# need to be linked against. Each entry can be a library path or +# the name of a valid CMake target. +# +# ``INCLUDES ...`` +# +# If specified, this gives a list of directories that must be added +# to the include path for using this library. +# +# ``COMPILE_OPTIONS ...`` +# +# If specified, this gives a list of compiler flags that must be used +# for using this library. +# +# ``LINK_OPTIONS ...`` +# +# If specified, this gives a list of linker flags that must be used +# for using this library. +MACRO(kokkos_create_imported_tpl NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "INTERFACE" + "LIBRARY" + "LINK_LIBRARIES;INCLUDES;COMPILE_OPTIONS;LINK_OPTIONS" + ${ARGN}) + + + IF (KOKKOS_HAS_TRILINOS) + #TODO: we need to set a bunch of cache variables here + ELSEIF (TPL_INTERFACE) + ADD_LIBRARY(${NAME} INTERFACE) + #Give this an importy-looking name + ADD_LIBRARY(Kokkos::${NAME} ALIAS ${NAME}) + IF (TPL_LIBRARY) + MESSAGE(SEND_ERROR "TPL Interface library ${NAME} should not have an IMPORTED_LOCATION") + ENDIF() + #Things have to go in quoted in case we have multiple list entries + IF(TPL_LINK_LIBRARIES) + TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_LIBRARIES}) + ENDIF() + IF(TPL_INCLUDES) + TARGET_INCLUDE_DIRECTORIES(${NAME} INTERFACE ${TPL_INCLUDES}) + ENDIF() + IF(TPL_COMPILE_OPTIONS) + TARGET_COMPILE_OPTIONS(${NAME} INTERFACE ${TPL_COMPILE_OPTIONS}) + ENDIF() + IF(TPL_LINK_OPTIONS) + TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_OPTIONS}) + ENDIF() + ELSE() + ADD_LIBRARY(${NAME} UNKNOWN IMPORTED) + IF(TPL_LIBRARY) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + IMPORTED_LOCATION ${TPL_LIBRARY}) + ENDIF() + #Things have to go in quoted in case we have multiple list entries + IF(TPL_LINK_LIBRARIES) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_LINK_LIBRARIES "${TPL_LINK_LIBRARIES}") + ENDIF() + IF(TPL_INCLUDES) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${TPL_INCLUDES}") + ENDIF() + IF(TPL_COMPILE_OPTIONS) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_COMPILE_OPTIONS "${TPL_COMPILE_OPTIONS}") + ENDIF() + IF(TPL_LINK_OPTIONS) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_LINK_LIBRARIES "${TPL_LINK_OPTIONS}") + ENDIF() + ENDIF() +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_HEADER +# +# Function that finds a particular header. This searches custom paths +# or default system paths depending on options. In constrast to CMake +# default, custom paths are prioritized over system paths. The searched +# order is: +# 1. _ROOT variable +# 2. _ROOT environment variable +# 3. Kokkos__DIR variable +# 4. Locations in the PATHS option +# 5. Default system paths, if allowed. +# +# Default system paths are allowed if none of options (1)-(4) are specified +# or if default paths are specifically allowed via ALLOW_SYSTEM_PATH_FALLBACK +# +# Usage:: +# +# KOKKOS_FIND_HEADER( +# +#
+# +# [ALLOW_SYSTEM_PATH_FALLBACK] +# [PATHS path1 [path2 ...]] +# ) +# +# ```` +# +# The variable to define with the success or failure of the find +# +# ``
`` +# +# The name of the header to find +# +# ```` +# +# The name of the TPL the header corresponds to +# +# ``[ALLOW_SYSTEM_PATH_FALLBACK]`` +# +# If custom paths are given and the header is not found +# should we be allowed to search default system paths +# or error out if not found in given paths +# +# ``[PATHS path1 [path2 ...]]`` +# +# Custom paths to search for the header +# +MACRO(kokkos_find_header VAR_NAME HEADER TPL_NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "ALLOW_SYSTEM_PATH_FALLBACK" + "" + "PATHS" + ${ARGN}) + + SET(${VAR_NAME} "${VARNAME}-NOTFOUND") + SET(HAVE_CUSTOM_PATHS FALSE) + + IF(DEFINED ${TPL_NAME}_ROOT OR + DEFINED ENV{${TPL_NAME}_ROOT} OR + DEFINED KOKKOS_${TPL_NAME}_DIR OR + TPL_PATHS) + FIND_PATH(${VAR_NAME} ${HEADER} + PATHS + ${${TPL_NAME}_ROOT} + $ENV{${TPL_NAME}_ROOT} + ${KOKKOS_${TPL_NAME}_DIR} + ${TPL_PATHS} + PATH_SUFFIXES include + NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + #No-op if ${VAR_NAME} set by previous call + FIND_PATH(${VAR_NAME} ${HEADER}) + ENDIF() + +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_LIBRARY +# +# Function that find a particular library. This searches custom paths +# or default system paths depending on options. In constrast to CMake +# default, custom paths are prioritized over system paths. The search +# order is: +# 1. _ROOT variable +# 2. _ROOT environment variable +# 3. Kokkos__DIR variable +# 4. Locations in the PATHS option +# 5. Default system paths, if allowed. +# +# Default system paths are allowed if none of options (1)-(3) are specified +# or if default paths are specifically allowed via ALLOW_SYSTEM_PATH_FALLBACK +# +# Usage:: +# +# KOKKOS_FIND_LIBRARY( +# +#
+# +# [ALLOW_SYSTEM_PATH_FALLBACK] +# [PATHS path1 [path2 ...]] +# [SUFFIXES suffix1 [suffix2 ...]] +# ) +# +# ```` +# +# The variable to define with the success or failure of the find +# +# ```` +# +# The name of the library to find (NOT prefixed with -l) +# +# ```` +# +# The name of the TPL the library corresponds to +# +# ``ALLOW_SYSTEM_PATH_FALLBACK`` +# +# If custom paths are given and the library is not found +# should we be allowed to search default system paths +# or error out if not found in given paths +# +# ``PATHS`` +# +# Custom paths to search for the library +# +# ``SUFFIXES`` +# +# Suffixes appended to PATHS when attempting to locate +# the library. Defaults to {lib, lib64}. +# +MACRO(kokkos_find_library VAR_NAME LIB TPL_NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "ALLOW_SYSTEM_PATH_FALLBACK" + "" + "PATHS;SUFFIXES" + ${ARGN}) + + IF(NOT TPL_SUFFIXES) + SET(TPL_SUFFIXES lib lib64) + ENDIF() + + SET(${VAR_NAME} "${VARNAME}-NOTFOUND") + SET(HAVE_CUSTOM_PATHS FALSE) + + IF(DEFINED ${TPL_NAME}_ROOT OR + DEFINED ENV{${TPL_NAME}_ROOT} OR + DEFINED KOKKOS_${TPL_NAME}_DIR OR + TPL_PATHS) + FIND_LIBRARY(${VAR_NAME} ${LIB} + PATHS + ${${TPL_NAME}_ROOT} + $ENV{${TPL_NAME}_ROOT} + ${KOKKOS_${TPL_NAME}_DIR} + ${TPL_PATHS} + PATH_SUFFIXES + ${TPL_SUFFIXES} + NO_DEFAULT_PATH) + SET(HAVE_CUSTOM_PATHS TRUE) + ENDIF() + + IF(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + #No-op if ${VAR_NAME} set by previous call + FIND_LIBRARY(${VAR_NAME} ${LIB} PATH_SUFFIXES ${TPL_SUFFIXES}) + ENDIF() + +ENDMACRO() + +# +# @MACRO: KOKKOS_FIND_IMPORTED +# +# Function that finds all libraries and headers needed for the tpl +# and creates an imported target encapsulating all the flags and libraries +# +# Usage:: +# +# KOKKOS_FIND_IMPORTED( +# +# INTERFACE +# ALLOW_SYSTEM_PATH_FALLBACK +# MODULE_NAME +# IMPORTED_NAME +# LIBRARY +# LIBRARIES ... +# LIBRARY_PATHS ... +# LIBRARY_SUFFIXES ... +# HEADER +# HEADERS ... +# HEADER_PATHS ... +# ) +# +# ``INTERFACE`` +# +# If specified, this TPL will build an INTERFACE library rather than an +# IMPORTED target +# +# ``ALLOW_SYSTEM_PATH_FALLBACK`` +# +# If custom paths are given and the library is not found +# should we be allowed to search default system paths +# or error out if not found in given paths. +# +# ``MODULE_NAME `` +# +# If specified, the name of the enclosing module passed to +# FIND_PACKAGE(). Defaults to TPL${NAME} if not +# given. +# +# ``IMPORTED_NAME `` +# +# If specified, this gives the name of the target to build. +# Defaults to Kokkos:: +# +# ``LIBRARY `` +# +# If specified, this gives the name of the library to look for +# +# ``LIBRARIES ...`` +# +# If specified, this gives a list of libraries to find for the package +# +# ``LIBRARY_PATHS ...`` +# +# If specified, this gives a list of paths to search for the library. +# If not given, _ROOT will be searched. +# +# ``LIBRARY_SUFFIXES ...`` +# +# Suffixes appended to LIBRARY_PATHS when attempting to locate +# libraries. If not given, defaults to {lib, lib64}. +# +# ``HEADER `` +# +# If specified, this gives the name of a header to to look for +# +# ``HEADERS ...`` +# +# If specified, this gives a list of headers to find for the package +# +# ``HEADER_PATHS ...`` +# +# If specified, this gives a list of paths to search for the headers +# If not given, _ROOT/include and _ROOT/include will be searched. +# +MACRO(kokkos_find_imported NAME) + CMAKE_PARSE_ARGUMENTS(TPL + "INTERFACE;ALLOW_SYSTEM_PATH_FALLBACK" + "IMPORTED_NAME;MODULE_NAME;LIBRARY;HEADER" + "LIBRARIES;LIBRARY_PATHS;LIBRARY_SUFFIXES;HEADERS;HEADER_PATHS" + ${ARGN}) + + IF(NOT TPL_MODULE_NAME) + SET(TPL_MODULE_NAME TPL${NAME}) + ENDIF() + + IF (TPL_ALLOW_SYSTEM_PATH_FALLBACK) + SET(ALLOW_PATH_FALLBACK_OPT ALLOW_SYSTEM_PATH_FALLBACK) + ELSE() + SET(ALLOW_PATH_FALLBACK_OPT) + ENDIF() + + IF (NOT TPL_IMPORTED_NAME) + IF (TPL_INTERFACE) + SET(TPL_IMPORTED_NAME ${NAME}) + ELSE() + SET(TPL_IMPORTED_NAME Kokkos::${NAME}) + ENDIF() + ENDIF() + + IF (NOT TPL_LIBRARY_SUFFIXES) + SET(TPL_LIBRARY_SUFFIXES lib lib64) + ENDIF() + + SET(${NAME}_INCLUDE_DIRS) + IF (TPL_HEADER) + KOKKOS_FIND_HEADER(${NAME}_INCLUDE_DIRS ${TPL_HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + ENDIF() + + FOREACH(HEADER ${TPL_HEADERS}) + KOKKOS_FIND_HEADER(HEADER_FIND_TEMP ${HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + IF(HEADER_FIND_TEMP) + LIST(APPEND ${NAME}_INCLUDE_DIRS ${HEADER_FIND_TEMP}) + ENDIF() + ENDFOREACH() + + SET(${NAME}_LIBRARY) + IF(TPL_LIBRARY) + KOKKOS_FIND_LIBRARY(${NAME}_LIBRARY ${TPL_LIBRARY} ${NAME} + ${ALLOW_PATH_FALLBACK_OPT} + PATHS ${TPL_LIBRARY_PATHS} + SUFFIXES ${TPL_LIBRARY_SUFFIXES}) + ENDIF() + + SET(${NAME}_FOUND_LIBRARIES) + FOREACH(LIB ${TPL_LIBRARIES}) + KOKKOS_FIND_LIBRARY(${LIB}_LOCATION ${LIB} ${NAME} + ${ALLOW_PATH_FALLBACK_OPT} + PATHS ${TPL_LIBRARY_PATHS} + SUFFIXES ${TPL_LIBRARY_SUFFIXES}) + IF(${LIB}_LOCATION) + LIST(APPEND ${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + ELSE() + SET(${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + BREAK() + ENDIF() + ENDFOREACH() + + INCLUDE(FindPackageHandleStandardArgs) + #Collect all the variables we need to be valid for + #find_package to have succeeded + SET(TPL_VARS_NEEDED) + IF (TPL_LIBRARY) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_LIBRARY) + ENDIF() + IF(TPL_HEADER) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_INCLUDE_DIRS) + ENDIF() + IF(TPL_LIBRARIES) + LIST(APPEND TPL_VARS_NEEDED ${NAME}_FOUND_LIBRARIES) + ENDIF() + FIND_PACKAGE_HANDLE_STANDARD_ARGS(${TPL_MODULE_NAME} REQUIRED_VARS ${TPL_VARS_NEEDED}) + + MARK_AS_ADVANCED(${NAME}_INCLUDE_DIRS ${NAME}_FOUND_LIBRARIES ${NAME}_LIBRARY) + + #this is so much fun on a Cray system + #/usr/include should never be added as a -isystem include + #this freaks out the compiler include search order + IF (KOKKOS_IS_CRAYPE) + LIST(REMOVE_ITEM ${NAME}_INCLUDE_DIRS "/usr/include") + ENDIF() + + IF (${TPL_MODULE_NAME}_FOUND) + SET(IMPORT_TYPE) + IF (TPL_INTERFACE) + SET(IMPORT_TYPE "INTERFACE") + ENDIF() + KOKKOS_CREATE_IMPORTED_TPL(${TPL_IMPORTED_NAME} + ${IMPORT_TYPE} + INCLUDES "${${NAME}_INCLUDE_DIRS}" + LIBRARY "${${NAME}_LIBRARY}" + LINK_LIBRARIES "${${NAME}_FOUND_LIBRARIES}") + ENDIF() +ENDMACRO(kokkos_find_imported) + +# +# @MACRO: KOKKOS_LINK_TPL() +# +# Function that checks if a third-party library (TPL) has been enabled and +# calls target_link_libraries on the given target +# +# Usage:: +# +# KOKKOS_LINK_TPL( +# +# PUBLIC +# PRIVATE +# INTERFACE +# IMPORTED_NAME +# +# +# Checks if Kokkos_ENABLE_=ON and if so links the library +# +# ``PUBLIC/PRIVATE/INTERFACE`` +# +# Specifies the linkage mode. One of these arguments should be given. +# This will then invoke target_link_libraries( PUBLIC/PRIVATE/INTERFACE ) +# +# ``IMPORTED_NAME `` +# +# If specified, this gives the exact name of the target to link against +# target_link_libraries( ) +# +FUNCTION(kokkos_link_tpl TARGET) + CMAKE_PARSE_ARGUMENTS(TPL + "PUBLIC;PRIVATE;INTERFACE" + "IMPORTED_NAME" + "" + ${ARGN}) + #the name of the TPL + SET(TPL ${TPL_UNPARSED_ARGUMENTS}) + IF (KOKKOS_HAS_TRILINOS) + #Do nothing, they will have already been linked + ELSE() + IF (NOT TPL_IMPORTED_NAME) + SET(TPL_IMPORTED_NAME Kokkos::${TPL}) + ENDIF() + IF (KOKKOS_ENABLE_${TPL}) + IF (TPL_PUBLIC) + TARGET_LINK_LIBRARIES(${TARGET} PUBLIC ${TPL_IMPORTED_NAME}) + ELSEIF (TPL_PRIVATE) + TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ${TPL_IMPORTED_NAME}) + ELSEIF (TPL_INTERFACE) + TARGET_LINK_LIBRARIES(${TARGET} INTERFACE ${TPL_IMPORTED_NAME}) + ELSE() + TARGET_LINK_LIBRARIES(${TARGET} ${TPL_IMPORTED_NAME}) + ENDIF() + ENDIF() + ENDIF() +ENDFUNCTION() + +FUNCTION(COMPILER_SPECIFIC_OPTIONS_HELPER) + SET(COMPILERS NVIDIA PGI XL DEFAULT Cray Intel Clang AppleClang GNU) + CMAKE_PARSE_ARGUMENTS( + PARSE + "LINK_OPTIONS;COMPILE_OPTIONS;COMPILE_DEFINITIONS;LINK_LIBRARIES" + "" + "${COMPILERS}" + ${ARGN}) + IF(PARSE_UNPARSED_ARGUMENTS) + MESSAGE(SEND_ERROR "'${PARSE_UNPARSED_ARGUMENTS}' argument(s) not recognized when providing compiler specific options") + ENDIF() + + SET(COMPILER ${KOKKOS_CXX_COMPILER_ID}) + + SET(COMPILER_SPECIFIC_FLAGS_TMP) + FOREACH(COMP ${COMPILERS}) + IF (COMPILER STREQUAL "${COMP}") + IF (PARSE_${COMPILER}) + IF (NOT "${PARSE_${COMPILER}}" STREQUAL "NO-VALUE-SPECIFIED") + SET(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_${COMPILER}}) + ENDIF() + ELSEIF(PARSE_DEFAULT) + SET(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_DEFAULT}) + ENDIF() + ENDIF() + ENDFOREACH() + + IF (PARSE_COMPILE_OPTIONS) + # The funky logic here is for future handling of argument deduplication + # If we naively pass multiple -Xcompiler flags to target_compile_options + # -Xcompiler will get deduplicated and break the build + IF ("-Xcompiler" IN_LIST COMPILER_SPECIFIC_FLAGS_TMP) + LIST(REMOVE_ITEM COMPILER_SPECIFIC_FLAGS_TMP "-Xcompiler") + GLOBAL_APPEND(KOKKOS_XCOMPILER_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + ELSE() + GLOBAL_APPEND(KOKKOS_COMPILE_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + ENDIF() + ENDIF() + + IF (PARSE_LINK_OPTIONS) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + ENDIF() + + IF (PARSE_COMPILE_DEFINITIONS) + GLOBAL_APPEND(KOKKOS_COMPILE_DEFINITIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + ENDIF() + + IF (PARSE_LINK_LIBRARIES) + GLOBAL_APPEND(KOKKOS_LINK_LIBRARIES ${COMPILER_SPECIFIC_FLAGS_TMP}) + ENDIF() +ENDFUNCTION(COMPILER_SPECIFIC_OPTIONS_HELPER) + +FUNCTION(COMPILER_SPECIFIC_FLAGS) + COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} COMPILE_OPTIONS LINK_OPTIONS) +ENDFUNCTION(COMPILER_SPECIFIC_FLAGS) + +FUNCTION(COMPILER_SPECIFIC_DEFS) + COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} COMPILE_DEFINITIONS) +ENDFUNCTION(COMPILER_SPECIFIC_DEFS) + +FUNCTION(COMPILER_SPECIFIC_LIBS) + COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} LINK_LIBRARIES) +ENDFUNCTION(COMPILER_SPECIFIC_LIBS) diff --git a/lib/kokkos/cmake/kokkos_install.cmake b/lib/kokkos/cmake/kokkos_install.cmake new file mode 100644 index 0000000000..97bb2bd0b0 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_install.cmake @@ -0,0 +1,48 @@ +INCLUDE(CMakePackageConfigHelpers) +IF (NOT KOKKOS_HAS_TRILINOS) + INCLUDE(GNUInstallDirs) + + #Set all the variables needed for KokkosConfig.cmake + GET_PROPERTY(KOKKOS_PROP_LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + SET(KOKKOS_LIBRARIES ${KOKKOS_PROP_LIBS}) + + INCLUDE(CMakePackageConfigHelpers) + CONFIGURE_PACKAGE_CONFIG_FILE( + cmake/KokkosConfig.cmake.in + "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + + CONFIGURE_PACKAGE_CONFIG_FILE( + cmake/KokkosConfigCommon.cmake.in + "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + + WRITE_BASIC_PACKAGE_VERSION_FILE("${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" + VERSION "${Kokkos_VERSION}" + COMPATIBILITY SameMajorVersion) + + # Install the KokkosConfig*.cmake files + install(FILES + "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" + "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + "${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) + install(EXPORT KokkosTargets NAMESPACE Kokkos:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) +ELSE() + CONFIGURE_FILE(cmake/KokkosConfigCommon.cmake.in ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake @ONLY) + file(READ ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake KOKKOS_CONFIG_COMMON) + file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/KokkosConfig_install.cmake" "${KOKKOS_CONFIG_COMMON}") + CONFIGURE_FILE(cmake/KokkosTrilinosConfig.cmake.in ${Kokkos_BINARY_DIR}/KokkosTrilinosConfig.cmake @ONLY) + file(READ ${Kokkos_BINARY_DIR}/KokkosTrilinosConfig.cmake KOKKOS_TRILINOS_CONFIG) + file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/KokkosConfig_install.cmake" "${KOKKOS_TRILINOS_CONFIG}") + + WRITE_BASIC_PACKAGE_VERSION_FILE("${CMAKE_CURRENT_BINARY_DIR}/KokkosConfigVersion.cmake" + VERSION "${Kokkos_VERSION}" + COMPATIBILITY SameMajorVersion) + + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosConfigVersion.cmake + DESTINATION "${${PROJECT_NAME}_INSTALL_LIB_DIR}/cmake/${PACKAGE_NAME}") +ENDIF() + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h DESTINATION ${KOKKOS_HEADER_DIR}) + diff --git a/lib/kokkos/cmake/kokkos_options.cmake b/lib/kokkos/cmake/kokkos_options.cmake deleted file mode 100644 index 239301925c..0000000000 --- a/lib/kokkos/cmake/kokkos_options.cmake +++ /dev/null @@ -1,419 +0,0 @@ -########################## NOTES ############################################### -# List the options for configuring kokkos using CMake method of doing it. -# These options then get mapped onto KOKKOS_SETTINGS environment variable by -# kokkos_settings.cmake. It is separate to allow other packages to override -# these variables (e.g., TriBITS). - -########################## AVAILABLE OPTIONS ################################### -# Use lists for documentation, verification, and programming convenience - -# All CMake options of the type KOKKOS_ENABLE_* -set(KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST) -list(APPEND KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST - Serial - OpenMP - Pthread - Qthread - HPX - Cuda - ROCm - HWLOC - MEMKIND - LIBRT - Cuda_Lambda - Cuda_Relocatable_Device_Code - Cuda_UVM - Cuda_LDG_Intrinsic - HPX_ASYNC_DISPATCH - Debug - Debug_DualView_Modify_Check - Debug_Bounds_Check - Compiler_Warnings - Profiling - Profiling_Load_Print - Aggressive_Vectorization - Deprecated_Code - Explicit_Instantiation - ) - -#------------------------------------------------------------------------------- -#------------------------------- Recognize CamelCase Options --------------------------- -#------------------------------------------------------------------------------- - -foreach(opt ${KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST}) - string(TOUPPER ${opt} OPT ) - IF(DEFINED Kokkos_ENABLE_${opt}) - IF(DEFINED KOKKOS_ENABLE_${OPT}) - IF(NOT ("${KOKKOS_ENABLE_${OPT}}" STREQUAL "${Kokkos_ENABLE_${opt}}")) - IF(DEFINED KOKKOS_ENABLE_${OPT}_INTERNAL) - MESSAGE(WARNING "Defined both Kokkos_ENABLE_${opt}=[${Kokkos_ENABLE_${opt}}] and KOKKOS_ENABLE_${OPT}=[${KOKKOS_ENABLE_${OPT}}] and they differ! Could be caused by old CMakeCache Variable. Run CMake again and warning should disappear. If not you are truly setting both variables.") - IF(NOT ("${Kokkos_ENABLE_${opt}}" STREQUAL "${KOKKOS_ENABLE_${OPT}_INTERNAL}")) - UNSET(KOKKOS_ENABLE_${OPT} CACHE) - SET(KOKKOS_ENABLE_${OPT} ${Kokkos_ENABLE_${opt}}) - MESSAGE(WARNING "SET BOTH VARIABLES KOKKOS_ENABLE_${OPT}: ${KOKKOS_ENABLE_${OPT}}") - ELSE() - SET(Kokkos_ENABLE_${opt} ${KOKKOS_ENABLE_${OPT}}) - ENDIF() - ELSE() - MESSAGE(FATAL_ERROR "Defined both Kokkos_ENABLE_${opt}=[${Kokkos_ENABLE_${opt}}] and KOKKOS_ENABLE_${OPT}=[${KOKKOS_ENABLE_${OPT}}] and they differ!") - ENDIF() - ENDIF() - ELSE() - SET(KOKKOS_INTERNAL_ENABLE_${OPT}_DEFAULT ${Kokkos_ENABLE_${opt}}) - ENDIF() - ENDIF() -endforeach() - -IF(DEFINED Kokkos_ARCH) - MESSAGE(FATAL_ERROR "Defined Kokkos_ARCH, use KOKKOS_ARCH instead!") -ENDIF() -IF(DEFINED Kokkos_Arch) - MESSAGE(FATAL_ERROR "Defined Kokkos_Arch, use KOKKOS_ARCH instead!") -ENDIF() - -#------------------------------------------------------------------------------- -# List of possible host architectures. -#------------------------------------------------------------------------------- -set(KOKKOS_ARCH_LIST) -list(APPEND KOKKOS_ARCH_LIST - None # No architecture optimization - AMDAVX # (HOST) AMD chip - EPYC # (HOST) AMD EPYC Zen-Core CPU - ARMv80 # (HOST) ARMv8.0 Compatible CPU - ARMv81 # (HOST) ARMv8.1 Compatible CPU - ARMv8-ThunderX # (HOST) ARMv8 Cavium ThunderX CPU - ARMv8-TX2 # (HOST) ARMv8 Cavium ThunderX2 CPU - WSM # (HOST) Intel Westmere CPU - SNB # (HOST) Intel Sandy/Ivy Bridge CPUs - HSW # (HOST) Intel Haswell CPUs - BDW # (HOST) Intel Broadwell Xeon E-class CPUs - SKX # (HOST) Intel Sky Lake Xeon E-class HPC CPUs (AVX512) - KNC # (HOST) Intel Knights Corner Xeon Phi - KNL # (HOST) Intel Knights Landing Xeon Phi - BGQ # (HOST) IBM Blue Gene Q - Power7 # (HOST) IBM POWER7 CPUs - Power8 # (HOST) IBM POWER8 CPUs - Power9 # (HOST) IBM POWER9 CPUs - Kepler # (GPU) NVIDIA Kepler default (generation CC 3.5) - Kepler30 # (GPU) NVIDIA Kepler generation CC 3.0 - Kepler32 # (GPU) NVIDIA Kepler generation CC 3.2 - Kepler35 # (GPU) NVIDIA Kepler generation CC 3.5 - Kepler37 # (GPU) NVIDIA Kepler generation CC 3.7 - Maxwell # (GPU) NVIDIA Maxwell default (generation CC 5.0) - Maxwell50 # (GPU) NVIDIA Maxwell generation CC 5.0 - Maxwell52 # (GPU) NVIDIA Maxwell generation CC 5.2 - Maxwell53 # (GPU) NVIDIA Maxwell generation CC 5.3 - Pascal60 # (GPU) NVIDIA Pascal generation CC 6.0 - Pascal61 # (GPU) NVIDIA Pascal generation CC 6.1 - Volta70 # (GPU) NVIDIA Volta generation CC 7.0 - Volta72 # (GPU) NVIDIA Volta generation CC 7.2 - Turing75 # (GPU) NVIDIA Turing generation CC 7.5 - ) - -# List of possible device architectures. -# The case and spelling here needs to match Makefile.kokkos -set(KOKKOS_DEVICES_LIST) -# Options: Cuda,ROCm,OpenMP,Pthread,Qthreads,Serial -list(APPEND KOKKOS_DEVICES_LIST - Cuda # NVIDIA GPU -- see below - OpenMP # OpenMP - Pthread # pthread - Qthreads # qthreads - HPX # HPX - Serial # serial - ROCm # Relocatable device code - ) - -# List of possible TPLs for Kokkos -# From Makefile.kokkos: Options: hwloc,librt,experimental_memkind -set(KOKKOS_USE_TPLS_LIST) -if(APPLE) -list(APPEND KOKKOS_USE_TPLS_LIST - HWLOC # hwloc - MEMKIND # experimental_memkind - ) -else() -list(APPEND KOKKOS_USE_TPLS_LIST - HWLOC # hwloc - LIBRT # librt - MEMKIND # experimental_memkind - ) -endif() -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_HWLOC hwloc) -set(KOKKOS_INTERNAL_LIBRT librt) -set(KOKKOS_INTERNAL_MEMKIND experimental_memkind) - -# List of possible Advanced options -set(KOKKOS_OPTIONS_LIST) -list(APPEND KOKKOS_OPTIONS_LIST - AGGRESSIVE_VECTORIZATION - DISABLE_PROFILING - DISABLE_DUALVIEW_MODIFY_CHECK - ENABLE_PROFILE_LOAD_PRINT - ) -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_LDG_INTRINSIC use_ldg) -set(KOKKOS_INTERNAL_UVM librt) -set(KOKKOS_INTERNAL_RELOCATABLE_DEVICE_CODE rdc) - - -#------------------------------------------------------------------------------- -# List of possible Options for CUDA -#------------------------------------------------------------------------------- -# From Makefile.kokkos: Options: use_ldg,force_uvm,rdc -set(KOKKOS_CUDA_OPTIONS_LIST) -list(APPEND KOKKOS_CUDA_OPTIONS_LIST - LDG_INTRINSIC # use_ldg - UVM # force_uvm - RELOCATABLE_DEVICE_CODE # rdc - LAMBDA # enable_lambda - ) - -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_LDG_INTRINSIC use_ldg) -set(KOKKOS_INTERNAL_UVM force_uvm) -set(KOKKOS_INTERNAL_RELOCATABLE_DEVICE_CODE rdc) -set(KOKKOS_INTERNAL_LAMBDA enable_lambda) - - -#------------------------------------------------------------------------------- -# List of possible Options for HPX -#------------------------------------------------------------------------------- -# From Makefile.kokkos: Options: enable_async_dispatch -set(KOKKOS_HPX_OPTIONS_LIST) -list(APPEND KOKKOS_HPX_OPTIONS_LIST - ASYNC_DISPATCH # enable_async_dispatch - ) - -# Map of cmake variables to Makefile variables -set(KOKKOS_INTERNAL_ENABLE_ASYNC_DISPATCH enable_async_dispatch) - - -#------------------------------------------------------------------------------- -#------------------------------- Create doc strings ---------------------------- -#------------------------------------------------------------------------------- - -set(tmpr "\n ") -string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_ARCH_DOCSTR "${KOKKOS_ARCH_LIST}") -set(KOKKOS_INTERNAL_ARCH_DOCSTR "${tmpr}${KOKKOS_INTERNAL_ARCH_DOCSTR}") -# This would be useful, but we use Foo_ENABLE mechanisms -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_DEVICES_DOCSTR "${KOKKOS_DEVICES_LIST}") -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_USE_TPLS_DOCSTR "${KOKKOS_USE_TPLS_LIST}") -#string(REPLACE ";" ${tmpr} KOKKOS_INTERNAL_CUDA_OPTIONS_DOCSTR "${KOKKOS_CUDA_OPTIONS_LIST}") - -#------------------------------------------------------------------------------- -#------------------------------- GENERAL OPTIONS ------------------------------- -#------------------------------------------------------------------------------- - -# Setting this variable to a value other than "None" can improve host -# performance by turning on architecture specific code. -# NOT SET is used to determine if the option is passed in. It is reset to -# default "None" down below. -set(KOKKOS_ARCH "NOT_SET" CACHE STRING - "Optimize for specific host architecture. Options are: ${KOKKOS_INTERNAL_ARCH_DOCSTR}") - -# Whether to build separate libraries or now -set(KOKKOS_SEPARATE_LIBS OFF CACHE BOOL "OFF = kokkos. ON = kokkoscore, kokkoscontainers, and kokkosalgorithms.") - -# Qthreads options. -set(KOKKOS_QTHREADS_DIR "" CACHE PATH "Location of Qthreads library.") - -# HPX options. -set(KOKKOS_HPX_DIR "" CACHE PATH "Location of HPX library.") - -# Whether to build separate libraries or now -set(KOKKOS_SEPARATE_TESTS OFF CACHE BOOL "Provide unit test targets with finer granularity.") - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_DEVICES -------------------------------- -#------------------------------------------------------------------------------- -# Figure out default settings -IF(Trilinos_ENABLE_Kokkos) - set_kokkos_default_default(SERIAL ON) - set_kokkos_default_default(PTHREAD OFF) - IF(TPL_ENABLE_QTHREAD) - set_kokkos_default_default(QTHREADS ${TPL_ENABLE_QTHREAD}) - ELSE() - set_kokkos_default_default(QTHREADS OFF) - ENDIF() - IF(TPL_ENABLE_HPX) - set_kokkos_default_default(HPX ON) - ELSE() - set_kokkos_default_default(HPX OFF) - ENDIF() - IF(Trilinos_ENABLE_OpenMP) - set_kokkos_default_default(OPENMP ${Trilinos_ENABLE_OpenMP}) - ELSE() - set_kokkos_default_default(OPENMP OFF) - ENDIF() - IF(TPL_ENABLE_CUDA) - set_kokkos_default_default(CUDA ${TPL_ENABLE_CUDA}) - ELSE() - set_kokkos_default_default(CUDA OFF) - ENDIF() - set_kokkos_default_default(ROCM OFF) -ELSE() - set_kokkos_default_default(SERIAL ON) - set_kokkos_default_default(OPENMP OFF) - set_kokkos_default_default(PTHREAD OFF) - set_kokkos_default_default(QTHREAD OFF) - set_kokkos_default_default(HPX OFF) - set_kokkos_default_default(CUDA OFF) - set_kokkos_default_default(ROCM OFF) -ENDIF() - -# Set which Kokkos backend to use. -# These are the actual options that define the settings. -set(KOKKOS_ENABLE_SERIAL ${KOKKOS_INTERNAL_ENABLE_SERIAL_DEFAULT} CACHE BOOL "Whether to enable the Kokkos::Serial device. This device executes \"parallel\" kernels sequentially on a single CPU thread. It is enabled by default. If you disable this device, please enable at least one other CPU device, such as Kokkos::OpenMP or Kokkos::Threads.") -set(KOKKOS_ENABLE_OPENMP ${KOKKOS_INTERNAL_ENABLE_OPENMP_DEFAULT} CACHE BOOL "Enable OpenMP support in Kokkos." FORCE) -set(KOKKOS_ENABLE_PTHREAD ${KOKKOS_INTERNAL_ENABLE_PTHREAD_DEFAULT} CACHE BOOL "Enable Pthread support in Kokkos.") -set(KOKKOS_ENABLE_QTHREADS ${KOKKOS_INTERNAL_ENABLE_QTHREADS_DEFAULT} CACHE BOOL "Enable Qthreads support in Kokkos.") -set(KOKKOS_ENABLE_HPX ${KOKKOS_INTERNAL_ENABLE_HPX_DEFAULT} CACHE BOOL "Enable HPX support in Kokkos.") -set(KOKKOS_ENABLE_CUDA ${KOKKOS_INTERNAL_ENABLE_CUDA_DEFAULT} CACHE BOOL "Enable CUDA support in Kokkos.") -set(KOKKOS_ENABLE_ROCM ${KOKKOS_INTERNAL_ENABLE_ROCM_DEFAULT} CACHE BOOL "Enable ROCm support in Kokkos.") - - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS DEBUG and PROFILING -------------------- -#------------------------------------------------------------------------------- - -# Debug related options enable compiler warnings - -set_kokkos_default_default(DEBUG OFF) -set(KOKKOS_ENABLE_DEBUG ${KOKKOS_INTERNAL_ENABLE_DEBUG_DEFAULT} CACHE BOOL "Enable Kokkos Debug.") - -# From Makefile.kokkos: Advanced Options: -#compiler_warnings, aggressive_vectorization, disable_profiling, disable_dualview_modify_check, enable_profile_load_print -set_kokkos_default_default(COMPILER_WARNINGS OFF) -set(KOKKOS_ENABLE_COMPILER_WARNINGS ${KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS_DEFAULT} CACHE BOOL "Enable compiler warnings.") - -set_kokkos_default_default(DEBUG_DUALVIEW_MODIFY_CHECK OFF) -set(KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK ${KOKKOS_INTERNAL_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK_DEFAULT} CACHE BOOL "Enable dualview modify check.") - -# Enable aggressive vectorization. -set_kokkos_default_default(AGGRESSIVE_VECTORIZATION OFF) -set(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION ${KOKKOS_INTERNAL_ENABLE_AGGRESSIVE_VECTORIZATION_DEFAULT} CACHE BOOL "Enable aggressive vectorization.") - -# Enable profiling. -set_kokkos_default_default(PROFILING ON) -set(KOKKOS_ENABLE_PROFILING ${KOKKOS_INTERNAL_ENABLE_PROFILING_DEFAULT} CACHE BOOL "Enable profiling.") - -set_kokkos_default_default(PROFILING_LOAD_PRINT OFF) -set(KOKKOS_ENABLE_PROFILING_LOAD_PRINT ${KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT_DEFAULT} CACHE BOOL "Enable profile load print.") - -set_kokkos_default_default(DEPRECATED_CODE ON) -set(KOKKOS_ENABLE_DEPRECATED_CODE ${KOKKOS_INTERNAL_ENABLE_DEPRECATED_CODE_DEFAULT} CACHE BOOL "Enable deprecated code.") - -set_kokkos_default_default(EXPLICIT_INSTANTIATION OFF) -set(KOKKOS_ENABLE_EXPLICIT_INSTANTIATION ${KOKKOS_INTERNAL_ENABLE_EXPLICIT_INSTANTIATION_DEFAULT} CACHE BOOL "Enable explicit template instantiation.") - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_USE_TPLS ------------------------------- -#------------------------------------------------------------------------------- -# Enable hwloc library. -# Figure out default: -IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HWLOC) - set_kokkos_default_default(HWLOC ON) -ELSE() - set_kokkos_default_default(HWLOC OFF) -ENDIF() -set(KOKKOS_ENABLE_HWLOC ${KOKKOS_INTERNAL_ENABLE_HWLOC_DEFAULT} CACHE BOOL "Enable hwloc for better process placement.") -set(KOKKOS_HWLOC_DIR "" CACHE PATH "Location of hwloc library. (kokkos tpl)") - -# Enable memkind library. -set_kokkos_default_default(MEMKIND OFF) -set(KOKKOS_ENABLE_MEMKIND ${KOKKOS_INTERNAL_ENABLE_MEMKIND_DEFAULT} CACHE BOOL "Enable memkind. (kokkos tpl)") -set(KOKKOS_MEMKIND_DIR "" CACHE PATH "Location of memkind library. (kokkos tpl)") - -# Enable rt library. -IF(Trilinos_ENABLE_Kokkos) - IF(DEFINED TPL_ENABLE_LIBRT) - set_kokkos_default_default(LIBRT ${TPL_ENABLE_LIBRT}) - ELSE() - set_kokkos_default_default(LIBRT OFF) - ENDIF() -ELSE() - set_kokkos_default_default(LIBRT ON) -ENDIF() -set(KOKKOS_ENABLE_LIBRT ${KOKKOS_INTERNAL_ENABLE_LIBRT_DEFAULT} CACHE BOOL "Enable librt for more precise timer. (kokkos tpl)") - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- -#------------------------------------------------------------------------------- - -# CUDA options. -# Set Defaults -set_kokkos_default_default(CUDA_LDG_INTRINSIC_DEFAULT OFF) -set_kokkos_default_default(CUDA_UVM_DEFAULT OFF) -set_kokkos_default_default(CUDA_RELOCATABLE_DEVICE_CODE OFF) -IF(Trilinos_ENABLE_Kokkos) - IF(KOKKOS_ENABLE_CUDA) - find_package(CUDA) - ENDIF() - IF (DEFINED CUDA_VERSION) - IF (CUDA_VERSION VERSION_GREATER "7.0") - set_kokkos_default_default(CUDA_LAMBDA ON) - ELSE() - set_kokkos_default_default(CUDA_LAMBDA OFF) - ENDIF() - ENDIF() -ELSE() - set_kokkos_default_default(CUDA_LAMBDA OFF) -ENDIF() - -# Set actual options -set(KOKKOS_CUDA_DIR "" CACHE PATH "Location of CUDA library. Defaults to where nvcc installed.") -set(KOKKOS_ENABLE_CUDA_LDG_INTRINSIC ${KOKKOS_INTERNAL_ENABLE_CUDA_LDG_INTRINSIC_DEFAULT} CACHE BOOL "Enable CUDA LDG. (cuda option)") -set(KOKKOS_ENABLE_CUDA_UVM ${KOKKOS_INTERNAL_ENABLE_CUDA_UVM_DEFAULT} CACHE BOOL "Enable CUDA unified virtual memory.") -set(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ${KOKKOS_INTERNAL_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE_DEFAULT} CACHE BOOL "Enable relocatable device code for CUDA. (cuda option)") -set(KOKKOS_ENABLE_CUDA_LAMBDA ${KOKKOS_INTERNAL_ENABLE_CUDA_LAMBDA_DEFAULT} CACHE BOOL "Enable lambdas for CUDA. (cuda option)") - - -#------------------------------------------------------------------------------- -#------------------------------- KOKKOS_HPX_OPTIONS ---------------------------- -#------------------------------------------------------------------------------- - -# HPX options. -# Set Defaults -set_kokkos_default_default(HPX_ASYNC_DISPATCH OFF) - -# Set actual options -set(KOKKOS_ENABLE_HPX_ASYNC_DISPATCH ${KOKKOS_INTERNAL_ENABLE_HPX_ASYNC_DISPATCH_DEFAULT} CACHE BOOL "Enable HPX async dispatch.") - - -#------------------------------------------------------------------------------- -#----------------------- HOST ARCH AND LEGACY TRIBITS -------------------------- -#------------------------------------------------------------------------------- - -# This defines the previous legacy TriBITS builds. -set(KOKKOS_LEGACY_TRIBITS False) -IF ("${KOKKOS_ARCH}" STREQUAL "NOT_SET") - set(KOKKOS_ARCH "None") - IF(KOKKOS_HAS_TRILINOS) - set(KOKKOS_LEGACY_TRIBITS True) - ENDIF() -ENDIF() -IF (KOKKOS_HAS_TRILINOS) - IF (KOKKOS_LEGACY_TRIBITS) - message(STATUS "Using the legacy tribits build because KOKKOS_ARCH not set") - ELSE() - message(STATUS "NOT using the legacy tribits build because KOKKOS_ARCH *is* set") - ENDIF() -ENDIF() - -#------------------------------------------------------------------------------- -#----------------------- Set CamelCase Options if they are not yet set --------- -#------------------------------------------------------------------------------- - -foreach(opt ${KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST}) - string(TOUPPER ${opt} OPT ) - UNSET(KOKKOS_ENABLE_${OPT}_INTERNAL CACHE) - SET(KOKKOS_ENABLE_${OPT}_INTERNAL ${KOKKOS_ENABLE_${OPT}} CACHE BOOL INTERNAL) - IF(DEFINED KOKKOS_ENABLE_${OPT}) - UNSET(Kokkos_ENABLE_${opt} CACHE) - SET(Kokkos_ENABLE_${opt} ${KOKKOS_ENABLE_${OPT}} CACHE BOOL "CamelCase Compatibility setting for KOKKOS_ENABLE_${OPT}") - ENDIF() -endforeach() diff --git a/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake new file mode 100644 index 0000000000..cf14948f43 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake @@ -0,0 +1,46 @@ +# From CMake 3.10 documentation + +#This can run at any time +KOKKOS_OPTION(CXX_STANDARD "" STRING "The C++ standard for Kokkos to use: 11, 14, 17, or 20. If empty, this will default to CMAKE_CXX_STANDARD. If both CMAKE_CXX_STANDARD and Kokkos_CXX_STANDARD are empty, this will default to 11") + +# Set CXX standard flags +SET(KOKKOS_ENABLE_CXX11 OFF) +SET(KOKKOS_ENABLE_CXX14 OFF) +SET(KOKKOS_ENABLE_CXX17 OFF) +SET(KOKKOS_ENABLE_CXX20 OFF) +IF (KOKKOS_CXX_STANDARD) + IF (${KOKKOS_CXX_STANDARD} STREQUAL "c++98") + MESSAGE(FATAL_ERROR "Kokkos no longer supports C++98 - minimum C++11") + ELSEIF (${KOKKOS_CXX_STANDARD} STREQUAL "c++11") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++11'. Use '11' instead.") + SET(KOKKOS_CXX_STANDARD "11") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++14") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++14'. Use '14' instead.") + SET(KOKKOS_CXX_STANDARD "14") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++17") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++17'. Use '17' instead.") + SET(KOKKOS_CXX_STANDARD "17") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++1y") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++1y'. Use '1Y' instead.") + SET(KOKKOS_CXX_STANDARD "1Y") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++1z") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++1z'. Use '1Z' instead.") + SET(KOKKOS_CXX_STANDARD "1Z") + ELSEIF(${KOKKOS_CXX_STANDARD} STREQUAL "c++2a") + MESSAGE(WARNING "Deprecated Kokkos C++ standard set as 'c++2a'. Use '2A' instead.") + SET(KOKKOS_CXX_STANDARD "2A") + ENDIF() +ENDIF() + +IF (NOT KOKKOS_CXX_STANDARD AND NOT CMAKE_CXX_STANDARD) + MESSAGE(STATUS "Setting default Kokkos CXX standard to 11") + SET(KOKKOS_CXX_STANDARD "11") +ELSEIF(NOT KOKKOS_CXX_STANDARD) + MESSAGE(STATUS "Setting default Kokkos CXX standard to ${CMAKE_CXX_STANDARD}") + SET(KOKKOS_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +ENDIF() + + + + + diff --git a/lib/kokkos/cmake/kokkos_settings.cmake b/lib/kokkos/cmake/kokkos_settings.cmake deleted file mode 100644 index 2c622d0de9..0000000000 --- a/lib/kokkos/cmake/kokkos_settings.cmake +++ /dev/null @@ -1,259 +0,0 @@ -########################## NOTES ############################################### -# This files goal is to take CMake options found in kokkos_options.cmake but -# possibly set from elsewhere -# (see: trilinos/cmake/ProjectCOmpilerPostConfig.cmake) -# using CMake idioms and map them onto the KOKKOS_SETTINGS variables that gets -# passed to the kokkos makefile configuration: -# make -f ${CMAKE_SOURCE_DIR}/core/src/Makefile ${KOKKOS_SETTINGS} build-makefile-cmake-kokkos -# that generates KokkosCore_config.h and kokkos_generated_settings.cmake -# To understand how to form KOKKOS_SETTINGS, see -# /Makefile.kokkos - -#------------------------------------------------------------------------------- -#------------------------------- GENERAL OPTIONS ------------------------------- -#------------------------------------------------------------------------------- - -# Ensure that KOKKOS_ARCH is in the ARCH_LIST -if (KOKKOS_ARCH MATCHES ",") - message("-- Detected a comma in: KOKKOS_ARCH=`${KOKKOS_ARCH}`") - message("-- Although we prefer KOKKOS_ARCH to be semicolon-delimited, we do allow") - message("-- comma-delimited values for compatibility with scripts (see github.com/trilinos/Trilinos/issues/2330)") - string(REPLACE "," ";" KOKKOS_ARCH "${KOKKOS_ARCH}") - message("-- Commas were changed to semicolons, now KOKKOS_ARCH=`${KOKKOS_ARCH}`") -endif() -foreach(arch ${KOKKOS_ARCH}) - list(FIND KOKKOS_ARCH_LIST ${arch} indx) - if (indx EQUAL -1) - message(FATAL_ERROR "`${arch}` is not an accepted value in KOKKOS_ARCH=`${KOKKOS_ARCH}`." - " Please pick from these choices: ${KOKKOS_INTERNAL_ARCH_DOCSTR}") - endif () -endforeach() - -# KOKKOS_SETTINGS uses KOKKOS_ARCH -string(REPLACE ";" "," KOKKOS_GMAKE_ARCH "${KOKKOS_ARCH}") - -# From Makefile.kokkos: Options: yes,no -if(${KOKKOS_ENABLE_DEBUG}) - set(KOKKOS_GMAKE_DEBUG yes) -else() - set(KOKKOS_GMAKE_DEBUG no) -endif() - -#------------------------------- KOKKOS_DEVICES -------------------------------- -# Can have multiple devices -set(KOKKOS_DEVICESl) -foreach(devopt ${KOKKOS_DEVICES_LIST}) - string(TOUPPER ${devopt} devoptuc) - if (${KOKKOS_ENABLE_${devoptuc}}) - list(APPEND KOKKOS_DEVICESl ${devopt}) - endif () -endforeach() -# List needs to be comma-delmitted -string(REPLACE ";" "," KOKKOS_GMAKE_DEVICES "${KOKKOS_DEVICESl}") - -#------------------------------- KOKKOS_OPTIONS -------------------------------- -# From Makefile.kokkos: Options: aggressive_vectorization,disable_profiling,disable_deprecated_code -#compiler_warnings, aggressive_vectorization, disable_profiling, disable_dualview_modify_check, enable_profile_load_print - -set(KOKKOS_OPTIONSl) -if(${KOKKOS_ENABLE_COMPILER_WARNINGS}) - list(APPEND KOKKOS_OPTIONSl compiler_warnings) -endif() -if(${KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION}) - list(APPEND KOKKOS_OPTIONSl aggressive_vectorization) -endif() -if(NOT ${KOKKOS_ENABLE_PROFILING}) - list(APPEND KOKKOS_OPTIONSl disable_profiling) -endif() -if(NOT ${KOKKOS_ENABLE_DEPRECATED_CODE}) - list(APPEND KOKKOS_OPTIONSl disable_deprecated_code) -endif() -if(NOT ${KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK}) - list(APPEND KOKKOS_OPTIONSl disable_dualview_modify_check) -endif() -if(${KOKKOS_ENABLE_PROFILING_LOAD_PRINT}) - list(APPEND KOKKOS_OPTIONSl enable_profile_load_print) -endif() -if(${KOKKOS_ENABLE_EXPLICIT_INSTANTIATION}) - list(APPEND KOKKOS_OPTIONSl enable_eti) -endif() -# List needs to be comma-delimitted -string(REPLACE ";" "," KOKKOS_GMAKE_OPTIONS "${KOKKOS_OPTIONSl}") - - -#------------------------------- KOKKOS_USE_TPLS ------------------------------- -# Construct the Makefile options -set(KOKKOS_USE_TPLSl) -foreach(tplopt ${KOKKOS_USE_TPLS_LIST}) - if (${KOKKOS_ENABLE_${tplopt}}) - list(APPEND KOKKOS_USE_TPLSl ${KOKKOS_INTERNAL_${tplopt}}) - endif () -endforeach() -# List needs to be comma-delimitted -string(REPLACE ";" "," KOKKOS_GMAKE_USE_TPLS "${KOKKOS_USE_TPLSl}") - - -#------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- -# Construct the Makefile options -set(KOKKOS_CUDA_OPTIONSl) -foreach(cudaopt ${KOKKOS_CUDA_OPTIONS_LIST}) - if (${KOKKOS_ENABLE_CUDA_${cudaopt}}) - list(APPEND KOKKOS_CUDA_OPTIONSl ${KOKKOS_INTERNAL_${cudaopt}}) - endif () -endforeach() -# List needs to be comma-delmitted -string(REPLACE ";" "," KOKKOS_GMAKE_CUDA_OPTIONS "${KOKKOS_CUDA_OPTIONSl}") - -#------------------------------- PATH VARIABLES -------------------------------- -# Want makefile to use same executables specified which means modifying -# the path so the $(shell ...) commands in the makefile see the right exec -# Also, the Makefile's use FOO_PATH naming scheme for -I/-L construction -#TODO: Makefile.kokkos allows this to be overwritten? ROCM_HCC_PATH - -set(KOKKOS_INTERNAL_PATHS) -set(addpathl) -foreach(kvar IN LISTS KOKKOS_USE_TPLS_LIST ITEMS CUDA QTHREADS) - if(${KOKKOS_ENABLE_${kvar}}) - if(DEFINED KOKKOS_${kvar}_DIR) - set(KOKKOS_INTERNAL_PATHS ${KOKKOS_INTERNAL_PATHS} "${kvar}_PATH=${KOKKOS_${kvar}_DIR}") - if(IS_DIRECTORY ${KOKKOS_${kvar}_DIR}/bin) - list(APPEND addpathl ${KOKKOS_${kvar}_DIR}/bin) - endif() - endif() - endif() -endforeach() -# Path env is : delimitted -string(REPLACE ";" ":" KOKKOS_INTERNAL_ADDTOPATH "${addpathl}") - - -######################### SET KOKKOS_SETTINGS ################################## -# Set the KOKKOS_SETTINGS String -- this is the primary communication with the -# makefile configuration. See Makefile.kokkos - -set(KOKKOS_SETTINGS KOKKOS_CMAKE=yes) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_SRC_PATH=${KOKKOS_SRC_PATH}) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_PATH=${KOKKOS_PATH}) -set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_INSTALL_PATH=${CMAKE_INSTALL_PREFIX}) - -# Form of KOKKOS_foo=$KOKKOS_foo -foreach(kvar ARCH;DEVICES;DEBUG;OPTIONS;CUDA_OPTIONS;USE_TPLS) - if(DEFINED KOKKOS_GMAKE_${kvar}) - if (NOT "${KOKKOS_GMAKE_${kvar}}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} KOKKOS_${kvar}=${KOKKOS_GMAKE_${kvar}}) - endif() - endif() -endforeach() - -# Form of VAR=VAL -#TODO: Makefile supports MPICH_CXX, OMPI_CXX as well -foreach(ovar CXX;CXXFLAGS;LDFLAGS) - if(DEFINED ${ovar}) - if (NOT "${${ovar}}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} ${ovar}=${${ovar}}) - endif() - endif() -endforeach() - -# Finally, do the paths -if (NOT "${KOKKOS_INTERNAL_PATHS}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} ${KOKKOS_INTERNAL_PATHS}) -endif() -if (NOT "${KOKKOS_INTERNAL_ADDTOPATH}" STREQUAL "") - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} "PATH=${KOKKOS_INTERNAL_ADDTOPATH}:$ENV{PATH}") -endif() - -if (CMAKE_CXX_STANDARD) - if (CMAKE_CXX_STANDARD STREQUAL "98") - message(FATAL_ERROR "Kokkos requires C++11 or newer!") - endif() - set(KOKKOS_CXX_STANDARD "c++${CMAKE_CXX_STANDARD}") - if (CMAKE_CXX_EXTENSIONS) - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(KOKKOS_CXX_STANDARD "gnu++${CMAKE_CXX_STANDARD}") - endif() - endif() - set(KOKKOS_SETTINGS ${KOKKOS_SETTINGS} "KOKKOS_CXX_STANDARD=\"${KOKKOS_CXX_STANDARD}\"") -endif() - -# Final form that gets passed to make -set(KOKKOS_SETTINGS env ${KOKKOS_SETTINGS}) - - -############################ PRINT CONFIGURE STATUS ############################ - -if(KOKKOS_CMAKE_VERBOSE) - message(STATUS "") - message(STATUS "****************** Kokkos Settings ******************") - message(STATUS "Execution Spaces") - - if(KOKKOS_ENABLE_CUDA) - message(STATUS " Device Parallel: Cuda") - else() - message(STATUS " Device Parallel: None") - endif() - - if(KOKKOS_ENABLE_OPENMP) - message(STATUS " Host Parallel: OpenMP") - elseif(KOKKOS_ENABLE_PTHREAD) - message(STATUS " Host Parallel: Pthread") - elseif(KOKKOS_ENABLE_QTHREADS) - message(STATUS " Host Parallel: Qthreads") - elseif(KOKKOS_ENABLE_HPX) - message(STATUS " Host Parallel: HPX") - else() - message(STATUS " Host Parallel: None") - endif() - - if(KOKKOS_ENABLE_SERIAL) - message(STATUS " Host Serial: Serial") - else() - message(STATUS " Host Serial: None") - endif() - - message(STATUS "") - message(STATUS "Architectures:") - message(STATUS " ${KOKKOS_GMAKE_ARCH}") - - message(STATUS "") - message(STATUS "Enabled options") - - if(KOKKOS_SEPARATE_LIBS) - message(STATUS " KOKKOS_SEPARATE_LIBS") - endif() - - foreach(opt IN LISTS KOKKOS_INTERNAL_ENABLE_OPTIONS_LIST) - string(TOUPPER ${opt} OPT) - if (KOKKOS_ENABLE_${OPT}) - message(STATUS " KOKKOS_ENABLE_${OPT}") - endif() - endforeach() - - if(KOKKOS_ENABLE_CUDA) - if(KOKKOS_CUDA_DIR) - message(STATUS " KOKKOS_CUDA_DIR: ${KOKKOS_CUDA_DIR}") - endif() - endif() - - if(KOKKOS_QTHREADS_DIR) - message(STATUS " KOKKOS_QTHREADS_DIR: ${KOKKOS_QTHREADS_DIR}") - endif() - - if(KOKKOS_HWLOC_DIR) - message(STATUS " KOKKOS_HWLOC_DIR: ${KOKKOS_HWLOC_DIR}") - endif() - - if(KOKKOS_MEMKIND_DIR) - message(STATUS " KOKKOS_MEMKIND_DIR: ${KOKKOS_MEMKIND_DIR}") - endif() - - if(KOKKOS_HPX_DIR) - message(STATUS " KOKKOS_HPX_DIR: ${KOKKOS_HPX_DIR}") - endif() - - message(STATUS "") - message(STATUS "Final kokkos settings variable:") - message(STATUS " ${KOKKOS_SETTINGS}") - - message(STATUS "*****************************************************") - message(STATUS "") -endif() diff --git a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake new file mode 100644 index 0000000000..603b4b3d7a --- /dev/null +++ b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake @@ -0,0 +1,151 @@ +KOKKOS_CFG_DEPENDS(CXX_STD COMPILER_ID) + +FUNCTION(kokkos_set_cxx_standard_feature standard) + SET(EXTENSION_NAME CMAKE_CXX${standard}_EXTENSION_COMPILE_OPTION) + SET(STANDARD_NAME CMAKE_CXX${standard}_STANDARD_COMPILE_OPTION) + SET(FEATURE_NAME cxx_std_${standard}) + #CMake's way of telling us that the standard (or extension) + #flags are supported is the extension/standard variables + IF (NOT DEFINED CMAKE_CXX_EXTENSIONS) + IF(KOKKOS_DONT_ALLOW_EXTENSIONS) + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) + ELSE() + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) + ENDIF() + ELSEIF(CMAKE_CXX_EXTENSIONS) + IF(KOKKOS_DONT_ALLOW_EXTENSIONS) + MESSAGE(FATAL_ERROR "The chosen configuration does not support CXX extensions flags: ${KOKKOS_DONT_ALLOW_EXTENSIONS}. Must set CMAKE_CXX_EXTENSIONS=OFF to continue") + ELSE() + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) + ENDIF() + ELSE() + #For trilinos, we need to make sure downstream projects + GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) + ENDIF() + + IF (KOKKOS_USE_CXX_EXTENSIONS AND ${EXTENSION_NAME}) + MESSAGE(STATUS "Using ${${EXTENSION_NAME}} for C++${standard} extensions as feature") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + ELSEIF(NOT KOKKOS_USE_CXX_EXTENSIONS AND ${STANDARD_NAME}) + MESSAGE(STATUS "Using ${${STANDARD_NAME}} for C++${standard} standard as feature") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") + #MSVC doesn't need a command line flag, that doesn't mean it has no support + MESSAGE(STATUS "Using no flag for C++${standard} standard as feature") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + ELSE() + #nope, we can't do anything here + MESSAGE(WARNING "C++${standard} is not supported as a compiler feature. We will choose custom flags for now, but this behavior has been deprecated. Please open an issue at https://github.com/kokkos/kokkos/issues reporting that ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION} failed for ${KOKKOS_CXX_STANDARD}, preferrably including your CMake command.") + GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE "") + ENDIF() + + IF(NOT ${FEATURE_NAME} IN_LIST CMAKE_CXX_COMPILE_FEATURES) + MESSAGE(FATAL_ERROR "Compiler ${KOKKOS_CXX_COMPILER_ID} should support ${FEATURE_NAME}, but CMake reports feature not supported") + ENDIF() +ENDFUNCTION() + + +IF (KOKKOS_CXX_STANDARD AND CMAKE_CXX_STANDARD) + #make sure these are consistent + IF (NOT KOKKOS_CXX_STANDARD STREQUAL CMAKE_CXX_STANDARD) + MESSAGE(WARNING "Specified both CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} and KOKKOS_CXX_STANDARD=${KOKKOS_CXX_STANDARD}, but they don't match") + SET(CMAKE_CXX_STANDARD ${KOKKOS_CXX_STANDARD} CACHE STRING "C++ standard" FORCE) + ENDIF() +ENDIF() + + +IF (KOKKOS_CXX_STANDARD STREQUAL "11" ) + kokkos_set_cxx_standard_feature(11) + SET(KOKKOS_ENABLE_CXX11 ON) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "11") +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "14") + kokkos_set_cxx_standard_feature(14) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Y") + SET(KOKKOS_ENABLE_CXX14 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "17") + kokkos_set_cxx_standard_feature(17) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Z") + SET(KOKKOS_ENABLE_CXX17 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "20") + kokkos_set_cxx_standard_feature(20) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "2A") + SET(KOKKOS_ENABLE_CXX20 ON) +ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "98") + MESSAGE(FATAL_ERROR "Kokkos requires C++11 or newer!") +ELSE() + MESSAGE(FATAL_ERROR "Unknown C++ standard ${KOKKOS_CXX_STANDARD} - must be 11, 14, 17, or 20") +ENDIF() + + + +# Enforce that extensions are turned off for nvcc_wrapper. +# For compiling CUDA code using nvcc_wrapper, we will use the host compiler's +# flags for turning on C++11. Since for compiler ID and versioning purposes +# CMake recognizes the host compiler when calling nvcc_wrapper, this just +# works. Both NVCC and nvcc_wrapper only recognize '-std=c++11' which means +# that we can only use host compilers for CUDA builds that use those flags. +# It also means that extensions (gnu++11) can't be turned on for CUDA builds. + +IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) + SET(CMAKE_CXX_EXTENSIONS OFF) + ELSEIF(CMAKE_CXX_EXTENSIONS) + MESSAGE(FATAL_ERROR "NVCC doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") + ENDIF() +ENDIF() + +IF(KOKKOS_ENABLE_CUDA) + # ENFORCE that the compiler can compile CUDA code. + IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) + MESSAGE(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") + ENDIF() + IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) + SET(CMAKE_CXX_EXTENSIONS OFF) + ELSEIF(CMAKE_CXX_EXTENSIONS) + MESSAGE(FATAL_ERROR "Compiling CUDA code with clang doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") + ENDIF() + ELSEIF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + MESSAGE(FATAL_ERROR "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang, but compiler ID was ${KOKKOS_CXX_COMPILER_ID}") + ENDIF() +ENDIF() + +IF (NOT KOKKOS_CXX_STANDARD_FEATURE) + #we need to pick the C++ flags ourselves + UNSET(CMAKE_CXX_STANDARD) + UNSET(CMAKE_CXX_STANDARD CACHE) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/cray.cmake) + kokkos_set_cray_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL PGI) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/pgi.cmake) + kokkos_set_pgi_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/intel.cmake) + kokkos_set_intel_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") + INCLUDE(${KOKKOS_SRC_PATH}/cmake/msvc.cmake) + kokkos_set_msvc_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ELSE() + INCLUDE(${KOKKOS_SRC_PATH}/cmake/gnu.cmake) + kokkos_set_gnu_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) + ENDIF() + #check that the compiler accepts the C++ standard flag + INCLUDE(CheckCXXCompilerFlag) + IF (DEFINED CXX_STD_FLAGS_ACCEPTED) + UNSET(CXX_STD_FLAGS_ACCEPTED CACHE) + ENDIF() + CHECK_CXX_COMPILER_FLAG("${KOKKOS_CXX_STANDARD_FLAG}" CXX_STD_FLAGS_ACCEPTED) + IF (NOT CXX_STD_FLAGS_ACCEPTED) + CHECK_CXX_COMPILER_FLAG("${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}" CXX_INT_STD_FLAGS_ACCEPTED) + IF (NOT CXX_INT_STD_FLAGS_ACCEPTED) + MESSAGE(FATAL_ERROR "${KOKKOS_CXX_COMPILER_ID} did not accept ${KOKKOS_CXX_STANDARD_FLAG} or ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}. You likely need to reduce the level of the C++ standard from ${KOKKOS_CXX_STANDARD}") + ENDIF() + SET(KOKKOS_CXX_STANDARD_FLAG ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}) + ENDIF() + MESSAGE(STATUS "Compiler features not supported, but ${KOKKOS_CXX_COMPILER_ID} accepts ${KOKKOS_CXX_STANDARD_FLAG}") +ENDIF() + + + + diff --git a/lib/kokkos/cmake/kokkos_tpls.cmake b/lib/kokkos/cmake/kokkos_tpls.cmake new file mode 100644 index 0000000000..76efd42847 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_tpls.cmake @@ -0,0 +1,51 @@ +KOKKOS_CFG_DEPENDS(TPLS OPTIONS) +KOKKOS_CFG_DEPENDS(TPLS DEVICES) + +FUNCTION(KOKKOS_TPL_OPTION PKG DEFAULT) + KOKKOS_ENABLE_OPTION(${PKG} ${DEFAULT} "Whether to enable the ${PKG} library") + KOKKOS_OPTION(${PKG}_DIR "" PATH "Location of ${PKG} library") + SET(KOKKOS_ENABLE_${PKG} ${KOKKOS_ENABLE_${PKG}} PARENT_SCOPE) + SET(KOKKOS_${PKG}_DIR ${KOKKOS_${PKG}_DIR} PARENT_SCOPE) +ENDFUNCTION() + +KOKKOS_TPL_OPTION(HWLOC Off) +KOKKOS_TPL_OPTION(LIBNUMA Off) +KOKKOS_TPL_OPTION(MEMKIND Off) +KOKKOS_TPL_OPTION(CUDA Off) +KOKKOS_TPL_OPTION(LIBRT Off) +KOKKOS_TPL_OPTION(LIBDL On) + +IF(KOKKOS_ENABLE_PROFILING AND NOT KOKKOS_ENABLE_LIBDL) + MESSAGE(SEND_ERROR "Kokkos_ENABLE_PROFILING requires Kokkos_ENABLE_LIBDL=ON") +ENDIF() + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HPX) +SET(HPX_DEFAULT ON) +ELSE() +SET(HPX_DEFAULT OFF) +ENDIF() +KOKKOS_TPL_OPTION(HPX ${HPX_DEFAULT}) + +IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_PTHREAD) +SET(PTHREAD_DEFAULT ON) +ELSE() +SET(PTHREAD_DEFAULT OFF) +ENDIF() +KOKKOS_TPL_OPTION(PTHREAD ${PTHREAD_DEFAULT}) + + +#Make sure we use our local FindKokkosCuda.cmake +KOKKOS_IMPORT_TPL(HPX INTERFACE) +KOKKOS_IMPORT_TPL(CUDA INTERFACE) +KOKKOS_IMPORT_TPL(HWLOC) +KOKKOS_IMPORT_TPL(LIBNUMA) +KOKKOS_IMPORT_TPL(LIBRT) +KOKKOS_IMPORT_TPL(LIBDL) +KOKKOS_IMPORT_TPL(MEMKIND) +KOKKOS_IMPORT_TPL(PTHREAD INTERFACE) + +#Convert list to newlines (which CMake doesn't always like in cache variables) +STRING(REPLACE ";" "\n" KOKKOS_TPL_EXPORT_TEMP "${KOKKOS_TPL_EXPORTS}") +#Convert to a regular variable +UNSET(KOKKOS_TPL_EXPORTS CACHE) +SET(KOKKOS_TPL_EXPORTS ${KOKKOS_TPL_EXPORT_TEMP}) diff --git a/lib/kokkos/cmake/kokkos_tribits.cmake b/lib/kokkos/cmake/kokkos_tribits.cmake new file mode 100644 index 0000000000..6ee1409aa7 --- /dev/null +++ b/lib/kokkos/cmake/kokkos_tribits.cmake @@ -0,0 +1,425 @@ +#These are tribits wrappers only ever called by Kokkos itself + +INCLUDE(CMakeParseArguments) +INCLUDE(CTest) +INCLUDE(GNUInstallDirs) + +MESSAGE(STATUS "The project name is: ${PROJECT_NAME}") + +#Leave this here for now - but only do for tribits +#This breaks the standalone CMake +IF (KOKKOS_HAS_TRILINOS) + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_OpenMP) + SET(${PROJECT_NAME}_ENABLE_OpenMP OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_HPX) + SET(${PROJECT_NAME}_ENABLE_HPX OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_DEBUG) + SET(${PROJECT_NAME}_ENABLE_DEBUG OFF) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_CXX11) + SET(${PROJECT_NAME}_ENABLE_CXX11 ON) + ENDIF() + + IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_TESTS) + SET(${PROJECT_NAME}_ENABLE_TESTS OFF) + ENDIF() + + IF(NOT DEFINED TPL_ENABLE_Pthread) + SET(TPL_ENABLE_Pthread OFF) + ENDIF() +ENDIF() + +MACRO(KOKKOS_SUBPACKAGE NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_SUBPACKAGE(${NAME}) + else() + SET(PACKAGE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + SET(PARENT_PACKAGE_NAME ${PACKAGE_NAME}) + SET(PACKAGE_NAME ${PACKAGE_NAME}${NAME}) + STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) + SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + #ADD_INTERFACE_LIBRARY(PACKAGE_${PACKAGE_NAME}) + #GLOBAL_SET(${PACKAGE_NAME}_LIBS "") + endif() +ENDMACRO() + +MACRO(KOKKOS_SUBPACKAGE_POSTPROCESS) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_SUBPACKAGE_POSTPROCESS() + endif() +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_DECL) + + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_DECL(Kokkos) + else() + SET(PACKAGE_NAME Kokkos) + SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) + endif() + + #SET(TRIBITS_DEPS_DIR "${CMAKE_SOURCE_DIR}/cmake/deps") + #FILE(GLOB TPLS_FILES "${TRIBITS_DEPS_DIR}/*.cmake") + #FOREACH(TPL_FILE ${TPLS_FILES}) + # TRIBITS_PROCESS_TPL_DEP_FILE(${TPL_FILE}) + #ENDFOREACH() + +ENDMACRO() + + +MACRO(KOKKOS_PROCESS_SUBPACKAGES) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PROCESS_SUBPACKAGES() + else() + ADD_SUBDIRECTORY(core) + ADD_SUBDIRECTORY(containers) + ADD_SUBDIRECTORY(algorithms) + ADD_SUBDIRECTORY(example) + endif() +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_DEF) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_DEF() + else() + #do nothing + endif() +ENDMACRO() + +MACRO(KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL LIBRARY_NAME) + KOKKOS_LIB_TYPE(${LIBRARY_NAME} INCTYPE) + TARGET_INCLUDE_DIRECTORIES(${LIBRARY_NAME} ${INCTYPE} $) + + INSTALL( + TARGETS ${LIBRARY_NAME} + EXPORT ${PROJECT_NAME} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT ${PACKAGE_NAME} + ) + + INSTALL( + TARGETS ${LIBRARY_NAME} + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + + VERIFY_EMPTY(KOKKOS_ADD_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +ENDMACRO() + +FUNCTION(KOKKOS_ADD_EXECUTABLE ROOT_NAME) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_EXECUTABLE(${ROOT_NAME} ${ARGN}) + else() + CMAKE_PARSE_ARGUMENTS(PARSE + "TESTONLY" + "" + "SOURCES;TESTONLYLIBS" + ${ARGN}) + + SET(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + ADD_EXECUTABLE(${EXE_NAME} ${PARSE_SOURCES}) + IF (PARSE_TESTONLYLIBS) + TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE ${PARSE_TESTONLYLIBS}) + ENDIF() + VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE ${PARSE_UNPARSED_ARGUMENTS}) + #All executables must link to all the kokkos targets + #This is just private linkage because exe is final + TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE kokkos) + endif() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_EXECUTABLE_AND_TEST ROOT_NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_EXECUTABLE_AND_TEST( + ${ROOT_NAME} + TESTONLYLIBS kokkos_gtest + ${ARGN} + NUM_MPI_PROCS 1 + COMM serial mpi + FAIL_REGULAR_EXPRESSION " FAILED " + ) +ELSE() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "SOURCES;CATEGORIES" + ${ARGN}) + VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) + KOKKOS_ADD_TEST_EXECUTABLE(${ROOT_NAME} + SOURCES ${PARSE_SOURCES} + ) + KOKKOS_ADD_TEST(NAME ${ROOT_NAME} + EXE ${ROOT_NAME} + FAIL_REGULAR_EXPRESSION " FAILED " + ) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_SET_EXE_PROPERTY ROOT_NAME) + SET(TARGET_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + IF (NOT TARGET ${TARGET_NAME}) + MESSAGE(SEND_ERROR "No target ${TARGET_NAME} exists - cannot set target properties") + ENDIF() + SET_PROPERTY(TARGET ${TARGET_NAME} PROPERTY ${ARGN}) +ENDFUNCTION() + +MACRO(KOKKOS_SETUP_BUILD_ENVIRONMENT) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_compiler_id.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_devices.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_options.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_test_cxx_std.cmake) + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_arch.cmake) + IF (NOT KOKKOS_HAS_TRILINOS) + SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tpls.cmake) + ENDIF() + INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_corner_cases.cmake) +ENDMACRO() + +MACRO(KOKKOS_ADD_TEST_EXECUTABLE ROOT_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "SOURCES" + ${ARGN}) + KOKKOS_ADD_EXECUTABLE(${ROOT_NAME} + SOURCES ${PARSE_SOURCES} + ${PARSE_UNPARSED_ARGUMENTS} + TESTONLYLIBS kokkos_gtest + ) +ENDMACRO() + +MACRO(KOKKOS_PACKAGE_POSTPROCESS) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_PACKAGE_POSTPROCESS() + endif() +ENDMACRO() + +FUNCTION(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "PLAIN_STYLE" + "" + "" + ${ARGN}) + + IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13") + #great, this works the "right" way + TARGET_LINK_OPTIONS( + ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} + ) + ELSE() + IF (PARSE_PLAIN_STYLE) + TARGET_LINK_LIBRARIES( + ${LIBRARY_NAME} ${KOKKOS_LINK_OPTIONS} + ) + ELSE() + #well, have to do it the wrong way for now + TARGET_LINK_LIBRARIES( + ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} + ) + ENDIF() + ENDIF() + + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} PUBLIC + $<$:${KOKKOS_COMPILE_OPTIONS}> + ) + + TARGET_COMPILE_DEFINITIONS( + ${LIBRARY_NAME} PUBLIC + $<$:${KOKKOS_COMPILE_DEFINITIONS}> + ) + + TARGET_LINK_LIBRARIES( + ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_LIBRARIES} + ) + + IF (KOKKOS_ENABLE_CUDA) + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${KOKKOS_CUDA_OPTIONS}> + ) + SET(NODEDUP_CUDAFE_OPTIONS) + FOREACH(OPT ${KOKKOS_CUDAFE_OPTIONS}) + LIST(APPEND NODEDUP_CUDAFE_OPTIONS -Xcudafe ${OPT}) + ENDFOREACH() + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${NODEDUP_CUDAFE_OPTIONS}> + ) + ENDIF() + + IF (KOKKOS_ENABLE_HIP) + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${KOKKOS_AMDGPU_OPTIONS}> + ) + ENDIF() + + LIST(LENGTH KOKKOS_XCOMPILER_OPTIONS XOPT_LENGTH) + IF (XOPT_LENGTH GREATER 1) + MESSAGE(FATAL_ERROR "CMake deduplication does not allow multiple -Xcompiler flags (${KOKKOS_XCOMPILER_OPTIONS}): will require Kokkos to upgrade to minimum 3.12") + ENDIF() + IF(KOKKOS_XCOMPILER_OPTIONS) + SET(NODEDUP_XCOMPILER_OPTIONS) + FOREACH(OPT ${KOKKOS_XCOMPILER_OPTIONS}) + #I have to do this for now because we can't guarantee 3.12 support + #I really should do this with the shell option + LIST(APPEND NODEDUP_XCOMPILER_OPTIONS -Xcompiler) + LIST(APPEND NODEDUP_XCOMPILER_OPTIONS ${OPT}) + ENDFOREACH() + TARGET_COMPILE_OPTIONS( + ${LIBRARY_NAME} + PUBLIC $<$:${NODEDUP_XCOMPILER_OPTIONS}> + ) + ENDIF() + + IF (KOKKOS_CXX_STANDARD_FEATURE) + #GREAT! I can do this the right way + TARGET_COMPILE_FEATURES(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FEATURE}) + IF (NOT KOKKOS_USE_CXX_EXTENSIONS) + SET_TARGET_PROPERTIES(${LIBRARY_NAME} PROPERTIES CXX_EXTENSIONS OFF) + ENDIF() + ELSE() + #OH, well, no choice but the wrong way + TARGET_COMPILE_OPTIONS(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FLAG}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_INTERNAL_ADD_LIBRARY LIBRARY_NAME) + CMAKE_PARSE_ARGUMENTS(PARSE + "STATIC;SHARED" + "" + "HEADERS;SOURCES" + ${ARGN}) + + IF(PARSE_HEADERS) + LIST(REMOVE_DUPLICATES PARSE_HEADERS) + ENDIF() + IF(PARSE_SOURCES) + LIST(REMOVE_DUPLICATES PARSE_SOURCES) + ENDIF() + + ADD_LIBRARY( + ${LIBRARY_NAME} + ${PARSE_HEADERS} + ${PARSE_SOURCES} + ) + + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${LIBRARY_NAME}) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT ${PACKAGE_NAME} + ) + + #In case we are building in-tree, add an alias name + #that matches the install Kokkos:: name + ADD_LIBRARY(Kokkos::${LIBRARY_NAME} ALIAS ${LIBRARY_NAME}) +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_LIBRARY LIBRARY_NAME) + IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${LIBRARY_NAME} ${ARGN}) + #Stolen from Tribits - it can add prefixes + SET(TRIBITS_LIBRARY_NAME_PREFIX "${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}") + SET(TRIBITS_LIBRARY_NAME ${TRIBITS_LIBRARY_NAME_PREFIX}${LIBRARY_NAME}) + #Tribits has way too much techinical debt and baggage to even + #allow PUBLIC target_compile_options to be used. It forces C++ flags on projects + #as a giant blob of space-separated strings. We end up with duplicated + #flags between the flags implicitly forced on Kokkos-dependent and those Kokkos + #has in its public INTERFACE_COMPILE_OPTIONS. + #These do NOT get de-deduplicated because Tribits + #creates flags as a giant monolithic space-separated string + #Do not set any transitive properties and keep everything working as before + #KOKKOS_SET_LIBRARY_PROPERTIES(${TRIBITS_LIBRARY_NAME} PLAIN_STYLE) + ELSE() + KOKKOS_INTERNAL_ADD_LIBRARY( + ${LIBRARY_NAME} ${ARGN}) + KOKKOS_SET_LIBRARY_PROPERTIES(${LIBRARY_NAME}) + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_ADD_INTERFACE_LIBRARY NAME) +IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_LIBRARY(${NAME} ${ARGN}) +ELSE() + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "HEADERS;SOURCES" + ${ARGN} + ) + + ADD_LIBRARY(${NAME} INTERFACE) + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${NAME}) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + + INSTALL( + FILES ${PARSE_HEADERS} + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT ${PACKAGE_NAME} + ) +ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LIB_INCLUDE_DIRECTORIES TARGET) + IF(KOKKOS_HAS_TRILINOS) + #ignore the target, tribits doesn't do anything directly with targets + TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) + ELSE() #append to a list for later + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + FOREACH(DIR ${ARGN}) + TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} $) + ENDFOREACH() + ENDIF() +ENDFUNCTION() + +FUNCTION(KOKKOS_LIB_COMPILE_OPTIONS TARGET) + IF(KOKKOS_HAS_TRILINOS) + #don't trust tribits to do this correctly + KOKKOS_TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) + ELSE() + KOKKOS_LIB_TYPE(${TARGET} INCTYPE) + KOKKOS_TARGET_COMPILE_OPTIONS(${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}${TARGET} ${INCTYPE} ${ARGN}) + ENDIF() +ENDFUNCTION() + +MACRO(KOKKOS_ADD_TEST_DIRECTORIES) + IF (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_TEST_DIRECTORIES(${ARGN}) + ELSE() + IF(KOKKOS_ENABLE_TESTS) + FOREACH(TEST_DIR ${ARGN}) + ADD_SUBDIRECTORY(${TEST_DIR}) + ENDFOREACH() + ENDIF() + ENDIF() +ENDMACRO() + +MACRO(KOKKOS_ADD_EXAMPLE_DIRECTORIES) + if (KOKKOS_HAS_TRILINOS) + TRIBITS_ADD_EXAMPLE_DIRECTORIES(${ARGN}) + else() + IF(KOKKOS_ENABLE_EXAMPLES) + FOREACH(EXAMPLE_DIR ${ARGN}) + ADD_SUBDIRECTORY(${EXAMPLE_DIR}) + ENDFOREACH() + ENDIF() + endif() +ENDMACRO() diff --git a/lib/kokkos/cmake/msvc.cmake b/lib/kokkos/cmake/msvc.cmake new file mode 100644 index 0000000000..85421bdbaa --- /dev/null +++ b/lib/kokkos/cmake/msvc.cmake @@ -0,0 +1,11 @@ + +FUNCTION(kokkos_set_msvc_flags full_standard int_standard) + IF (CMAKE_CXX_EXTENSIONS) + SET(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) + ELSE() + SET(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) + ENDIF() +ENDFUNCTION() + diff --git a/lib/kokkos/cmake/pgi.cmake b/lib/kokkos/cmake/pgi.cmake new file mode 100644 index 0000000000..e98e849558 --- /dev/null +++ b/lib/kokkos/cmake/pgi.cmake @@ -0,0 +1,8 @@ + +function(kokkos_set_pgi_flags full_standard int_standard) + STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) + STRING(TOLOWER ${int_standard} INT_LC_STANDARD) + SET(KOKKOS_CXX_STANDARD_FLAG "--c++${FULL_LC_STANDARD}" PARENT_SCOPE) + SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "--c++${INT_LC_STANDARD}" PARENT_SCOPE) +endfunction() + diff --git a/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake b/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake index aad1e2bad7..a59868b73b 100644 --- a/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLCUSPARSE.cmake @@ -55,19 +55,9 @@ # Check for CUDA support -IF (NOT TPL_ENABLE_CUDA OR CUDA_VERSION VERSION_LESS "4.1") - MESSAGE(FATAL_ERROR "\nCUSPARSE: did not find acceptable version of CUDA libraries (4.1 or greater)") +IF (NOT TPL_ENABLE_CUDA) + MESSAGE(FATAL_ERROR "\nCUSPARSE requires CUDA") ELSE() - IF(CMAKE_VERSION VERSION_LESS "2.8.8") - # FindCUDA before CMake 2.8.8 does not find cusparse library; therefore, we must - find_library(CUDA_cusparse_LIBRARY - cusparse - HINTS ${CUDA_TOOLKIT_ROOT_DIR}/lib - ) - IF(CUDA_cusparse_LIBRARY STREQUAL "CUDA_cusparse_LIBRARY-NOTFOUND") - MESSAGE(FATAL_ERROR "\nCUSPARSE: could not find cuspasre library.") - ENDIF() - ENDIF(CMAKE_VERSION VERSION_LESS "2.8.8") GLOBAL_SET(TPL_CUSPARSE_LIBRARY_DIRS) GLOBAL_SET(TPL_CUSPARSE_INCLUDE_DIRS ${TPL_CUDA_INCLUDE_DIRS}) GLOBAL_SET(TPL_CUSPARSE_LIBRARIES ${CUDA_cusparse_LIBRARY}) diff --git a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake index 715b3e9bde..a4c55e1d7b 100644 --- a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake @@ -64,7 +64,7 @@ # Version: 1.3 # -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC +KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc" ) diff --git a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake index fc401d7543..4dc1a87e18 100644 --- a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake @@ -75,7 +75,7 @@ IF(USE_THREADS) SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") SET(TPL_Pthread_LIBRARY_DIRS "") ELSE() - TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread + KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread ) diff --git a/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake b/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake deleted file mode 100644 index c312f2590b..0000000000 --- a/lib/kokkos/cmake/tpls/FindTPLQTHREADS.cmake +++ /dev/null @@ -1,69 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trilinos: An Object-Oriented Solver Framework -# Copyright (2001) Sandia Corporation -# -# -# Copyright (2001) Sandia Corporation. Under the terms of Contract -# DE-AC04-94AL85000, there is a non-exclusive license for use of this -# work by or on behalf of the U.S. Government. Export of this program -# may require a license from the United States Government. -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# NOTICE: The United States Government is granted for itself and others -# acting on its behalf a paid-up, nonexclusive, irrevocable worldwide -# license in this data to reproduce, prepare derivative works, and -# perform publicly and display publicly. Beginning five (5) years from -# July 25, 2001, the United States Government is granted for itself and -# others acting on its behalf a paid-up, nonexclusive, irrevocable -# worldwide license in this data to reproduce, prepare derivative works, -# distribute copies to the public, perform publicly and display -# publicly, and to permit others to do so. -# -# NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT -# OF ENERGY, NOR SANDIA CORPORATION, NOR ANY OF THEIR EMPLOYEES, MAKES -# ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR -# RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY -# INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS -# THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. -# -# ************************************************************************ -# @HEADER - - -#----------------------------------------------------------------------------- -# Hardware locality detection and control library. -# -# Acquisition information: -# Date checked: July 2014 -# Checked by: H. Carter Edwards -# Source: https://code.google.com/p/qthreads -# - -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( QTHREADS - REQUIRED_HEADERS qthread.h - REQUIRED_LIBS_NAMES "qthread" - ) diff --git a/lib/kokkos/cmake/tribits.cmake b/lib/kokkos/cmake/tribits.cmake deleted file mode 100644 index 1f467f0662..0000000000 --- a/lib/kokkos/cmake/tribits.cmake +++ /dev/null @@ -1,531 +0,0 @@ -INCLUDE(CMakeParseArguments) -INCLUDE(CTest) - -cmake_policy(SET CMP0054 NEW) - -MESSAGE(STATUS "The project name is: ${PROJECT_NAME}") - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_OpenMP) - SET(${PROJECT_NAME}_ENABLE_OpenMP OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_HPX) - SET(${PROJECT_NAME}_ENABLE_HPX OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_DEBUG) - SET(${PROJECT_NAME}_ENABLE_DEBUG OFF) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_CXX11) - SET(${PROJECT_NAME}_ENABLE_CXX11 ON) -ENDIF() - -IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_TESTS) - SET(${PROJECT_NAME}_ENABLE_TESTS OFF) -ENDIF() - -IF(NOT DEFINED TPL_ENABLE_Pthread) - SET(TPL_ENABLE_Pthread OFF) -ENDIF() - -FUNCTION(ASSERT_DEFINED VARS) - FOREACH(VAR ${VARS}) - IF(NOT DEFINED ${VAR}) - MESSAGE(SEND_ERROR "Error, the variable ${VAR} is not defined!") - ENDIF() - ENDFOREACH() -ENDFUNCTION() - -MACRO(GLOBAL_SET VARNAME) - SET(${VARNAME} ${ARGN} CACHE INTERNAL "") -ENDMACRO() - -MACRO(PREPEND_GLOBAL_SET VARNAME) - ASSERT_DEFINED(${VARNAME}) - GLOBAL_SET(${VARNAME} ${ARGN} ${${VARNAME}}) -ENDMACRO() - -#FUNCTION(REMOVE_GLOBAL_DUPLICATES VARNAME) -# ASSERT_DEFINED(${VARNAME}) -# IF (${VARNAME}) -# SET(TMP ${${VARNAME}}) -# LIST(REMOVE_DUPLICATES TMP) -# GLOBAL_SET(${VARNAME} ${TMP}) -# ENDIF() -#ENDFUNCTION() - -#MACRO(TRIBITS_ADD_OPTION_AND_DEFINE USER_OPTION_NAME MACRO_DEFINE_NAME DOCSTRING DEFAULT_VALUE) -# MESSAGE(STATUS "TRIBITS_ADD_OPTION_AND_DEFINE: '${USER_OPTION_NAME}' '${MACRO_DEFINE_NAME}' '${DEFAULT_VALUE}'") -# SET( ${USER_OPTION_NAME} "${DEFAULT_VALUE}" CACHE BOOL "${DOCSTRING}" ) -# IF(NOT ${MACRO_DEFINE_NAME} STREQUAL "") -# IF(${USER_OPTION_NAME}) -# GLOBAL_SET(${MACRO_DEFINE_NAME} ON) -# ELSE() -# GLOBAL_SET(${MACRO_DEFINE_NAME} OFF) -# ENDIF() -# ENDIF() -#ENDMACRO() - -FUNCTION(TRIBITS_CONFIGURE_FILE PACKAGE_NAME_CONFIG_FILE) - - # Configure the file - CONFIGURE_FILE( - ${PACKAGE_SOURCE_DIR}/cmake/${PACKAGE_NAME_CONFIG_FILE}.in - ${CMAKE_CURRENT_BINARY_DIR}/${PACKAGE_NAME_CONFIG_FILE} - ) - -ENDFUNCTION() - -#MACRO(TRIBITS_ADD_DEBUG_OPTION) -# TRIBITS_ADD_OPTION_AND_DEFINE( -# ${PROJECT_NAME}_ENABLE_DEBUG -# HAVE_${PROJECT_NAME_UC}_DEBUG -# "Enable a host of runtime debug checking." -# OFF -# ) -#ENDMACRO() - - -MACRO(TRIBITS_ADD_TEST_DIRECTORIES) - IF(${${PROJECT_NAME}_ENABLE_TESTS}) - FOREACH(TEST_DIR ${ARGN}) - ADD_SUBDIRECTORY(${TEST_DIR}) - ENDFOREACH() - ENDIF() -ENDMACRO() - -MACRO(TRIBITS_ADD_EXAMPLE_DIRECTORIES) - IF(${PACKAGE_NAME}_ENABLE_EXAMPLES OR ${PARENT_PACKAGE_NAME}_ENABLE_EXAMPLES) - FOREACH(EXAMPLE_DIR ${ARGN}) - ADD_SUBDIRECTORY(${EXAMPLE_DIR}) - ENDFOREACH() - ENDIF() -ENDMACRO() - - -function(INCLUDE_DIRECTORIES) - cmake_parse_arguments(INCLUDE_DIRECTORIES "REQUIRED_DURING_INSTALLATION_TESTING" "" "" ${ARGN}) - _INCLUDE_DIRECTORIES(${INCLUDE_DIRECTORIES_UNPARSED_ARGUMENTS}) -endfunction() - - -MACRO(TARGET_TRANSFER_PROPERTY TARGET_NAME PROP_IN PROP_OUT) - SET(PROP_VALUES) - FOREACH(TARGET_X ${ARGN}) - LIST(APPEND PROP_VALUES "$") - ENDFOREACH() - SET_TARGET_PROPERTIES(${TARGET_NAME} PROPERTIES ${PROP_OUT} "${PROP_VALUES}") -ENDMACRO() - -MACRO(ADD_INTERFACE_LIBRARY LIB_NAME) - FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") - ADD_LIBRARY(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) - SET_TARGET_PROPERTIES(${LIB_NAME} PROPERTIES INTERFACE TRUE) -ENDMACRO() - -# Older versions of cmake does not make include directories transitive -MACRO(TARGET_LINK_AND_INCLUDE_LIBRARIES TARGET_NAME) - TARGET_LINK_LIBRARIES(${TARGET_NAME} LINK_PUBLIC ${ARGN}) - FOREACH(DEP_LIB ${ARGN}) - TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC $) - TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC $) - ENDFOREACH() -ENDMACRO() - -FUNCTION(TRIBITS_ADD_LIBRARY LIBRARY_NAME) - - SET(options STATIC SHARED TESTONLY NO_INSTALL_LIB_OR_HEADERS CUDALIBRARY) - SET(oneValueArgs) - SET(multiValueArgs HEADERS HEADERS_INSTALL_SUBDIR NOINSTALLHEADERS SOURCES DEPLIBS IMPORTEDLIBS DEFINES ADDED_LIB_TARGET_NAME_OUT) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - IF(PARSE_HEADERS) - LIST(REMOVE_DUPLICATES PARSE_HEADERS) - ENDIF() - IF(PARSE_SOURCES) - LIST(REMOVE_DUPLICATES PARSE_SOURCES) - ENDIF() - - # Local variable to hold all of the libraries that will be directly linked - # to this library. - SET(LINK_LIBS ${${PACKAGE_NAME}_DEPS}) - - # Add dependent libraries passed directly in - - IF (PARSE_IMPORTEDLIBS) - LIST(APPEND LINK_LIBS ${PARSE_IMPORTEDLIBS}) - ENDIF() - - IF (PARSE_DEPLIBS) - LIST(APPEND LINK_LIBS ${PARSE_DEPLIBS}) - ENDIF() - - # Add the library and all the dependencies - - IF (PARSE_DEFINES) - ADD_DEFINITIONS(${PARSE_DEFINES}) - ENDIF() - - IF (PARSE_STATIC) - SET(STATIC_KEYWORD "STATIC") - ELSE() - SET(STATIC_KEYWORD) - ENDIF() - - IF (PARSE_SHARED) - SET(SHARED_KEYWORD "SHARED") - ELSE() - SET(SHARED_KEYWORD) - ENDIF() - - IF (PARSE_TESTONLY) - SET(EXCLUDE_FROM_ALL_KEYWORD "EXCLUDE_FROM_ALL") - ELSE() - SET(EXCLUDE_FROM_ALL_KEYWORD) - ENDIF() - IF (NOT PARSE_CUDALIBRARY) - ADD_LIBRARY( - ${LIBRARY_NAME} - ${STATIC_KEYWORD} - ${SHARED_KEYWORD} - ${EXCLUDE_FROM_ALL_KEYWORD} - ${PARSE_HEADERS} - ${PARSE_NOINSTALLHEADERS} - ${PARSE_SOURCES} - ) - ELSE() - CUDA_ADD_LIBRARY( - ${LIBRARY_NAME} - ${PARSE_HEADERS} - ${PARSE_NOINSTALLHEADERS} - ${PARSE_SOURCES} - ) - ENDIF() - - TARGET_LINK_AND_INCLUDE_LIBRARIES(${LIBRARY_NAME} ${LINK_LIBS}) - - IF (NOT PARSE_TESTONLY OR PARSE_NO_INSTALL_LIB_OR_HEADERS) - - INSTALL( - TARGETS ${LIBRARY_NAME} - EXPORT ${PROJECT_NAME} - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - COMPONENT ${PACKAGE_NAME} - ) - - INSTALL( - FILES ${PARSE_HEADERS} - EXPORT ${PROJECT_NAME} - DESTINATION include - COMPONENT ${PACKAGE_NAME} - ) - - INSTALL( - DIRECTORY ${PARSE_HEADERS_INSTALL_SUBDIR} - EXPORT ${PROJECT_NAME} - DESTINATION include - COMPONENT ${PACKAGE_NAME} - ) - - ENDIF() - - IF (NOT PARSE_TESTONLY) - PREPEND_GLOBAL_SET(${PACKAGE_NAME}_LIBS ${LIBRARY_NAME}) - REMOVE_GLOBAL_DUPLICATES(${PACKAGE_NAME}_LIBS) - ENDIF() - -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_EXECUTABLE EXE_NAME) - - SET(options NOEXEPREFIX NOEXESUFFIX ADD_DIR_TO_NAME INSTALLABLE TESTONLY) - SET(oneValueArgs ADDED_EXE_TARGET_NAME_OUT) - SET(multiValueArgs SOURCES CATEGORIES HOST XHOST HOSTTYPE XHOSTTYPE DIRECTORY TESTONLYLIBS IMPORTEDLIBS DEPLIBS COMM LINKER_LANGUAGE TARGET_DEFINES DEFINES) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - IF (PARSE_TARGET_DEFINES) - TARGET_COMPILE_DEFINITIONS(${EXE_NAME} PUBLIC ${PARSE_TARGET_DEFINES}) - ENDIF() - - SET(LINK_LIBS PACKAGE_${PACKAGE_NAME}) - - IF (PARSE_TESTONLYLIBS) - LIST(APPEND LINK_LIBS ${PARSE_TESTONLYLIBS}) - ENDIF() - - IF (PARSE_IMPORTEDLIBS) - LIST(APPEND LINK_LIBS ${PARSE_IMPORTEDLIBS}) - ENDIF() - - SET (EXE_SOURCES) - IF(PARSE_DIRECTORY) - FOREACH( SOURCE_FILE ${PARSE_SOURCES} ) - IF(IS_ABSOLUTE ${SOURCE_FILE}) - SET (EXE_SOURCES ${EXE_SOURCES} ${SOURCE_FILE}) - ELSE() - SET (EXE_SOURCES ${EXE_SOURCES} ${PARSE_DIRECTORY}/${SOURCE_FILE}) - ENDIF() - ENDFOREACH( ) - ELSE() - FOREACH( SOURCE_FILE ${PARSE_SOURCES} ) - SET (EXE_SOURCES ${EXE_SOURCES} ${SOURCE_FILE}) - ENDFOREACH( ) - ENDIF() - - SET(EXE_BINARY_NAME ${EXE_NAME}) - IF(DEFINED PACKAGE_NAME AND NOT PARSE_NOEXEPREFIX) - SET(EXE_BINARY_NAME ${PACKAGE_NAME}_${EXE_BINARY_NAME}) - ENDIF() - - # IF (PARSE_TESTONLY) - # SET(EXCLUDE_FROM_ALL_KEYWORD "EXCLUDE_FROM_ALL") - # ELSE() - # SET(EXCLUDE_FROM_ALL_KEYWORD) - # ENDIF() - ADD_EXECUTABLE(${EXE_BINARY_NAME} ${EXCLUDE_FROM_ALL_KEYWORD} ${EXE_SOURCES}) - - TARGET_LINK_AND_INCLUDE_LIBRARIES(${EXE_BINARY_NAME} ${LINK_LIBS}) - - IF(PARSE_ADDED_EXE_TARGET_NAME_OUT) - SET(${PARSE_ADDED_EXE_TARGET_NAME_OUT} ${EXE_BINARY_NAME} PARENT_SCOPE) - ENDIF() - - IF(PARSE_INSTALLABLE) - INSTALL( - TARGETS ${EXE_BINARY_NAME} - EXPORT ${PROJECT_NAME} - DESTINATION bin - ) - ENDIF() -ENDFUNCTION() - -IF(NOT TARGET check) - ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} -VV -C ${CMAKE_CFG_INTDIR}) -ENDIF() - -FUNCTION(TRIBITS_ADD_TEST) -ENDFUNCTION() -FUNCTION(TRIBITS_TPL_TENTATIVELY_ENABLE) -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_ADVANCED_TEST) - # TODO Write this -ENDFUNCTION() - -FUNCTION(TRIBITS_ADD_EXECUTABLE_AND_TEST EXE_NAME) - - SET(options STANDARD_PASS_OUTPUT WILL_FAIL) - SET(oneValueArgs PASS_REGULAR_EXPRESSION FAIL_REGULAR_EXPRESSION ENVIRONMENT TIMEOUT CATEGORIES ADDED_TESTS_NAMES_OUT ADDED_EXE_TARGET_NAME_OUT) - SET(multiValueArgs) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - TRIBITS_ADD_EXECUTABLE(${EXE_NAME} TESTONLY ADDED_EXE_TARGET_NAME_OUT TEST_NAME ${PARSE_UNPARSED_ARGUMENTS}) - - IF(WIN32) - ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} COMMAND ${TEST_NAME}${CMAKE_EXECUTABLE_SUFFIX}) - ELSE() - ADD_TEST(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) - ENDIF() - ADD_DEPENDENCIES(check ${TEST_NAME}) - - IF(PARSE_FAIL_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${PARSE_FAIL_REGULAR_EXPRESSION}) - ENDIF() - - IF(PARSE_PASS_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${PARSE_PASS_REGULAR_EXPRESSION}) - ENDIF() - - IF(PARSE_WILL_FAIL) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${PARSE_WILL_FAIL}) - ENDIF() - - IF(PARSE_ADDED_TESTS_NAMES_OUT) - SET(${PARSE_ADDED_TESTS_NAMES_OUT} ${TEST_NAME} PARENT_SCOPE) - ENDIF() - - IF(PARSE_ADDED_EXE_TARGET_NAME_OUT) - SET(${PARSE_ADDED_EXE_TARGET_NAME_OUT} ${TEST_NAME} PARENT_SCOPE) - ENDIF() - -ENDFUNCTION() - -MACRO(TIBITS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) - ADD_INTERFACE_LIBRARY(TPL_LIB_${TPL_NAME}) - TARGET_LINK_LIBRARIES(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) - TARGET_INCLUDE_DIRECTORIES(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) -ENDMACRO() - -FUNCTION(TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) - - SET(options MUST_FIND_ALL_LIBS MUST_FIND_ALL_HEADERS NO_PRINT_ENABLE_SUCCESS_FAIL) - SET(oneValueArgs) - SET(multiValueArgs REQUIRED_HEADERS REQUIRED_LIBS_NAMES) - - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - SET(_${TPL_NAME}_ENABLE_SUCCESS TRUE) - IF (PARSE_REQUIRED_LIBS_NAMES) - FIND_LIBRARY(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) - IF(NOT TPL_${TPL_NAME}_LIBRARIES) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - IF (PARSE_REQUIRED_HEADERS) - FIND_PATH(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) - IF(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - - - IF (_${TPL_NAME}_ENABLE_SUCCESS) - TIBITS_CREATE_IMPORTED_TPL_LIBRARY(${TPL_NAME}) - ENDIF() - -ENDFUNCTION() - -#MACRO(TRIBITS_PROCESS_TPL_DEP_FILE TPL_FILE) -# GET_FILENAME_COMPONENT(TPL_NAME ${TPL_FILE} NAME_WE) -# INCLUDE("${TPL_FILE}") -# IF(TARGET TPL_LIB_${TPL_NAME}) -# MESSAGE(STATUS "Found tpl library: ${TPL_NAME}") -# SET(TPL_ENABLE_${TPL_NAME} TRUE) -# ELSE() -# MESSAGE(STATUS "Tpl library not found: ${TPL_NAME}") -# SET(TPL_ENABLE_${TPL_NAME} FALSE) -# ENDIF() -#ENDMACRO() - -MACRO(PREPEND_TARGET_SET VARNAME TARGET_NAME TYPE) - IF(TYPE STREQUAL "REQUIRED") - SET(REQUIRED TRUE) - ELSE() - SET(REQUIRED FALSE) - ENDIF() - IF(TARGET ${TARGET_NAME}) - PREPEND_GLOBAL_SET(${VARNAME} ${TARGET_NAME}) - ELSE() - IF(REQUIRED) - MESSAGE(FATAL_ERROR "Missing dependency ${TARGET_NAME}") - ENDIF() - ENDIF() -ENDMACRO() - -MACRO(TRIBITS_APPEND_PACKAGE_DEPS DEP_LIST TYPE) - FOREACH(DEP ${ARGN}) - PREPEND_GLOBAL_SET(${DEP_LIST} PACKAGE_${DEP}) - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_APPEND_TPLS_DEPS DEP_LIST TYPE) - FOREACH(DEP ${ARGN}) - PREPEND_TARGET_SET(${DEP_LIST} TPL_LIB_${DEP} ${TYPE}) - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_ENABLE_TPLS) - FOREACH(TPL ${ARGN}) - IF(TARGET ${TPL}) - GLOBAL_SET(${PACKAGE_NAME}_ENABLE_${TPL} TRUE) - ELSE() - GLOBAL_SET(${PACKAGE_NAME}_ENABLE_${TPL} FALSE) - ENDIF() - ENDFOREACH() -ENDMACRO() - -MACRO(TRIBITS_PACKAGE_DEFINE_DEPENDENCIES) - - SET(options) - SET(oneValueArgs) - SET(multiValueArgs - LIB_REQUIRED_PACKAGES - LIB_OPTIONAL_PACKAGES - TEST_REQUIRED_PACKAGES - TEST_OPTIONAL_PACKAGES - LIB_REQUIRED_TPLS - LIB_OPTIONAL_TPLS - TEST_REQUIRED_TPLS - TEST_OPTIONAL_TPLS - REGRESSION_EMAIL_LIST - SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS - ) - CMAKE_PARSE_ARGUMENTS(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - GLOBAL_SET(${PACKAGE_NAME}_DEPS "") - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_DEPS REQUIRED ${PARSE_LIB_REQUIRED_PACKAGES}) - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_DEPS OPTIONAL ${PARSE_LIB_OPTIONAL_PACKAGES}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_DEPS REQUIRED ${PARSE_LIB_REQUIRED_TPLS}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_DEPS OPTIONAL ${PARSE_LIB_OPTIONAL_TPLS}) - - GLOBAL_SET(${PACKAGE_NAME}_TEST_DEPS "") - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_TEST_DEPS REQUIRED ${PARSE_TEST_REQUIRED_PACKAGES}) - TRIBITS_APPEND_PACKAGE_DEPS(${PACKAGE_NAME}_TEST_DEPS OPTIONAL ${PARSE_TEST_OPTIONAL_PACKAGES}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_TEST_DEPS REQUIRED ${PARSE_TEST_REQUIRED_TPLS}) - TRIBITS_APPEND_TPLS_DEPS(${PACKAGE_NAME}_TEST_DEPS OPTIONAL ${PARSE_TEST_OPTIONAL_TPLS}) - - TRIBITS_ENABLE_TPLS(${PARSE_LIB_REQUIRED_TPLS} ${PARSE_LIB_OPTIONAL_TPLS} ${PARSE_TEST_REQUIRED_TPLS} ${PARSE_TEST_OPTIONAL_TPLS}) - -ENDMACRO() - -MACRO(TRIBITS_SUBPACKAGE NAME) - SET(PACKAGE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - SET(PARENT_PACKAGE_NAME ${PACKAGE_NAME}) - SET(PACKAGE_NAME ${PACKAGE_NAME}${NAME}) - STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) - SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - - ADD_INTERFACE_LIBRARY(PACKAGE_${PACKAGE_NAME}) - - GLOBAL_SET(${PACKAGE_NAME}_LIBS "") - - INCLUDE(${PACKAGE_SOURCE_DIR}/cmake/Dependencies.cmake) - -ENDMACRO(TRIBITS_SUBPACKAGE) - -MACRO(TRIBITS_SUBPACKAGE_POSTPROCESS) - TARGET_LINK_AND_INCLUDE_LIBRARIES(PACKAGE_${PACKAGE_NAME} ${${PACKAGE_NAME}_LIBS}) -ENDMACRO(TRIBITS_SUBPACKAGE_POSTPROCESS) - -MACRO(TRIBITS_PACKAGE_DECL NAME) - - SET(PACKAGE_NAME ${NAME}) - SET(${PACKAGE_NAME}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - STRING(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UC) - - #SET(TRIBITS_DEPS_DIR "${CMAKE_SOURCE_DIR}/cmake/deps") - #FILE(GLOB TPLS_FILES "${TRIBITS_DEPS_DIR}/*.cmake") - #FOREACH(TPL_FILE ${TPLS_FILES}) - # TRIBITS_PROCESS_TPL_DEP_FILE(${TPL_FILE}) - #ENDFOREACH() - -ENDMACRO() - - -MACRO(TRIBITS_PROCESS_SUBPACKAGES) - FILE(GLOB SUBPACKAGES RELATIVE ${CMAKE_SOURCE_DIR} */cmake/Dependencies.cmake) - FOREACH(SUBPACKAGE ${SUBPACKAGES}) - GET_FILENAME_COMPONENT(SUBPACKAGE_CMAKE ${SUBPACKAGE} DIRECTORY) - GET_FILENAME_COMPONENT(SUBPACKAGE_DIR ${SUBPACKAGE_CMAKE} DIRECTORY) - ADD_SUBDIRECTORY(${CMAKE_BINARY_DIR}/../${SUBPACKAGE_DIR}) - ENDFOREACH() -ENDMACRO(TRIBITS_PROCESS_SUBPACKAGES) - -MACRO(TRIBITS_PACKAGE_DEF) -ENDMACRO(TRIBITS_PACKAGE_DEF) - -MACRO(TRIBITS_EXCLUDE_AUTOTOOLS_FILES) -ENDMACRO(TRIBITS_EXCLUDE_AUTOTOOLS_FILES) - -MACRO(TRIBITS_EXCLUDE_FILES) -ENDMACRO(TRIBITS_EXCLUDE_FILES) - -MACRO(TRIBITS_PACKAGE_POSTPROCESS) -ENDMACRO(TRIBITS_PACKAGE_POSTPROCESS) - diff --git a/lib/kokkos/config/test_all_sandia b/lib/kokkos/config/test_all_sandia index d94c38cbc6..193a162a4e 100755 --- a/lib/kokkos/config/test_all_sandia +++ b/lib/kokkos/config/test_all_sandia @@ -76,19 +76,18 @@ CLANG_BUILD_LIST="Pthread,Serial,Pthread_Serial" CUDA_BUILD_LIST="Cuda_OpenMP,Cuda_Pthread,Cuda_Serial" CUDA_IBM_BUILD_LIST="Cuda_OpenMP,Cuda_Serial" -GCC_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wignored-qualifiers,-Wempty-body,-Wclobbered,-Wuninitialized" -IBM_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" -CLANG_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" -INTEL_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" -#CUDA_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" -CUDA_WARNING_FLAGS="-Wall,-Wshadow,-pedantic,-Wsign-compare,-Wtype-limits,-Wuninitialized" +GCC_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wignored-qualifiers,-Wempty-body,-Wclobbered,-Wuninitialized" +IBM_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" +CLANG_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" +INTEL_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" +#CUDA_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Werror,-Wsign-compare,-Wtype-limits,-Wuninitialized" +CUDA_WARNING_FLAGS="-Wall,-Wunused-parameter,-Wshadow,-pedantic,-Wsign-compare,-Wtype-limits,-Wuninitialized" PGI_WARNING_FLAGS="" # Default. Machine specific can override. DEBUG=False ARGS="" CUSTOM_BUILD_LIST="" -QTHREADS_PATH="" DRYRUN=False BUILD_ONLY=False declare -i NUM_JOBS_TO_RUN_IN_PARALLEL=1 @@ -114,9 +113,6 @@ do --kokkos-path*) KOKKOS_PATH="${key#*=}" ;; - --qthreads-path*) - QTHREADS_PATH="${key#*=}" - ;; --build-list*) CUSTOM_BUILD_LIST="${key#*=}" ;; @@ -417,8 +413,8 @@ if [ "$PRINT_HELP" = "True" ]; then echo "--build-list=BUILD,BUILD,BUILD..." echo " Provide a comma-separated list of builds instead of running all builds" echo " Valid items:" - echo " OpenMP, Pthread, Qthreads, Serial, OpenMP_Serial, Pthread_Serial" - echo " Qthreads_Serial, Cuda_OpenMP, Cuda_Pthread, Cuda_Serial" + echo " OpenMP, Pthread, Serial, OpenMP_Serial, Pthread_Serial" + echo " Cuda_OpenMP, Cuda_Pthread, Cuda_Serial" echo "" echo "ARGS: list of expressions matching compilers to test" @@ -483,33 +479,6 @@ for ARG in $ARGS; do done done -# Check if Qthreads build requested. -HAVE_QTHREADS_BUILD="False" -if [ -n "$CUSTOM_BUILD_LIST" ]; then - if [[ "$CUSTOM_BUILD_LIST" = *Qthreads* ]]; then - HAVE_QTHREADS_BUILD="True" - fi -else - for COMPILER_DATA in "${COMPILERS[@]}"; do - ARR=($COMPILER_DATA) - BUILD_LIST=${ARR[2]} - if [[ "$BUILD_LIST" = *Qthreads* ]]; then - HAVE_QTHREADS_BUILD="True" - fi - done -fi - -# Ensure Qthreads path is set if Qthreads build is requested. -if [ "$HAVE_QTHREADS_BUILD" = "True" ]; then - if [ -z "$QTHREADS_PATH" ]; then - echo "Need to supply Qthreads path (--qthreads-path) when testing Qthreads backend." >&2 - exit 1 - else - # Strip trailing slashes from path. - QTHREADS_PATH=$(echo $QTHREADS_PATH | sed 's/\/*$//') - fi -fi - # # Functions. # @@ -627,14 +596,6 @@ single_build_and_test() { local extra_args=--with-hwloc=$(dirname $(dirname $(which hwloc-info))) fi - if [[ "$build" = *Qthreads* ]]; then - if [[ "$build_type" = hwloc* ]]; then - local extra_args="$extra_args --qthreads-path=${QTHREADS_PATH}_hwloc" - else - local extra_args="$extra_args --qthreads-path=$QTHREADS_PATH" - fi - fi - if [[ "$OPT_FLAG" = "" ]]; then OPT_FLAG="-O3" fi diff --git a/lib/kokkos/containers/CMakeLists.txt b/lib/kokkos/containers/CMakeLists.txt index c37aa3e3e2..2bfaea7a13 100644 --- a/lib/kokkos/containers/CMakeLists.txt +++ b/lib/kokkos/containers/CMakeLists.txt @@ -1,13 +1,10 @@ - - -TRIBITS_SUBPACKAGE(Containers) - - -IF(KOKKOS_HAS_TRILINOS) - ADD_SUBDIRECTORY(src) -ENDIF() - -TRIBITS_ADD_TEST_DIRECTORIES(unit_tests) -TRIBITS_ADD_TEST_DIRECTORIES(performance_tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() + + +KOKKOS_SUBPACKAGE(Containers) + +ADD_SUBDIRECTORY(src) + +KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) +KOKKOS_ADD_TEST_DIRECTORIES(performance_tests) + +KOKKOS_SUBPACKAGE_POSTPROCESS() diff --git a/lib/kokkos/containers/performance_tests/CMakeLists.txt b/lib/kokkos/containers/performance_tests/CMakeLists.txt index 3c6584bc34..1011cb8fd1 100644 --- a/lib/kokkos/containers/performance_tests/CMakeLists.txt +++ b/lib/kokkos/containers/performance_tests/CMakeLists.txt @@ -1,49 +1,46 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) -IF(NOT KOKKOS_HAS_TRILINOS) - IF(KOKKOS_SEPARATE_LIBS) - set(TEST_LINK_TARGETS kokkoscore) - ELSE() - set(TEST_LINK_TARGETS kokkos) - ENDIF() +IF(Kokkos_ENABLE_CUDA) + SET(SOURCES + TestMain.cpp + TestCuda.cpp + ) + + KOKKOS_ADD_EXECUTABLE_AND_TEST( PerformanceTest_Cuda + SOURCES ${SOURCES} + ) ENDIF() -SET(SOURCES - TestMain.cpp - TestCuda.cpp - ) - -IF(Kokkos_ENABLE_Pthread) - LIST( APPEND SOURCES TestThreads.cpp) +IF(Kokkos_ENABLE_PTHREAD) + SET(SOURCES + TestMain.cpp + TestThreads.cpp + ) + KOKKOS_ADD_EXECUTABLE_AND_TEST( PerformanceTest_Threads + SOURCES ${SOURCES} + ) ENDIF() -IF(Kokkos_ENABLE_OpenMP) - LIST( APPEND SOURCES TestOpenMP.cpp) +IF(Kokkos_ENABLE_OPENMP) + SET(SOURCES + TestMain.cpp + TestOpenMP.cpp + ) + KOKKOS_ADD_EXECUTABLE_AND_TEST( PerformanceTest_OpenMP + SOURCES ${SOURCES} + ) ENDIF() IF(Kokkos_ENABLE_HPX) - LIST( APPEND SOURCES TestHPX.cpp) + SET(SOURCES + TestMain.cpp + TestHPX.cpp + ) + KOKKOS_ADD_EXECUTABLE_AND_TEST( PerformanceTest_HPX + SOURCES ${SOURCES} + ) ENDIF() -# Per #374, we always want to build this test, but we only want to run -# it as a PERFORMANCE test. That's why we separate building the test -# from running the test. - -TRIBITS_ADD_EXECUTABLE( - PerfTestExec - SOURCES ${SOURCES} - COMM serial mpi - TESTONLYLIBS kokkos_gtest ${TEST_LINK_TARGETS} - ) - -TRIBITS_ADD_TEST( - PerformanceTest - NAME PerfTestExec - COMM serial mpi - NUM_MPI_PROCS 1 - CATEGORIES PERFORMANCE - FAIL_REGULAR_EXPRESSION " FAILED " - ) diff --git a/lib/kokkos/containers/performance_tests/TestCuda.cpp b/lib/kokkos/containers/performance_tests/TestCuda.cpp index 351fb86df3..697a006c3c 100644 --- a/lib/kokkos/containers/performance_tests/TestCuda.cpp +++ b/lib/kokkos/containers/performance_tests/TestCuda.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_CUDA ) +#if defined(KOKKOS_ENABLE_CUDA) #include #include @@ -66,45 +67,38 @@ namespace Performance { class cuda : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::InitArguments args(-1, -1, 0); Kokkos::initialize(args); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( cuda, dynrankview_perf ) -{ +TEST_F(cuda, dynrankview_perf) { std::cout << "Cuda" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 40960 ); + test_dynrankview_op_perf(40960); } -TEST_F( cuda, global_2_local) -{ +TEST_F(cuda, global_2_local) { std::cout << "Cuda" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( cuda, unordered_map_performance_near) -{ - Perf::run_performance_tests("cuda-near"); +TEST_F(cuda, unordered_map_performance_near) { + Perf::run_performance_tests("cuda-near"); } -TEST_F( cuda, unordered_map_performance_far) -{ - Perf::run_performance_tests("cuda-far"); +TEST_F(cuda, unordered_map_performance_far) { + Perf::run_performance_tests("cuda-far"); } -} +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTCUDA_PREVENT_EMPTY_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ +#endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ diff --git a/lib/kokkos/containers/performance_tests/TestDynRankView.hpp b/lib/kokkos/containers/performance_tests/TestDynRankView.hpp index db6274e057..ee13f7e58b 100644 --- a/lib/kokkos/containers/performance_tests/TestDynRankView.hpp +++ b/lib/kokkos/containers/performance_tests/TestDynRankView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -49,109 +50,102 @@ #include -// Compare performance of DynRankView to View, specific focus on the parenthesis operators +// Compare performance of DynRankView to View, specific focus on the parenthesis +// operators namespace Performance { -//View functor +// View functor template struct InitViewFunctor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - struct SumComputationTest - { - typedef Kokkos::View inviewtype; + struct SumComputationTest { + typedef Kokkos::View inviewtype; inviewtype _inview; - typedef Kokkos::View outviewtype; + typedef Kokkos::View outviewtype; outviewtype _outview; KOKKOS_INLINE_FUNCTION - SumComputationTest(inviewtype &inview_ , outviewtype &outview_) : _inview(inview_), _outview(outview_) {} + SumComputationTest(inviewtype &inview_, outviewtype &outview_) + : _inview(inview_), _outview(outview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _outview(i) += _inview(i,j,k) ; + _outview(i) += _inview(i, j, k); } } } }; - }; template struct InitStrideViewFunctor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitStrideViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitStrideViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - }; template struct InitViewRank7Functor { - typedef Kokkos::View inviewtype; + typedef Kokkos::View inviewtype; inviewtype _inview; - InitViewRank7Functor( inviewtype &inview_ ) : _inview(inview_) - {} + InitViewRank7Functor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k,0,0,0,0) = i/2 -j*j + k/3; + _inview(i, j, k, 0, 0, 0, 0) = i / 2 - j * j + k / 3; } } } - }; -//DynRankView functor +// DynRankView functor template struct InitDynRankViewFunctor { typedef Kokkos::DynRankView inviewtype; inviewtype _inview; - InitDynRankViewFunctor( inviewtype &inview_ ) : _inview(inview_) - {} + InitDynRankViewFunctor(inviewtype &inview_) : _inview(inview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _inview(i,j,k) = i/2 -j*j + k/3; + _inview(i, j, k) = i / 2 - j * j + k / 3; } } } - struct SumComputationTest - { + struct SumComputationTest { typedef Kokkos::DynRankView inviewtype; inviewtype _inview; @@ -159,108 +153,121 @@ struct InitDynRankViewFunctor { outviewtype _outview; KOKKOS_INLINE_FUNCTION - SumComputationTest(inviewtype &inview_ , outviewtype &outview_) : _inview(inview_), _outview(outview_) {} + SumComputationTest(inviewtype &inview_, outviewtype &outview_) + : _inview(inview_), _outview(outview_) {} KOKKOS_INLINE_FUNCTION void operator()(const int i) const { for (unsigned j = 0; j < _inview.extent(1); ++j) { for (unsigned k = 0; k < _inview.extent(2); ++k) { - _outview(i) += _inview(i,j,k) ; + _outview(i) += _inview(i, j, k); } } } }; - }; - template -void test_dynrankview_op_perf( const int par_size ) -{ - +void test_dynrankview_op_perf(const int par_size) { typedef DeviceType execution_space; typedef typename execution_space::size_type size_type; const size_type dim_2 = 90; const size_type dim_3 = 30; - double elapsed_time_view = 0; - double elapsed_time_compview = 0; + double elapsed_time_view = 0; + double elapsed_time_compview = 0; double elapsed_time_strideview = 0; double elapsed_time_view_rank7 = 0; - double elapsed_time_drview = 0; + double elapsed_time_drview = 0; double elapsed_time_compdrview = 0; Kokkos::Timer timer; { - Kokkos::View testview("testview",par_size,dim_2,dim_3); + Kokkos::View testview("testview", par_size, dim_2, + dim_3); typedef InitViewFunctor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testview)); DeviceType().fence(); elapsed_time_view = timer.seconds(); std::cout << " View time (init only): " << elapsed_time_view << std::endl; - timer.reset(); - Kokkos::View sumview("sumview",par_size); - Kokkos::parallel_for( policy , typename FunctorType::SumComputationTest(testview, sumview) ); + Kokkos::View sumview("sumview", par_size); + Kokkos::parallel_for( + policy, typename FunctorType::SumComputationTest(testview, sumview)); DeviceType().fence(); elapsed_time_compview = timer.seconds(); - std::cout << " View sum computation time: " << elapsed_time_view << std::endl; + std::cout << " View sum computation time: " << elapsed_time_view + << std::endl; - - Kokkos::View teststrideview = Kokkos::subview(testview, Kokkos::ALL, Kokkos::ALL,Kokkos::ALL); + Kokkos::View teststrideview = + Kokkos::subview(testview, Kokkos::ALL, Kokkos::ALL, Kokkos::ALL); typedef InitStrideViewFunctor FunctorStrideType; timer.reset(); - Kokkos::parallel_for( policy , FunctorStrideType(teststrideview) ); + Kokkos::parallel_for(policy, FunctorStrideType(teststrideview)); DeviceType().fence(); elapsed_time_strideview = timer.seconds(); - std::cout << " Strided View time (init only): " << elapsed_time_strideview << std::endl; + std::cout << " Strided View time (init only): " << elapsed_time_strideview + << std::endl; } { - Kokkos::View testview("testview",par_size,dim_2,dim_3,1,1,1,1); + Kokkos::View testview("testview", par_size, + dim_2, dim_3, 1, 1, 1, 1); typedef InitViewRank7Functor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testview)); DeviceType().fence(); elapsed_time_view_rank7 = timer.seconds(); - std::cout << " View Rank7 time (init only): " << elapsed_time_view_rank7 << std::endl; + std::cout << " View Rank7 time (init only): " << elapsed_time_view_rank7 + << std::endl; } { - Kokkos::DynRankView testdrview("testdrview",par_size,dim_2,dim_3); + Kokkos::DynRankView testdrview("testdrview", par_size, + dim_2, dim_3); typedef InitDynRankViewFunctor FunctorType; timer.reset(); - Kokkos::RangePolicy policy(0,par_size); - Kokkos::parallel_for( policy , FunctorType(testdrview) ); + Kokkos::RangePolicy policy(0, par_size); + Kokkos::parallel_for(policy, FunctorType(testdrview)); DeviceType().fence(); elapsed_time_drview = timer.seconds(); - std::cout << " DynRankView time (init only): " << elapsed_time_drview << std::endl; + std::cout << " DynRankView time (init only): " << elapsed_time_drview + << std::endl; timer.reset(); - Kokkos::DynRankView sumview("sumview",par_size); - Kokkos::parallel_for( policy , typename FunctorType::SumComputationTest(testdrview, sumview) ); + Kokkos::DynRankView sumview("sumview", par_size); + Kokkos::parallel_for( + policy, typename FunctorType::SumComputationTest(testdrview, sumview)); DeviceType().fence(); elapsed_time_compdrview = timer.seconds(); - std::cout << " DynRankView sum computation time: " << elapsed_time_compdrview << std::endl; - + std::cout << " DynRankView sum computation time: " + << elapsed_time_compdrview << std::endl; } - std::cout << " Ratio of View to DynRankView time: " << elapsed_time_view / elapsed_time_drview << std::endl; //expect < 1 - std::cout << " Ratio of View to DynRankView sum computation time: " << elapsed_time_compview / elapsed_time_compdrview << std::endl; //expect < 1 - std::cout << " Ratio of View to View Rank7 time: " << elapsed_time_view / elapsed_time_view_rank7 << std::endl; //expect < 1 - std::cout << " Ratio of StrideView to DynRankView time: " << elapsed_time_strideview / elapsed_time_drview << std::endl; //expect < 1 - std::cout << " Ratio of DynRankView to View Rank7 time: " << elapsed_time_drview / elapsed_time_view_rank7 << std::endl; //expect ? + std::cout << " Ratio of View to DynRankView time: " + << elapsed_time_view / elapsed_time_drview + << std::endl; // expect < 1 + std::cout << " Ratio of View to DynRankView sum computation time: " + << elapsed_time_compview / elapsed_time_compdrview + << std::endl; // expect < 1 + std::cout << " Ratio of View to View Rank7 time: " + << elapsed_time_view / elapsed_time_view_rank7 + << std::endl; // expect < 1 + std::cout << " Ratio of StrideView to DynRankView time: " + << elapsed_time_strideview / elapsed_time_drview + << std::endl; // expect < 1 + std::cout << " Ratio of DynRankView to View Rank7 time: " + << elapsed_time_drview / elapsed_time_view_rank7 + << std::endl; // expect ? timer.reset(); -} //end test_dynrankview +} // end test_dynrankview - -} //end Performance +} // namespace Performance #endif - diff --git a/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp b/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp index 98997b3239..0d2ee4bc8d 100644 --- a/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp +++ b/lib/kokkos/containers/performance_tests/TestGlobal2LocalIds.hpp @@ -1,12 +1,13 @@ //@HEADER // ************************************************************************ -// -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -35,7 +36,7 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// +// // ************************************************************************ //@HEADER @@ -54,153 +55,137 @@ namespace Performance { static const unsigned begin_id_size = 256u; -static const unsigned end_id_size = 1u << 22; -static const unsigned id_step = 2u; +static const unsigned end_id_size = 1u << 22; +static const unsigned id_step = 2u; -union helper -{ +union helper { uint32_t word; uint8_t byte[4]; }; - template -struct generate_ids -{ +struct generate_ids { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; + typedef Kokkos::View local_id_view; local_id_view local_2_global; - generate_ids( local_id_view & ids) - : local_2_global(ids) - { + generate_ids(local_id_view& ids) : local_2_global(ids) { Kokkos::parallel_for(local_2_global.extent(0), *this); } - KOKKOS_INLINE_FUNCTION - void operator()(size_type i) const - { - + void operator()(size_type i) const { helper x = {static_cast(i)}; // shuffle the bytes of i to create a unique, semi-random global_id x.word = ~x.word; uint8_t tmp = x.byte[3]; - x.byte[3] = x.byte[1]; - x.byte[1] = tmp; + x.byte[3] = x.byte[1]; + x.byte[1] = tmp; - tmp = x.byte[2]; + tmp = x.byte[2]; x.byte[2] = x.byte[0]; x.byte[0] = tmp; local_2_global[i] = x.word; } - }; template -struct fill_map -{ +struct fill_map { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View + local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; global_id_view global_2_local; local_id_view local_2_global; - fill_map( global_id_view gIds, local_id_view lIds) - : global_2_local(gIds) , local_2_global(lIds) - { + fill_map(global_id_view gIds, local_id_view lIds) + : global_2_local(gIds), local_2_global(lIds) { Kokkos::parallel_for(local_2_global.extent(0), *this); } KOKKOS_INLINE_FUNCTION - void operator()(size_type i) const - { - global_2_local.insert( local_2_global[i], i); + void operator()(size_type i) const { + global_2_local.insert(local_2_global[i], i); } - }; template -struct find_test -{ +struct find_test { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View + local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; global_id_view global_2_local; local_id_view local_2_global; typedef size_t value_type; - find_test( global_id_view gIds, local_id_view lIds, value_type & num_errors) - : global_2_local(gIds) , local_2_global(lIds) - { + find_test(global_id_view gIds, local_id_view lIds, value_type& num_errors) + : global_2_local(gIds), local_2_global(lIds) { Kokkos::parallel_reduce(local_2_global.extent(0), *this, num_errors); } KOKKOS_INLINE_FUNCTION - void init(value_type & v) const - { v = 0; } + void init(value_type& v) const { v = 0; } KOKKOS_INLINE_FUNCTION - void join(volatile value_type & dst, volatile value_type const & src) const - { dst += src; } - - KOKKOS_INLINE_FUNCTION - void operator()(size_type i, value_type & num_errors) const - { - uint32_t index = global_2_local.find( local_2_global[i] ); - - if ( global_2_local.value_at(index) != i) ++num_errors; + void join(volatile value_type& dst, volatile value_type const& src) const { + dst += src; } + KOKKOS_INLINE_FUNCTION + void operator()(size_type i, value_type& num_errors) const { + uint32_t index = global_2_local.find(local_2_global[i]); + + if (global_2_local.value_at(index) != i) ++num_errors; + } }; template -void test_global_to_local_ids(unsigned num_ids) -{ - +void test_global_to_local_ids(unsigned num_ids) { typedef Device execution_space; typedef typename execution_space::size_type size_type; - typedef Kokkos::View local_id_view; - typedef Kokkos::UnorderedMap global_id_view; + typedef Kokkos::View local_id_view; + typedef Kokkos::UnorderedMap + global_id_view; - //size + // size std::cout << num_ids << ", "; double elasped_time = 0; Kokkos::Timer timer; local_id_view local_2_global("local_ids", num_ids); - global_id_view global_2_local((3u*num_ids)/2u); + global_id_view global_2_local((3u * num_ids) / 2u); - //create + // create elasped_time = timer.seconds(); std::cout << elasped_time << ", "; timer.reset(); // generate unique ids - { - generate_ids gen(local_2_global); - } + { generate_ids gen(local_2_global); } Device().fence(); // generate elasped_time = timer.seconds(); std::cout << elasped_time << ", "; timer.reset(); - { - fill_map fill(global_2_local, local_2_global); - } + { fill_map fill(global_2_local, local_2_global); } Device().fence(); // fill @@ -208,11 +193,9 @@ void test_global_to_local_ids(unsigned num_ids) std::cout << elasped_time << ", "; timer.reset(); - size_t num_errors = 0; - for (int i=0; i<100; ++i) - { - find_test find(global_2_local, local_2_global,num_errors); + for (int i = 0; i < 100; ++i) { + find_test find(global_2_local, local_2_global, num_errors); } Device().fence(); @@ -220,12 +203,9 @@ void test_global_to_local_ids(unsigned num_ids) elasped_time = timer.seconds(); std::cout << elasped_time << std::endl; - ASSERT_EQ( num_errors, 0u); + ASSERT_EQ(num_errors, 0u); } +} // namespace Performance -} // namespace Performance - - -#endif //KOKKOS_TEST_GLOBAL_TO_LOCAL_IDS_HPP - +#endif // KOKKOS_TEST_GLOBAL_TO_LOCAL_IDS_HPP diff --git a/lib/kokkos/containers/performance_tests/TestHPX.cpp b/lib/kokkos/containers/performance_tests/TestHPX.cpp index 0f43377cee..48be466bfa 100644 --- a/lib/kokkos/containers/performance_tests/TestHPX.cpp +++ b/lib/kokkos/containers/performance_tests/TestHPX.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_HPX ) +#if defined(KOKKOS_ENABLE_HPX) #include @@ -61,70 +62,63 @@ #include #include - namespace Performance { class hpx : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::initialize(); - Kokkos::print_configuration( std::cout ); + Kokkos::print_configuration(std::cout); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( hpx, dynrankview_perf ) -{ +TEST_F(hpx, dynrankview_perf) { std::cout << "HPX" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( hpx, global_2_local) -{ +TEST_F(hpx, global_2_local) { std::cout << "HPX" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( hpx, unordered_map_performance_near) -{ +TEST_F(hpx, unordered_map_performance_near) { unsigned num_hpx = 4; std::ostringstream base_file_name; base_file_name << "hpx-" << num_hpx << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests( + base_file_name.str()); } -TEST_F( hpx, unordered_map_performance_far) -{ +TEST_F(hpx, unordered_map_performance_far) { unsigned num_hpx = 4; std::ostringstream base_file_name; base_file_name << "hpx-" << num_hpx << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests( + base_file_name.str()); } -TEST_F( hpx, scatter_view) -{ +TEST_F(hpx, scatter_view) { std::cout << "ScatterView data-duplicated test:\n"; Perf::test_scatter_view(10, 1000 * 1000); -//std::cout << "ScatterView atomics test:\n"; -//Perf::test_scatter_view(10, 1000 * 1000); + Kokkos::Experimental::ScatterDuplicated, + Kokkos::Experimental::ScatterNonAtomic>(10, + 1000 * 1000); + // std::cout << "ScatterView atomics test:\n"; + // Perf::test_scatter_view(10, 1000 * 1000); } -} // namespace test +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTHPX_PREVENT_EMPTY_LINK_ERROR() {} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestMain.cpp b/lib/kokkos/containers/performance_tests/TestMain.cpp index 217b01a57a..e3c8edb045 100644 --- a/lib/kokkos/containers/performance_tests/TestMain.cpp +++ b/lib/kokkos/containers/performance_tests/TestMain.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -47,7 +48,6 @@ #include int main(int argc, char *argv[]) { - ::testing::InitGoogleTest(&argc,argv); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - diff --git a/lib/kokkos/containers/performance_tests/TestOpenMP.cpp b/lib/kokkos/containers/performance_tests/TestOpenMP.cpp index e6218074ea..a9c8639ed4 100644 --- a/lib/kokkos/containers/performance_tests/TestOpenMP.cpp +++ b/lib/kokkos/containers/performance_tests/TestOpenMP.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_OPENMP ) +#if defined(KOKKOS_ENABLE_OPENMP) #include @@ -61,82 +62,72 @@ #include #include - namespace Performance { class openmp : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::initialize(); - Kokkos::OpenMP::print_configuration( std::cout ); + Kokkos::OpenMP::print_configuration(std::cout); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( openmp, dynrankview_perf ) -{ +TEST_F(openmp, dynrankview_perf) { std::cout << "OpenMP" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( openmp, global_2_local) -{ +TEST_F(openmp, global_2_local) { std::cout << "OpenMP" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( openmp, unordered_map_performance_near) -{ +TEST_F(openmp, unordered_map_performance_near) { unsigned num_openmp = 4; if (Kokkos::hwloc::available()) { num_openmp = Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core(); - + Kokkos::hwloc::get_available_cores_per_numa() * + Kokkos::hwloc::get_available_threads_per_core(); } std::ostringstream base_file_name; base_file_name << "openmp-" << num_openmp << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( openmp, unordered_map_performance_far) -{ +TEST_F(openmp, unordered_map_performance_far) { unsigned num_openmp = 4; if (Kokkos::hwloc::available()) { num_openmp = Kokkos::hwloc::get_available_numa_count() * - Kokkos::hwloc::get_available_cores_per_numa() * - Kokkos::hwloc::get_available_threads_per_core(); - + Kokkos::hwloc::get_available_cores_per_numa() * + Kokkos::hwloc::get_available_threads_per_core(); } std::ostringstream base_file_name; base_file_name << "openmp-" << num_openmp << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( openmp, scatter_view) -{ +TEST_F(openmp, scatter_view) { std::cout << "ScatterView data-duplicated test:\n"; Perf::test_scatter_view(10, 1000 * 1000); -//std::cout << "ScatterView atomics test:\n"; -//Perf::test_scatter_view(10, 1000 * 1000); + Kokkos::Experimental::ScatterDuplicated, + Kokkos::Experimental::ScatterNonAtomic>(10, + 1000 * 1000); + // std::cout << "ScatterView atomics test:\n"; + // Perf::test_scatter_view(10, 1000 * 1000); } -} // namespace test +} // namespace Performance #else -void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTOPENMP_PREVENT_EMPTY_LINK_ERROR() {} +void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTOPENMP_PREVENT_EMPTY_LINK_ERROR() { +} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestROCm.cpp b/lib/kokkos/containers/performance_tests/TestROCm.cpp index 3cf9f3bd14..55b770b49c 100644 --- a/lib/kokkos/containers/performance_tests/TestROCm.cpp +++ b/lib/kokkos/containers/performance_tests/TestROCm.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_ROCM ) +#if defined(KOKKOS_ENABLE_ROCM) #include #include @@ -66,15 +67,14 @@ namespace Performance { class rocm : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; Kokkos::HostSpace::execution_space::initialize(); - Kokkos::Experimental::ROCm::initialize( Kokkos::Experimental::ROCm::SelectDevice(0) ); + Kokkos::Experimental::ROCm::initialize( + Kokkos::Experimental::ROCm::SelectDevice(0)); } - static void TearDownTestCase() - { + static void TearDownTestCase() { Kokkos::Experimental::ROCm::finalize(); Kokkos::HostSpace::execution_space::finalize(); } @@ -97,17 +97,15 @@ TEST_F( rocm, global_2_local) } #endif -TEST_F( rocm, unordered_map_performance_near) -{ - Perf::run_performance_tests("rocm-near"); +TEST_F(rocm, unordered_map_performance_near) { + Perf::run_performance_tests("rocm-near"); } -TEST_F( rocm, unordered_map_performance_far) -{ - Perf::run_performance_tests("rocm-far"); +TEST_F(rocm, unordered_map_performance_far) { + Perf::run_performance_tests("rocm-far"); } -} +} // namespace Performance #else void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTROCM_PREVENT_EMPTY_LINK_ERROR() {} -#endif /* #if defined( KOKKOS_ENABLE_ROCM ) */ +#endif /* #if defined( KOKKOS_ENABLE_ROCM ) */ diff --git a/lib/kokkos/containers/performance_tests/TestScatterView.hpp b/lib/kokkos/containers/performance_tests/TestScatterView.hpp index bd9121bb82..3d4c57f3e2 100644 --- a/lib/kokkos/containers/performance_tests/TestScatterView.hpp +++ b/lib/kokkos/containers/performance_tests/TestScatterView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -49,67 +50,68 @@ namespace Perf { -template -void test_scatter_view(int m, int n) -{ - Kokkos::View original_view("original_view", n); +template +void test_scatter_view(int m, int n) { + Kokkos::View original_view("original_view", + n); { - auto scatter_view = Kokkos::Experimental::create_scatter_view - < Kokkos::Experimental::ScatterSum - , duplication - , contribution - > (original_view); + auto scatter_view = Kokkos::Experimental::create_scatter_view< + Kokkos::Experimental::ScatterSum, duplication, contribution>( + original_view); Kokkos::Experimental::UniqueToken< - ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> - unique_token{ExecSpace()}; - //auto internal_view = scatter_view.internal_view; + ExecSpace, Kokkos::Experimental::UniqueTokenScope::Global> + unique_token{ExecSpace()}; + // auto internal_view = scatter_view.internal_view; auto policy = Kokkos::RangePolicy(0, n); for (int foo = 0; foo < 5; ++foo) { - { - auto num_threads = unique_token.size(); - std::cout << "num_threads " << num_threads << '\n'; - Kokkos::View hand_coded_duplicate_view("hand_coded_duplicate", num_threads, n); - auto f2 = KOKKOS_LAMBDA(int i) { - auto thread_id = unique_token.acquire(); - for (int j = 0; j < 10; ++j) { - auto k = (i + j) % n; - hand_coded_duplicate_view(thread_id, k, 0) += 4.2; - hand_coded_duplicate_view(thread_id, k, 1) += 2.0; - hand_coded_duplicate_view(thread_id, k, 2) += 1.0; + { + auto num_threads = unique_token.size(); + std::cout << "num_threads " << num_threads << '\n'; + Kokkos::View + hand_coded_duplicate_view("hand_coded_duplicate", num_threads, n); + auto f2 = KOKKOS_LAMBDA(int i) { + auto thread_id = unique_token.acquire(); + for (int j = 0; j < 10; ++j) { + auto k = (i + j) % n; + hand_coded_duplicate_view(thread_id, k, 0) += 4.2; + hand_coded_duplicate_view(thread_id, k, 1) += 2.0; + hand_coded_duplicate_view(thread_id, k, 2) += 1.0; + } + }; + Kokkos::Timer timer; + timer.reset(); + for (int k = 0; k < m; ++k) { + Kokkos::parallel_for(policy, f2, + "hand_coded_duplicate_scatter_view_test"); } - }; - Kokkos::Timer timer; - timer.reset(); - for (int k = 0; k < m; ++k) { - Kokkos::parallel_for(policy, f2, "hand_coded_duplicate_scatter_view_test"); + Kokkos::fence(); + auto t = timer.seconds(); + std::cout << "hand-coded test took " << t << " seconds\n"; } - Kokkos::fence(); - auto t = timer.seconds(); - std::cout << "hand-coded test took " << t << " seconds\n"; - } - { - auto f = KOKKOS_LAMBDA(int i) { - auto scatter_access = scatter_view.access(); - for (int j = 0; j < 10; ++j) { - auto k = (i + j) % n; - scatter_access(k, 0) += 4.2; - scatter_access(k, 1) += 2.0; - scatter_access(k, 2) += 1.0; + { + auto f = KOKKOS_LAMBDA(int i) { + auto scatter_access = scatter_view.access(); + for (int j = 0; j < 10; ++j) { + auto k = (i + j) % n; + scatter_access(k, 0) += 4.2; + scatter_access(k, 1) += 2.0; + scatter_access(k, 2) += 1.0; + } + }; + Kokkos::Timer timer; + timer.reset(); + for (int k = 0; k < m; ++k) { + Kokkos::parallel_for(policy, f, "scatter_view_test"); } - }; - Kokkos::Timer timer; - timer.reset(); - for (int k = 0; k < m; ++k) { - Kokkos::parallel_for(policy, f, "scatter_view_test"); + Kokkos::fence(); + auto t = timer.seconds(); + std::cout << "test took " << t << " seconds\n"; } - Kokkos::fence(); - auto t = timer.seconds(); - std::cout << "test took " << t << " seconds\n"; } } - } } -} +} // namespace Perf #endif diff --git a/lib/kokkos/containers/performance_tests/TestThreads.cpp b/lib/kokkos/containers/performance_tests/TestThreads.cpp index 6a02e67b25..2f37404539 100644 --- a/lib/kokkos/containers/performance_tests/TestThreads.cpp +++ b/lib/kokkos/containers/performance_tests/TestThreads.cpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -42,7 +43,7 @@ */ #include -#if defined( KOKKOS_ENABLE_THREADS ) +#if defined(KOKKOS_ENABLE_THREADS) #include @@ -65,9 +66,8 @@ namespace Performance { class threads : public ::testing::Test { -protected: - static void SetUpTestCase() - { + protected: + static void SetUpTestCase() { std::cout << std::setprecision(5) << std::scientific; unsigned num_threads = 4; @@ -76,66 +76,57 @@ protected: num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::cout << "Threads: " << num_threads << std::endl; - Kokkos::initialize( Kokkos::InitArguments(num_threads) ); + Kokkos::initialize(Kokkos::InitArguments(num_threads)); } - static void TearDownTestCase() - { - Kokkos::finalize(); - } + static void TearDownTestCase() { Kokkos::finalize(); } }; -TEST_F( threads, dynrankview_perf ) -{ +TEST_F(threads, dynrankview_perf) { std::cout << "Threads" << std::endl; std::cout << " DynRankView vs View: Initialization Only " << std::endl; - test_dynrankview_op_perf( 8192 ); + test_dynrankview_op_perf(8192); } -TEST_F( threads, global_2_local) -{ +TEST_F(threads, global_2_local) { std::cout << "Threads" << std::endl; std::cout << "size, create, generate, fill, find" << std::endl; - for (unsigned i=Performance::begin_id_size; i<=Performance::end_id_size; i *= Performance::id_step) + for (unsigned i = Performance::begin_id_size; i <= Performance::end_id_size; + i *= Performance::id_step) test_global_to_local_ids(i); } -TEST_F( threads, unordered_map_performance_near) -{ +TEST_F(threads, unordered_map_performance_near) { unsigned num_threads = 4; if (Kokkos::hwloc::available()) { num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::ostringstream base_file_name; base_file_name << "threads-" << num_threads << "-near"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -TEST_F( threads, unordered_map_performance_far) -{ +TEST_F(threads, unordered_map_performance_far) { unsigned num_threads = 4; if (Kokkos::hwloc::available()) { num_threads = Kokkos::hwloc::get_available_numa_count() * Kokkos::hwloc::get_available_cores_per_numa() * Kokkos::hwloc::get_available_threads_per_core(); - } std::ostringstream base_file_name; base_file_name << "threads-" << num_threads << "-far"; - Perf::run_performance_tests(base_file_name.str()); + Perf::run_performance_tests(base_file_name.str()); } -} // namespace Performance +} // namespace Performance #else -void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTTHREADS_PREVENT_EMPTY_LINK_ERROR() {} +void KOKKOS_CONTAINERS_PERFORMANCE_TESTS_TESTTHREADS_PREVENT_EMPTY_LINK_ERROR() { +} #endif - diff --git a/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp b/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp index 8d09281ed3..9057842340 100644 --- a/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp +++ b/lib/kokkos/containers/performance_tests/TestUnorderedMapPerformance.hpp @@ -1,10 +1,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -22,10 +23,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -50,12 +51,10 @@ #include #include - namespace Perf { template -struct UnorderedMapTest -{ +struct UnorderedMapTest { typedef Device execution_space; typedef Kokkos::UnorderedMap map_type; typedef typename map_type::histogram_type histogram_type; @@ -68,22 +67,22 @@ struct UnorderedMapTest uint32_t capacity; uint32_t inserts; uint32_t collisions; - double seconds; + double seconds; map_type map; histogram_type histogram; - UnorderedMapTest( uint32_t arg_capacity, uint32_t arg_inserts, uint32_t arg_collisions) - : capacity(arg_capacity) - , inserts(arg_inserts) - , collisions(arg_collisions) - , seconds(0) - , map(capacity) - , histogram(map.get_histogram()) - { - Kokkos::Timer wall_clock ; + UnorderedMapTest(uint32_t arg_capacity, uint32_t arg_inserts, + uint32_t arg_collisions) + : capacity(arg_capacity), + inserts(arg_inserts), + collisions(arg_collisions), + seconds(0), + map(capacity), + histogram(map.get_histogram()) { + Kokkos::Timer wall_clock; wall_clock.reset(); - value_type v = {}; + value_type v = {}; int loop_count = 0; do { ++loop_count; @@ -92,81 +91,79 @@ struct UnorderedMapTest Kokkos::parallel_reduce(inserts, *this, v); if (v.failed_count > 0u) { - const uint32_t new_capacity = map.capacity() + ((map.capacity()*3ull)/20u) + v.failed_count/collisions ; - map.rehash( new_capacity ); + const uint32_t new_capacity = map.capacity() + + ((map.capacity() * 3ull) / 20u) + + v.failed_count / collisions; + map.rehash(new_capacity); } } while (v.failed_count > 0u); seconds = wall_clock.seconds(); - switch (loop_count) - { - case 1u: std::cout << " \033[0;32m" << loop_count << "\033[0m "; break; - case 2u: std::cout << " \033[1;31m" << loop_count << "\033[0m "; break; - default: std::cout << " \033[0;31m" << loop_count << "\033[0m "; break; + switch (loop_count) { + case 1u: std::cout << " \033[0;32m" << loop_count << "\033[0m "; break; + case 2u: std::cout << " \033[1;31m" << loop_count << "\033[0m "; break; + default: std::cout << " \033[0;31m" << loop_count << "\033[0m "; break; } - std::cout << std::setprecision(2) << std::fixed << std::setw(5) << (1e9*(seconds/(inserts))) << "; " << std::flush; + std::cout << std::setprecision(2) << std::fixed << std::setw(5) + << (1e9 * (seconds / (inserts))) << "; " << std::flush; histogram.calculate(); Device().fence(); } - void print(std::ostream & metrics_out, std::ostream & length_out, std::ostream & distance_out, std::ostream & block_distance_out) - { + void print(std::ostream& metrics_out, std::ostream& length_out, + std::ostream& distance_out, std::ostream& block_distance_out) { metrics_out << map.capacity() << " , "; - metrics_out << inserts/collisions << " , "; - metrics_out << (100.0 * inserts/collisions) / map.capacity() << " , "; + metrics_out << inserts / collisions << " , "; + metrics_out << (100.0 * inserts / collisions) / map.capacity() << " , "; metrics_out << inserts << " , "; metrics_out << (map.failed_insert() ? "true" : "false") << " , "; metrics_out << collisions << " , "; - metrics_out << 1e9*(seconds/inserts) << " , "; + metrics_out << 1e9 * (seconds / inserts) << " , "; metrics_out << seconds << std::endl; length_out << map.capacity() << " , "; - length_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + length_out << ((100.0 * inserts / collisions) / map.capacity()) << " , "; length_out << collisions << " , "; histogram.print_length(length_out); distance_out << map.capacity() << " , "; - distance_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + distance_out << ((100.0 * inserts / collisions) / map.capacity()) << " , "; distance_out << collisions << " , "; histogram.print_distance(distance_out); block_distance_out << map.capacity() << " , "; - block_distance_out << ((100.0 *inserts/collisions) / map.capacity()) << " , "; + block_distance_out << ((100.0 * inserts / collisions) / map.capacity()) + << " , "; block_distance_out << collisions << " , "; histogram.print_block_distance(block_distance_out); } - KOKKOS_INLINE_FUNCTION - void init( value_type & v ) const - { + void init(value_type& v) const { v.failed_count = 0; - v.max_list = 0; + v.max_list = 0; } KOKKOS_INLINE_FUNCTION - void join( volatile value_type & dst, const volatile value_type & src ) const - { + void join(volatile value_type& dst, const volatile value_type& src) const { dst.failed_count += src.failed_count; dst.max_list = src.max_list < dst.max_list ? dst.max_list : src.max_list; } KOKKOS_INLINE_FUNCTION - void operator()(uint32_t i, value_type & v) const - { - const uint32_t key = Near ? i/collisions : i%(inserts/collisions); - typename map_type::insert_result result = map.insert(key,i); + void operator()(uint32_t i, value_type& v) const { + const uint32_t key = Near ? i / collisions : i % (inserts / collisions); + typename map_type::insert_result result = map.insert(key, i); v.failed_count += !result.failed() ? 0 : 1; - v.max_list = result.list_position() < v.max_list ? v.max_list : result.list_position(); + v.max_list = result.list_position() < v.max_list ? v.max_list + : result.list_position(); } - }; template -void run_performance_tests(std::string const & base_file_name) -{ +void run_performance_tests(std::string const& base_file_name) { #if 0 std::string metrics_file_name = base_file_name + std::string("-metrics.csv"); std::string length_file_name = base_file_name + std::string("-length.csv"); @@ -254,7 +251,6 @@ void run_performance_tests(std::string const & base_file_name) #endif } +} // namespace Perf -} // namespace Perf - -#endif //KOKKOS_TEST_UNORDERED_MAP_PERFORMANCE_HPP +#endif // KOKKOS_TEST_UNORDERED_MAP_PERFORMANCE_HPP diff --git a/lib/kokkos/containers/src/CMakeLists.txt b/lib/kokkos/containers/src/CMakeLists.txt index e68fcad5e9..0c9d24d641 100644 --- a/lib/kokkos/containers/src/CMakeLists.txt +++ b/lib/kokkos/containers/src/CMakeLists.txt @@ -1,47 +1,34 @@ - -TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) - -#----------------------------------------------------------------------------- - -SET(TRILINOS_INCDIR ${CMAKE_INSTALL_PREFIX}/${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}) - -if(KOKKOS_LEGACY_TRIBITS) - - SET(HEADERS "") - SET(SOURCES "") - - SET(HEADERS_IMPL "") - - FILE(GLOB HEADERS *.hpp) - FILE(GLOB HEADERS_IMPL impl/*.hpp) - FILE(GLOB SOURCES impl/*.cpp) - - INSTALL(FILES ${HEADERS_IMPL} DESTINATION ${TRILINOS_INCDIR}/impl/) - - TRIBITS_ADD_LIBRARY( - kokkoscontainers - HEADERS ${HEADERS} - NOINSTALLHEADERS ${HEADERS_IMPL} - SOURCES ${SOURCES} - DEPLIBS - ) - -else() - - INSTALL ( - DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" - DESTINATION ${TRILINOS_INCDIR} - FILES_MATCHING PATTERN "*.hpp" - ) - - TRIBITS_ADD_LIBRARY( - kokkoscontainers - SOURCES ${KOKKOS_CONTAINERS_SRCS} - DEPLIBS - ) - -endif() -#----------------------------------------------------------------------------- + +KOKKOS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) + +#need these here for now +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) +KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +#----------------------------------------------------------------------------- + +SET(KOKKOS_CONTAINERS_SRCS) +APPEND_GLOB(KOKKOS_CONTAINERS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) + +INSTALL ( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" + DESTINATION ${KOKKOS_HEADER_DIR} + FILES_MATCHING PATTERN "*.hpp" +) + +KOKKOS_ADD_LIBRARY( + kokkoscontainers + SOURCES ${KOKKOS_CONTAINERS_SRCS} +) + +SET_TARGET_PROPERTIES(kokkoscontainers PROPERTIES VERSION ${Kokkos_VERSION}) + +KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscontainers + ${KOKKOS_TOP_BUILD_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) +KOKKOS_LINK_INTERNAL_LIBRARY(kokkoscontainers kokkoscore) + +#----------------------------------------------------------------------------- + diff --git a/lib/kokkos/containers/src/Kokkos_Bitset.hpp b/lib/kokkos/containers/src/Kokkos_Bitset.hpp index 4d78430fc6..ab75fc1e1d 100644 --- a/lib/kokkos/containers/src/Kokkos_Bitset.hpp +++ b/lib/kokkos/containers/src/Kokkos_Bitset.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -53,27 +54,25 @@ namespace Kokkos { -template +template class Bitset; -template +template class ConstBitset; template -void deep_copy( Bitset & dst, Bitset const& src); +void deep_copy(Bitset& dst, Bitset const& src); template -void deep_copy( Bitset & dst, ConstBitset const& src); +void deep_copy(Bitset& dst, ConstBitset const& src); template -void deep_copy( ConstBitset & dst, ConstBitset const& src); - +void deep_copy(ConstBitset& dst, ConstBitset const& src); /// A thread safe view to a bitset template -class Bitset -{ -public: +class Bitset { + public: typedef Device execution_space; typedef unsigned size_type; @@ -81,98 +80,88 @@ public: enum { MOVE_HINT_BACKWARD = 2u }; enum { - BIT_SCAN_FORWARD_MOVE_HINT_FORWARD = 0u - , BIT_SCAN_REVERSE_MOVE_HINT_FORWARD = BIT_SCAN_REVERSE - , BIT_SCAN_FORWARD_MOVE_HINT_BACKWARD = MOVE_HINT_BACKWARD - , BIT_SCAN_REVERSE_MOVE_HINT_BACKWARD = BIT_SCAN_REVERSE | MOVE_HINT_BACKWARD + BIT_SCAN_FORWARD_MOVE_HINT_FORWARD = 0u, + BIT_SCAN_REVERSE_MOVE_HINT_FORWARD = BIT_SCAN_REVERSE, + BIT_SCAN_FORWARD_MOVE_HINT_BACKWARD = MOVE_HINT_BACKWARD, + BIT_SCAN_REVERSE_MOVE_HINT_BACKWARD = BIT_SCAN_REVERSE | MOVE_HINT_BACKWARD }; -private: - enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; - enum { block_mask = block_size-1u }; + private: + enum { block_size = static_cast(sizeof(unsigned) * CHAR_BIT) }; + enum { block_mask = block_size - 1u }; enum { block_shift = Kokkos::Impl::integral_power_of_two(block_size) }; -public: - - + public: /// constructor /// arg_size := number of bit in set Bitset(unsigned arg_size = 0u) - : m_size(arg_size) - , m_last_block_mask(0u) - , m_blocks("Bitset", ((m_size + block_mask) >> block_shift) ) - { - for (int i=0, end = static_cast(m_size & block_mask); i < end; ++i) { + : m_size(arg_size), + m_last_block_mask(0u), + m_blocks("Bitset", ((m_size + block_mask) >> block_shift)) { + for (int i = 0, end = static_cast(m_size & block_mask); i < end; ++i) { m_last_block_mask |= 1u << i; } } - KOKKOS_INLINE_FUNCTION - Bitset (const Bitset&) = default; + KOKKOS_DEFAULTED_FUNCTION + Bitset(const Bitset&) = default; - KOKKOS_INLINE_FUNCTION - Bitset& operator= (const Bitset&) = default; + KOKKOS_DEFAULTED_FUNCTION + Bitset& operator=(const Bitset&) = default; - KOKKOS_INLINE_FUNCTION - Bitset (Bitset&&) = default; + KOKKOS_DEFAULTED_FUNCTION + Bitset(Bitset&&) = default; - KOKKOS_INLINE_FUNCTION - Bitset& operator= (Bitset&&) = default; - - KOKKOS_INLINE_FUNCTION - ~Bitset () = default; + KOKKOS_DEFAULTED_FUNCTION + Bitset& operator=(Bitset&&) = default; + + KOKKOS_DEFAULTED_FUNCTION + ~Bitset() = default; /// number of bits in the set /// can be call from the host or the device KOKKOS_FORCEINLINE_FUNCTION - unsigned size() const - { return m_size; } + unsigned size() const { return m_size; } /// number of bits which are set to 1 /// can only be called from the host - unsigned count() const - { - Impl::BitsetCount< Bitset > f(*this); + unsigned count() const { + Impl::BitsetCount > f(*this); return f.apply(); } /// set all bits to 1 /// can only be called from the host - void set() - { - Kokkos::deep_copy(m_blocks, ~0u ); + void set() { + Kokkos::deep_copy(m_blocks, ~0u); if (m_last_block_mask) { - //clear the unused bits in the last block - typedef Kokkos::Impl::DeepCopy< typename execution_space::memory_space, Kokkos::HostSpace > raw_deep_copy; - raw_deep_copy( m_blocks.data() + (m_blocks.extent(0) -1u), &m_last_block_mask, sizeof(unsigned)); + // clear the unused bits in the last block + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(m_blocks.data() + (m_blocks.extent(0) - 1u), + &m_last_block_mask, sizeof(unsigned)); } } /// set all bits to 0 /// can only be called from the host - void reset() - { - Kokkos::deep_copy(m_blocks, 0u ); - } + void reset() { Kokkos::deep_copy(m_blocks, 0u); } /// set all bits to 0 /// can only be called from the host - void clear() - { - Kokkos::deep_copy(m_blocks, 0u ); - } + void clear() { Kokkos::deep_copy(m_blocks, 0u); } /// set i'th bit to 1 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool set( unsigned i ) const - { - if ( i < m_size ) { - unsigned * block_ptr = &m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool set(unsigned i) const { + if (i < m_size) { + unsigned* block_ptr = &m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); - return !( atomic_fetch_or( block_ptr, mask ) & mask ); + return !(atomic_fetch_or(block_ptr, mask) & mask); } return false; } @@ -180,13 +169,12 @@ public: /// set i'th bit to 0 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool reset( unsigned i ) const - { - if ( i < m_size ) { - unsigned * block_ptr = &m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool reset(unsigned i) const { + if (i < m_size) { + unsigned* block_ptr = &m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); - return atomic_fetch_and( block_ptr, ~mask ) & mask; + return atomic_fetch_and(block_ptr, ~mask) & mask; } return false; } @@ -194,11 +182,10 @@ public: /// return true if the i'th bit set to 1 /// can only be called from the device KOKKOS_FORCEINLINE_FUNCTION - bool test( unsigned i ) const - { - if ( i < m_size ) { - const unsigned block = volatile_load(&m_blocks[ i >> block_shift ]); - const unsigned mask = 1u << static_cast( i & block_mask ); + bool test(unsigned i) const { + if (i < m_size) { + const unsigned block = volatile_load(&m_blocks[i >> block_shift]); + const unsigned mask = 1u << static_cast(i & block_mask); return block & mask; } return false; @@ -208,90 +195,93 @@ public: /// returns the max number of times those functions should be call /// when searching for an available bit KOKKOS_FORCEINLINE_FUNCTION - unsigned max_hint() const - { - return m_blocks.extent(0); - } + unsigned max_hint() const { return m_blocks.extent(0); } /// find a bit set to 1 near the hint - /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found - /// and if result.first is false the result.second is a new hint + /// returns a pair< bool, unsigned> where if result.first is true then + /// result.second is the bit found and if result.first is false the + /// result.second is a new hint KOKKOS_INLINE_FUNCTION - Kokkos::pair find_any_set_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const - { - const unsigned block_idx = (hint >> block_shift) < m_blocks.extent(0) ? (hint >> block_shift) : 0; + Kokkos::pair find_any_set_near( + unsigned hint, + unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD) const { + const unsigned block_idx = + (hint >> block_shift) < m_blocks.extent(0) ? (hint >> block_shift) : 0; const unsigned offset = hint & block_mask; - unsigned block = volatile_load(&m_blocks[ block_idx ]); - block = !m_last_block_mask || (block_idx < (m_blocks.extent(0)-1)) ? block : block & m_last_block_mask ; + unsigned block = volatile_load(&m_blocks[block_idx]); + block = !m_last_block_mask || (block_idx < (m_blocks.extent(0) - 1)) + ? block + : block & m_last_block_mask; return find_any_helper(block_idx, offset, block, scan_direction); } /// find a bit set to 0 near the hint - /// returns a pair< bool, unsigned> where if result.first is true then result.second is the bit found - /// and if result.first is false the result.second is a new hint + /// returns a pair< bool, unsigned> where if result.first is true then + /// result.second is the bit found and if result.first is false the + /// result.second is a new hint KOKKOS_INLINE_FUNCTION - Kokkos::pair find_any_unset_near( unsigned hint , unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD ) const - { + Kokkos::pair find_any_unset_near( + unsigned hint, + unsigned scan_direction = BIT_SCAN_FORWARD_MOVE_HINT_FORWARD) const { const unsigned block_idx = hint >> block_shift; - const unsigned offset = hint & block_mask; - unsigned block = volatile_load(&m_blocks[ block_idx ]); - block = !m_last_block_mask || (block_idx < (m_blocks.extent(0)-1) ) ? ~block : ~block & m_last_block_mask ; + const unsigned offset = hint & block_mask; + unsigned block = volatile_load(&m_blocks[block_idx]); + block = !m_last_block_mask || (block_idx < (m_blocks.extent(0) - 1)) + ? ~block + : ~block & m_last_block_mask; return find_any_helper(block_idx, offset, block, scan_direction); } -private: - + private: KOKKOS_FORCEINLINE_FUNCTION - Kokkos::pair find_any_helper(unsigned block_idx, unsigned offset, unsigned block, unsigned scan_direction) const - { - Kokkos::pair result( block > 0u, 0); + Kokkos::pair find_any_helper(unsigned block_idx, + unsigned offset, unsigned block, + unsigned scan_direction) const { + Kokkos::pair result(block > 0u, 0); if (!result.first) { - result.second = update_hint( block_idx, offset, scan_direction ); - } - else { - result.second = scan_block( (block_idx << block_shift) - , offset - , block - , scan_direction - ); + result.second = update_hint(block_idx, offset, scan_direction); + } else { + result.second = + scan_block((block_idx << block_shift), offset, block, scan_direction); } return result; } - KOKKOS_FORCEINLINE_FUNCTION - unsigned scan_block(unsigned block_start, int offset, unsigned block, unsigned scan_direction ) const - { - offset = !(scan_direction & BIT_SCAN_REVERSE) ? offset : (offset + block_mask) & block_mask; + unsigned scan_block(unsigned block_start, int offset, unsigned block, + unsigned scan_direction) const { + offset = !(scan_direction & BIT_SCAN_REVERSE) + ? offset + : (offset + block_mask) & block_mask; block = Impl::rotate_right(block, offset); - return ((( !(scan_direction & BIT_SCAN_REVERSE) ? - Impl::bit_scan_forward(block) : - ::Kokkos::log2(block) - ) + offset - ) & block_mask - ) + block_start; + return (((!(scan_direction & BIT_SCAN_REVERSE) + ? Impl::bit_scan_forward(block) + : ::Kokkos::log2(block)) + + offset) & + block_mask) + + block_start; } KOKKOS_FORCEINLINE_FUNCTION - unsigned update_hint( long long block_idx, unsigned offset, unsigned scan_direction ) const - { + unsigned update_hint(long long block_idx, unsigned offset, + unsigned scan_direction) const { block_idx += scan_direction & MOVE_HINT_BACKWARD ? -1 : 1; block_idx = block_idx >= 0 ? block_idx : m_blocks.extent(0) - 1; - block_idx = block_idx < static_cast(m_blocks.extent(0)) ? block_idx : 0; + block_idx = + block_idx < static_cast(m_blocks.extent(0)) ? block_idx : 0; - return static_cast(block_idx)*block_size + offset; + return static_cast(block_idx) * block_size + offset; } -private: - + private: unsigned m_size; unsigned m_last_block_mask; - View< unsigned *, execution_space, MemoryTraits > m_blocks; + View > m_blocks; -private: + private: template friend class Bitset; @@ -302,87 +292,72 @@ private: friend struct Impl::BitsetCount; template - friend void deep_copy( Bitset & dst, Bitset const& src); + friend void deep_copy(Bitset& dst, Bitset const& src); template - friend void deep_copy( Bitset & dst, ConstBitset const& src); + friend void deep_copy(Bitset& dst, + ConstBitset const& src); }; /// a thread-safe view to a const bitset /// i.e. can only test bits template -class ConstBitset -{ -public: +class ConstBitset { + public: typedef Device execution_space; typedef unsigned size_type; -private: - enum { block_size = static_cast(sizeof(unsigned)*CHAR_BIT) }; - enum { block_mask = block_size -1u }; + private: + enum { block_size = static_cast(sizeof(unsigned) * CHAR_BIT) }; + enum { block_mask = block_size - 1u }; enum { block_shift = Kokkos::Impl::integral_power_of_two(block_size) }; -public: - ConstBitset() - : m_size (0) - {} + public: + ConstBitset() : m_size(0) {} ConstBitset(Bitset const& rhs) - : m_size(rhs.m_size) - , m_blocks(rhs.m_blocks) - {} + : m_size(rhs.m_size), m_blocks(rhs.m_blocks) {} ConstBitset(ConstBitset const& rhs) - : m_size( rhs.m_size ) - , m_blocks( rhs.m_blocks ) - {} + : m_size(rhs.m_size), m_blocks(rhs.m_blocks) {} - ConstBitset & operator = (Bitset const & rhs) - { - this->m_size = rhs.m_size; + ConstBitset& operator=(Bitset const& rhs) { + this->m_size = rhs.m_size; this->m_blocks = rhs.m_blocks; return *this; } - ConstBitset & operator = (ConstBitset const & rhs) - { - this->m_size = rhs.m_size; + ConstBitset& operator=(ConstBitset const& rhs) { + this->m_size = rhs.m_size; this->m_blocks = rhs.m_blocks; return *this; } - KOKKOS_FORCEINLINE_FUNCTION - unsigned size() const - { - return m_size; - } + unsigned size() const { return m_size; } - unsigned count() const - { - Impl::BitsetCount< ConstBitset > f(*this); + unsigned count() const { + Impl::BitsetCount > f(*this); return f.apply(); } KOKKOS_FORCEINLINE_FUNCTION - bool test( unsigned i ) const - { - if ( i < m_size ) { - const unsigned block = m_blocks[ i >> block_shift ]; - const unsigned mask = 1u << static_cast( i & block_mask ); + bool test(unsigned i) const { + if (i < m_size) { + const unsigned block = m_blocks[i >> block_shift]; + const unsigned mask = 1u << static_cast(i & block_mask); return block & mask; } return false; } -private: - + private: unsigned m_size; - View< const unsigned *, execution_space, MemoryTraits > m_blocks; + View > m_blocks; -private: + private: template friend class ConstBitset; @@ -390,47 +365,56 @@ private: friend struct Impl::BitsetCount; template - friend void deep_copy( Bitset & dst, ConstBitset const& src); + friend void deep_copy(Bitset& dst, + ConstBitset const& src); template - friend void deep_copy( ConstBitset & dst, ConstBitset const& src); + friend void deep_copy(ConstBitset& dst, + ConstBitset const& src); }; - template -void deep_copy( Bitset & dst, Bitset const& src) -{ +void deep_copy(Bitset& dst, Bitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } template -void deep_copy( Bitset & dst, ConstBitset const& src) -{ +void deep_copy(Bitset& dst, ConstBitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } template -void deep_copy( ConstBitset & dst, ConstBitset const& src) -{ +void deep_copy(ConstBitset& dst, ConstBitset const& src) { if (dst.size() != src.size()) { - throw std::runtime_error("Error: Cannot deep_copy bitsets of different sizes!"); + throw std::runtime_error( + "Error: Cannot deep_copy bitsets of different sizes!"); } - typedef Kokkos::Impl::DeepCopy< typename DstDevice::memory_space, typename SrcDevice::memory_space > raw_deep_copy; - raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), sizeof(unsigned)*src.m_blocks.extent(0)); + typedef Kokkos::Impl::DeepCopy + raw_deep_copy; + raw_deep_copy(dst.m_blocks.data(), src.m_blocks.data(), + sizeof(unsigned) * src.m_blocks.extent(0)); } -} // namespace Kokkos - -#endif //KOKKOS_BITSET_HPP +} // namespace Kokkos +#endif // KOKKOS_BITSET_HPP diff --git a/lib/kokkos/containers/src/Kokkos_DualView.hpp b/lib/kokkos/containers/src/Kokkos_DualView.hpp index d9b14d67a2..ede7d9a31f 100644 --- a/lib/kokkos/containers/src/Kokkos_DualView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DualView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -90,47 +91,41 @@ namespace Kokkos { * behavior. Please see the documentation of Kokkos::View for * examples. The default suffices for most users. */ -template< class DataType , - class Arg1Type = void , - class Arg2Type = void , +template -class DualView : public ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > -{ -template< class , class , class , class > friend class DualView ; -public: +class DualView : public ViewTraits { + template + friend class DualView; + + public: //! \name Typedefs for device types and various Kokkos::View specializations. //@{ - typedef ViewTraits< DataType , Arg1Type , Arg2Type, Arg3Type > traits ; + typedef ViewTraits traits; //! The Kokkos Host Device type; - typedef typename traits::host_mirror_space host_mirror_space ; + typedef typename traits::host_mirror_space host_mirror_space; //! The type of a Kokkos::View on the device. - typedef View< typename traits::data_type , - Arg1Type , - Arg2Type , - Arg3Type > t_dev ; + typedef View t_dev; /// \typedef t_host /// \brief The type of a Kokkos::View host mirror of \c t_dev. - typedef typename t_dev::HostMirror t_host ; + typedef typename t_dev::HostMirror t_host; //! The type of a const View on the device. //! The type of a Kokkos::View on the device. - typedef View< typename traits::const_data_type , - Arg1Type , - Arg2Type , - Arg3Type > t_dev_const ; + typedef View + t_dev_const; /// \typedef t_host_const /// \brief The type of a const View host mirror of \c t_dev_const. typedef typename t_dev_const::HostMirror t_host_const; //! The type of a const, random-access View on the device. - typedef View< typename traits::const_data_type , - typename traits::array_layout , - typename traits::device_type , - Kokkos::MemoryTraits > t_dev_const_randomread ; + typedef View > + t_dev_const_randomread; /// \typedef t_host_const_randomread /// \brief The type of a const, random-access View host mirror of @@ -138,39 +133,36 @@ public: typedef typename t_dev_const_randomread::HostMirror t_host_const_randomread; //! The type of an unmanaged View on the device. - typedef View< typename traits::data_type , - typename traits::array_layout , - typename traits::device_type , - MemoryUnmanaged> t_dev_um; + typedef View + t_dev_um; //! The type of an unmanaged View host mirror of \c t_dev_um. - typedef View< typename t_host::data_type , - typename t_host::array_layout , - typename t_host::device_type , - MemoryUnmanaged> t_host_um; + typedef View + t_host_um; //! The type of a const unmanaged View on the device. - typedef View< typename traits::const_data_type , - typename traits::array_layout , - typename traits::device_type , - MemoryUnmanaged> t_dev_const_um; + typedef View + t_dev_const_um; //! The type of a const unmanaged View host mirror of \c t_dev_const_um. - typedef View t_host_const_um; + typedef View + t_host_const_um; //! The type of a const, random-access View on the device. - typedef View< typename t_host::const_data_type , - typename t_host::array_layout , - typename t_host::device_type , - Kokkos::MemoryTraits > t_dev_const_randomread_um ; + typedef View > + t_dev_const_randomread_um; /// \typedef t_host_const_randomread /// \brief The type of a const, random-access View host mirror of /// \c t_dev_const_randomread. - typedef typename t_dev_const_randomread::HostMirror t_host_const_randomread_um; + typedef + typename t_dev_const_randomread::HostMirror t_host_const_randomread_um; //@} //! \name The two View instances. @@ -184,18 +176,20 @@ public: //@{ #ifndef KOKKOS_ENABLE_DEPRECATED_CODE -protected: + protected: // modified_flags[0] -> host // modified_flags[1] -> device - typedef View t_modified_flags; + typedef View t_modified_flags; t_modified_flags modified_flags; -public: + public: #else - typedef View t_modified_flags; - typedef View t_modified_flag; + typedef View + t_modified_flags; + typedef View + t_modified_flag; t_modified_flags modified_flags; - t_modified_flag modified_host,modified_device; + t_modified_flag modified_host, modified_device; #endif //@} @@ -208,11 +202,11 @@ public: /// default constructors. The "modified" flags are both initialized /// to "unmodified." #ifndef KOKKOS_ENABLE_DEPRECATED_CODE - DualView () = default; + DualView() = default; #else - DualView ():modified_flags (t_modified_flags("DualView::modified_flags")) { - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + DualView() : modified_flags(t_modified_flags("DualView::modified_flags")) { + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); } #endif @@ -225,52 +219,99 @@ public: /// View objects. For example, if the View has three dimensions, /// the first three integer arguments will be nonzero, and you may /// omit the integer arguments that follow. - DualView (const std::string& label, - const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : d_view (label, n0, n1, n2, n3, n4, n5, n6, n7) - , h_view (create_mirror_view (d_view)) // without UVM, host View mirrors - , modified_flags (t_modified_flags("DualView::modified_flags")) - { + DualView(const std::string& label, + const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : d_view(label, n0, n1, n2, n3, n4, n5, n6, n7), + h_view(create_mirror_view(d_view)) // without UVM, host View mirrors + , + modified_flags(t_modified_flags("DualView::modified_flags")) { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); #endif } - //! Copy constructor (shallow copy) - template - DualView (const DualView& src) : - d_view (src.d_view), - h_view (src.h_view), - modified_flags (src.modified_flags) + /// \brief Constructor that allocates View objects on both host and device. + /// + /// This constructor works like the analogous constructor of View. + /// The first arguments are wrapped up in a ViewCtor class, this allows + /// for a label, without initializing, and all of the other things that can + /// be wrapped up in a Ctor class. + /// The arguments that follow are the dimensions of the + /// View objects. For example, if the View has three dimensions, + /// the first three integer arguments will be nonzero, and you may + /// omit the integer arguments that follow. + template + DualView(const Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + size_t>::type const n0 = + KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : d_view(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7), + h_view(create_mirror_view(d_view)) // without UVM, host View mirrors + , + modified_flags(t_modified_flags("DualView::modified_flags")) { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - , modified_host(src.modified_host) - , modified_device(src.modified_device) + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); #endif - {} + } + + explicit inline DualView(const ViewAllocateWithoutInitializing& arg_prop, + const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : DualView(Impl::ViewCtorProp( + arg_prop.label, Kokkos::WithoutInitializing), + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, + arg_N7) {} + + //! Copy constructor (shallow copy) + template + DualView(const DualView& src) + : d_view(src.d_view), + h_view(src.h_view), + modified_flags(src.modified_flags) +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE + , + modified_host(src.modified_host), + modified_device(src.modified_device) +#endif + { + } //! Subview constructor - template< class SD, class S1 , class S2 , class S3 - , class Arg0 , class ... Args > - DualView( const DualView & src - , const Arg0 & arg0 - , Args ... args - ) - : d_view( Kokkos::subview( src.d_view , arg0 , args ... ) ) - , h_view( Kokkos::subview( src.h_view , arg0 , args ... ) ) - , modified_flags (src.modified_flags) + template + DualView(const DualView& src, const Arg0& arg0, Args... args) + : d_view(Kokkos::subview(src.d_view, arg0, args...)), + h_view(Kokkos::subview(src.h_view, arg0, args...)), + modified_flags(src.modified_flags) #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - , modified_host(src.modified_host) - , modified_device(src.modified_device) + , + modified_host(src.modified_host), + modified_device(src.modified_device) #endif - {} + { + } /// \brief Create DualView from existing device and host View objects. /// @@ -282,34 +323,34 @@ public: /// /// \param d_view_ Device View /// \param h_view_ Host View (must have type t_host = t_dev::HostMirror) - DualView (const t_dev& d_view_, const t_host& h_view_) : - d_view (d_view_), - h_view (h_view_), - modified_flags (t_modified_flags("DualView::modified_flags")) - { - if ( int(d_view.rank) != int(h_view.rank) || - d_view.extent(0) != h_view.extent(0) || - d_view.extent(1) != h_view.extent(1) || - d_view.extent(2) != h_view.extent(2) || - d_view.extent(3) != h_view.extent(3) || - d_view.extent(4) != h_view.extent(4) || - d_view.extent(5) != h_view.extent(5) || - d_view.extent(6) != h_view.extent(6) || - d_view.extent(7) != h_view.extent(7) || - d_view.stride_0() != h_view.stride_0() || - d_view.stride_1() != h_view.stride_1() || - d_view.stride_2() != h_view.stride_2() || - d_view.stride_3() != h_view.stride_3() || - d_view.stride_4() != h_view.stride_4() || - d_view.stride_5() != h_view.stride_5() || - d_view.stride_6() != h_view.stride_6() || - d_view.stride_7() != h_view.stride_7() || - d_view.span() != h_view.span() ) { - Kokkos::Impl::throw_runtime_exception("DualView constructed with incompatible views"); + DualView(const t_dev& d_view_, const t_host& h_view_) + : d_view(d_view_), + h_view(h_view_), + modified_flags(t_modified_flags("DualView::modified_flags")) { + if (int(d_view.rank) != int(h_view.rank) || + d_view.extent(0) != h_view.extent(0) || + d_view.extent(1) != h_view.extent(1) || + d_view.extent(2) != h_view.extent(2) || + d_view.extent(3) != h_view.extent(3) || + d_view.extent(4) != h_view.extent(4) || + d_view.extent(5) != h_view.extent(5) || + d_view.extent(6) != h_view.extent(6) || + d_view.extent(7) != h_view.extent(7) || + d_view.stride_0() != h_view.stride_0() || + d_view.stride_1() != h_view.stride_1() || + d_view.stride_2() != h_view.stride_2() || + d_view.stride_3() != h_view.stride_3() || + d_view.stride_4() != h_view.stride_4() || + d_view.stride_5() != h_view.stride_5() || + d_view.stride_6() != h_view.stride_6() || + d_view.stride_7() != h_view.stride_7() || + d_view.span() != h_view.span()) { + Kokkos::Impl::throw_runtime_exception( + "DualView constructed with incompatible views"); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - modified_host = t_modified_flag(modified_flags,0); - modified_device = t_modified_flag(modified_flags,1); + modified_host = t_modified_flag(modified_flags, 0); + modified_device = t_modified_flag(modified_flags, 1); #endif } @@ -326,119 +367,133 @@ public: /// /// For example, suppose you create a DualView on Cuda, like this: /// \code - /// typedef Kokkos::DualView dual_view_type; - /// dual_view_type DV ("my dual view", 100); - /// \endcode - /// If you want to get the CUDA device View, do this: - /// \code - /// typename dual_view_type::t_dev cudaView = DV.view (); - /// \endcode - /// and if you want to get the host mirror of that View, do this: - /// \code - /// typedef typename Kokkos::HostSpace::execution_space host_device_type; - /// typename dual_view_type::t_host hostView = DV.view (); - /// \endcode - template< class Device > - KOKKOS_INLINE_FUNCTION - const typename Impl::if_c< - std::is_same::value, - t_dev, - t_host>::type& view () const - { - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE - constexpr bool device_is_memspace = std::is_same::value; - constexpr bool device_is_execspace = std::is_same::value; - constexpr bool device_exec_is_t_dev_exec = std::is_same::value; - constexpr bool device_mem_is_t_dev_mem = std::is_same::value; - constexpr bool device_exec_is_t_host_exec = std::is_same::value; - constexpr bool device_mem_is_t_host_mem = std::is_same::value; - constexpr bool device_is_t_host_device = std::is_same::value; - constexpr bool device_is_t_dev_device = std::is_same::value; + /// typedef Kokkos::DualView + /// dual_view_type; dual_view_type DV ("my dual view", 100); \endcode If you + /// want to get the CUDA device View, do this: \code typename + /// dual_view_type::t_dev cudaView = DV.view (); \endcode and if + /// you want to get the host mirror of that View, do this: \code typedef + /// typename Kokkos::HostSpace::execution_space host_device_type; typename + /// dual_view_type::t_host hostView = DV.view (); \endcode + template + KOKKOS_INLINE_FUNCTION const typename Impl::if_c< + std::is_same::value, + t_dev, t_host>::type& + view() const { +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE + constexpr bool device_is_memspace = + std::is_same::value; + constexpr bool device_is_execspace = + std::is_same::value; + constexpr bool device_exec_is_t_dev_exec = + std::is_same::value; + constexpr bool device_mem_is_t_dev_mem = + std::is_same::value; + constexpr bool device_exec_is_t_host_exec = + std::is_same::value; + constexpr bool device_mem_is_t_host_mem = + std::is_same::value; + constexpr bool device_is_t_host_device = + std::is_same::value; + constexpr bool device_is_t_dev_device = + std::is_same::value; static_assert( device_is_t_dev_device || device_is_t_host_device || - (device_is_memspace && (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ) || - (device_is_execspace && (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) ) || - ( - (!device_is_execspace && !device_is_memspace) && ( - (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) - ) - ) - , - "Template parameter to .view() must exactly match one of the DualView's device types or one of the execution or memory spaces"); - #endif + (device_is_memspace && + (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) || + (device_is_execspace && + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) || + ((!device_is_execspace && !device_is_memspace) && + ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))), + "Template parameter to .view() must exactly match one of the " + "DualView's device types or one of the execution or memory spaces"); +#endif - return Impl::if_c< - std::is_same< - typename t_dev::memory_space, - typename Device::memory_space>::value, - t_dev, - t_host >::select (d_view , h_view); + return Impl::if_c::value, + t_dev, t_host>::select(d_view, h_view); } KOKKOS_INLINE_FUNCTION - t_host view_host() const { - return h_view; - } + t_host view_host() const { return h_view; } KOKKOS_INLINE_FUNCTION - t_dev view_device() const { - return d_view; - } + t_dev view_device() const { return d_view; } - template + template static int get_device_side() { - constexpr bool device_is_memspace = std::is_same::value; - constexpr bool device_is_execspace = std::is_same::value; - constexpr bool device_exec_is_t_dev_exec = std::is_same::value; - constexpr bool device_mem_is_t_dev_mem = std::is_same::value; - constexpr bool device_exec_is_t_host_exec = std::is_same::value; - constexpr bool device_mem_is_t_host_mem = std::is_same::value; - constexpr bool device_is_t_host_device = std::is_same::value; - constexpr bool device_is_t_dev_device = std::is_same::value; + constexpr bool device_is_memspace = + std::is_same::value; + constexpr bool device_is_execspace = + std::is_same::value; + constexpr bool device_exec_is_t_dev_exec = + std::is_same::value; + constexpr bool device_mem_is_t_dev_mem = + std::is_same::value; + constexpr bool device_exec_is_t_host_exec = + std::is_same::value; + constexpr bool device_mem_is_t_host_mem = + std::is_same::value; + constexpr bool device_is_t_host_device = + std::is_same::value; + constexpr bool device_is_t_dev_device = + std::is_same::value; - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE static_assert( device_is_t_dev_device || device_is_t_host_device || - (device_is_memspace && (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) ) || - (device_is_execspace && (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) ) || - ( - (!device_is_execspace && !device_is_memspace) && ( - (device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec) - ) - ) - , - "Template parameter to .sync() must exactly match one of the DualView's device types or one of the execution or memory spaces"); - #endif + (device_is_memspace && + (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) || + (device_is_execspace && + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) || + ((!device_is_execspace && !device_is_memspace) && + ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || + (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))), + "Template parameter to .sync() must exactly match one of the " + "DualView's device types or one of the execution or memory spaces"); +#endif - #ifndef KOKKOS_ENABLE_DEPRECATED_CODE +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE int dev = -1; - #else +#else int dev = 0; - #endif - if(device_is_t_dev_device) dev = 1; - else if(device_is_t_host_device) dev = 0; +#endif + if (device_is_t_dev_device) + dev = 1; + else if (device_is_t_host_device) + dev = 0; else { - if(device_is_memspace) { - if(device_mem_is_t_dev_mem) dev = 1; - if(device_mem_is_t_host_mem) dev = 0; - if(device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; + if (device_is_memspace) { + if (device_mem_is_t_dev_mem) dev = 1; + if (device_mem_is_t_host_mem) dev = 0; + if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; } - if(device_is_execspace) { - if(device_exec_is_t_dev_exec) dev = 1; - if(device_exec_is_t_host_exec) dev = 0; - if(device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; + if (device_is_execspace) { + if (device_exec_is_t_dev_exec) dev = 1; + if (device_exec_is_t_host_exec) dev = 0; + if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; } - if(!device_is_execspace && !device_is_memspace) { - if(device_mem_is_t_dev_mem) dev = 1; - if(device_mem_is_t_host_mem) dev = 0; - if(device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; - if(device_exec_is_t_dev_exec) dev = 1; - if(device_exec_is_t_host_exec) dev = 0; - if(device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; + if (!device_is_execspace && !device_is_memspace) { + if (device_mem_is_t_dev_mem) dev = 1; + if (device_mem_is_t_host_mem) dev = 0; + if (device_mem_is_t_host_mem && device_mem_is_t_dev_mem) dev = -1; + if (device_exec_is_t_dev_exec) dev = 1; + if (device_exec_is_t_host_exec) dev = 0; + if (device_exec_is_t_host_exec && device_exec_is_t_dev_exec) dev = -1; } } return dev; @@ -461,88 +516,134 @@ public: /// the data in either View. You must manually mark modified data /// as modified, by calling the modify() method with the /// appropriate template parameter. - template - void sync( const typename Impl::enable_if< - ( std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value) || - ( std::is_same< Device , int>::value) - , int >::type& = 0) - { - if(modified_flags.data()==NULL) return; + template + void sync(const typename std::enable_if< + (std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + if (modified_flags.data() == nullptr) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { - deep_copy (d_view, h_view); +#ifdef KOKKOS_ENABLE_CUDA + if (std::is_same::value) { + if (d_view.data() == h_view.data()) + Kokkos::Impl::cuda_prefetch_pointer( + Kokkos::Cuda(), d_view.data(), + sizeof(typename t_dev::value_type) * d_view.span(), true); + } +#endif + + deep_copy(d_view, h_view); modified_flags(0) = modified_flags(1) = 0; } } - if (dev == 0) { // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { - deep_copy (h_view, d_view); +#ifdef KOKKOS_ENABLE_CUDA + if (std::is_same::value) { + if (d_view.data() == h_view.data()) + Kokkos::Impl::cuda_prefetch_pointer( + Kokkos::Cuda(), d_view.data(), + sizeof(typename t_dev::value_type) * d_view.span(), false); + } +#endif + + deep_copy(h_view, d_view); modified_flags(0) = modified_flags(1) = 0; } } - if(std::is_same::value) { + if (std::is_same::value) { typename t_dev::execution_space().fence(); typename t_host::execution_space().fence(); } } - template - void sync ( const typename Impl::enable_if< - ( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) || - ( std::is_same< Device , int>::value) - , int >::type& = 0 ) - { - if(modified_flags.data()==NULL) return; + template + void sync(const typename std::enable_if< + (!std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + if (modified_flags.data() == nullptr) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { - Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + Impl::throw_runtime_exception( + "Calling sync on a DualView with a const datatype."); } } - if (dev == 0){ // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { - Impl::throw_runtime_exception("Calling sync on a DualView with a const datatype."); + Impl::throw_runtime_exception( + "Calling sync on a DualView with a const datatype."); } } } void sync_host() { - if( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) - Impl::throw_runtime_exception("Calling sync_host on a DualView with a const datatype."); - if(modified_flags.data()==NULL) return; - if(modified_flags(1) > modified_flags(0)) { - deep_copy (h_view, d_view); + if (!std::is_same::value) + Impl::throw_runtime_exception( + "Calling sync_host on a DualView with a const datatype."); + if (modified_flags.data() == nullptr) return; + if (modified_flags(1) > modified_flags(0)) { +#ifdef KOKKOS_ENABLE_CUDA + if (std::is_same::value) { + if (d_view.data() == h_view.data()) + Kokkos::Impl::cuda_prefetch_pointer( + Kokkos::Cuda(), d_view.data(), + sizeof(typename t_dev::value_type) * d_view.span(), false); + } +#endif + + deep_copy(h_view, d_view); modified_flags(1) = modified_flags(0) = 0; } } void sync_device() { - if( ! std::is_same< typename traits::data_type , typename traits::non_const_data_type>::value ) - Impl::throw_runtime_exception("Calling sync_device on a DualView with a const datatype."); - if(modified_flags.data()==NULL) return; - if(modified_flags(0) > modified_flags(1)) { - deep_copy (d_view, h_view); + if (!std::is_same::value) + Impl::throw_runtime_exception( + "Calling sync_device on a DualView with a const datatype."); + if (modified_flags.data() == nullptr) return; + if (modified_flags(0) > modified_flags(1)) { +#ifdef KOKKOS_ENABLE_CUDA + if (std::is_same::value) { + if (d_view.data() == h_view.data()) + Kokkos::Impl::cuda_prefetch_pointer( + Kokkos::Cuda(), d_view.data(), + sizeof(typename t_dev::value_type) * d_view.span(), true); + } +#endif + + deep_copy(d_view, h_view); modified_flags(1) = modified_flags(0) = 0; } } - template - bool need_sync() const - { - if(modified_flags.data()==NULL) return false; + template + bool need_sync() const { + if (modified_flags.data() == nullptr) return false; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type if ((modified_flags(0) > 0) && (modified_flags(0) >= modified_flags(1))) { return true; } } - if (dev == 0){ // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type if ((modified_flags(1) > 0) && (modified_flags(1) >= modified_flags(0))) { return true; } @@ -551,13 +652,13 @@ public: } inline bool need_sync_host() const { - if(modified_flags.data()==NULL) return false; - return modified_flags(0) - void modify () { - if(modified_flags.data()==NULL) return; + template + void modify() { + if (modified_flags.data() == nullptr) return; int dev = get_device_side(); - if (dev == 1) { // if Device is the same as DualView's device type + if (dev == 1) { // if Device is the same as DualView's device type // Increment the device's modified count. - modified_flags(1) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; + modified_flags(1) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; } - if (dev == 0) { // hopefully Device is the same as DualView's host type + if (dev == 0) { // hopefully Device is the same as DualView's host type // Increment the host's modified count. - modified_flags(0) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; + modified_flags(0) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; } #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK @@ -594,10 +699,12 @@ public: } inline void modify_host() { - if(modified_flags.data()!=NULL) { - modified_flags(0) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; - #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + if (modified_flags.data() != nullptr) { + modified_flags(0) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; +#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK if (modified_flags(0) && modified_flags(1)) { std::string msg = "Kokkos::DualView::modify_host ERROR: "; msg += "Concurrent modification of host and device views "; @@ -606,15 +713,17 @@ public: msg += "\"\n"; Kokkos::abort(msg.c_str()); } - #endif +#endif } } inline void modify_device() { - if(modified_flags.data()!=NULL) { - modified_flags(1) = (modified_flags(1) > modified_flags(0) ? - modified_flags(1) : modified_flags(0)) + 1; - #ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK + if (modified_flags.data() != nullptr) { + modified_flags(1) = + (modified_flags(1) > modified_flags(0) ? modified_flags(1) + : modified_flags(0)) + + 1; +#ifdef KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK if (modified_flags(0) && modified_flags(1)) { std::string msg = "Kokkos::DualView::modify_device ERROR: "; msg += "Concurrent modification of host and device views "; @@ -623,12 +732,12 @@ public: msg += "\"\n"; Kokkos::abort(msg.c_str()); } - #endif +#endif } } inline void clear_sync_state() { - if(modified_flags.data()!=NULL) + if (modified_flags.data() != nullptr) modified_flags(1) = modified_flags(0) = 0; } @@ -641,75 +750,72 @@ public: /// This discards any existing contents of the objects, and resets /// their modified flags. It does not copy the old contents /// of either View into the new View objects. - void realloc( const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG ) { - ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); - h_view = create_mirror_view( d_view ); + void realloc(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) { + ::Kokkos::realloc(d_view, n0, n1, n2, n3, n4, n5, n6, n7); + h_view = create_mirror_view(d_view); - /* Reset dirty flags */ - if(modified_flags.data()==NULL) { - modified_flags = t_modified_flags("DualView::modified_flags"); - } else - modified_flags(1) = modified_flags(0) = 0; + /* Reset dirty flags */ + if (modified_flags.data() == nullptr) { + modified_flags = t_modified_flags("DualView::modified_flags"); + } else + modified_flags(1) = modified_flags(0) = 0; } /// \brief Resize both views, copying old contents into new if necessary. /// /// This method only copies the old contents into the new View /// objects for the device which was last marked as modified. - void resize( const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG , - const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG ) { - if(modified_flags.data()==NULL) { - modified_flags = t_modified_flags("DualView::modified_flags"); - } - if(modified_flags(1) >= modified_flags(0)) { - /* Resize on Device */ - ::Kokkos::resize(d_view,n0,n1,n2,n3,n4,n5,n6,n7); - h_view = create_mirror_view( d_view ); + void resize(const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) { + if (modified_flags.data() == nullptr) { + modified_flags = t_modified_flags("DualView::modified_flags"); + } + if (modified_flags(1) >= modified_flags(0)) { + /* Resize on Device */ + ::Kokkos::resize(d_view, n0, n1, n2, n3, n4, n5, n6, n7); + h_view = create_mirror_view(d_view); - /* Mark Device copy as modified */ - modified_flags(1) = modified_flags(1)+1; + /* Mark Device copy as modified */ + modified_flags(1) = modified_flags(1) + 1; - } else { - /* Realloc on Device */ + } else { + /* Realloc on Device */ - ::Kokkos::realloc(d_view,n0,n1,n2,n3,n4,n5,n6,n7); + ::Kokkos::realloc(d_view, n0, n1, n2, n3, n4, n5, n6, n7); - const bool sizeMismatch = ( h_view.extent(0) != n0 ) || - ( h_view.extent(1) != n1 ) || - ( h_view.extent(2) != n2 ) || - ( h_view.extent(3) != n3 ) || - ( h_view.extent(4) != n4 ) || - ( h_view.extent(5) != n5 ) || - ( h_view.extent(6) != n6 ) || - ( h_view.extent(7) != n7 ); - if ( sizeMismatch ) - ::Kokkos::resize(h_view,n0,n1,n2,n3,n4,n5,n6,n7); + const bool sizeMismatch = + (h_view.extent(0) != n0) || (h_view.extent(1) != n1) || + (h_view.extent(2) != n2) || (h_view.extent(3) != n3) || + (h_view.extent(4) != n4) || (h_view.extent(5) != n5) || + (h_view.extent(6) != n6) || (h_view.extent(7) != n7); + if (sizeMismatch) + ::Kokkos::resize(h_view, n0, n1, n2, n3, n4, n5, n6, n7); - t_host temp_view = create_mirror_view( d_view ); + t_host temp_view = create_mirror_view(d_view); - /* Remap on Host */ - Kokkos::deep_copy( temp_view , h_view ); + /* Remap on Host */ + Kokkos::deep_copy(temp_view, h_view); - h_view = temp_view; + h_view = temp_view; - d_view = create_mirror_view( typename t_dev::execution_space(), h_view ); + d_view = create_mirror_view(typename t_dev::execution_space(), h_view); - /* Mark Host copy as modified */ - modified_flags(0) = modified_flags(0)+1; - } + /* Mark Host copy as modified */ + modified_flags(0) = modified_flags(0) + 1; + } } //@} @@ -718,37 +824,35 @@ public: #ifdef KOKKOS_ENABLE_DEPRECATED_CODE //! The allocation size (same as Kokkos::View::capacity). - size_t capacity() const { - return d_view.span(); - } + size_t capacity() const { return d_view.span(); } #endif //! The allocation size (same as Kokkos::View::span). - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { - return d_view.span(); - } + KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return d_view.span(); } - KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { - return d_view.span_is_contiguous(); + KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { + return d_view.span_is_contiguous(); } //! Get stride(s) for each dimension. - template< typename iType> + template void stride(iType* stride_) const { d_view.stride(stride_); } - template< typename iType > - KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - extent( const iType & r ) const - { return d_view.extent(r); } + template + KOKKOS_INLINE_FUNCTION constexpr + typename std::enable_if::value, size_t>::type + extent(const iType& r) const { + return d_view.extent(r); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , int >::type - extent_int( const iType & r ) const - { return static_cast(d_view.extent(r)); } + template + KOKKOS_INLINE_FUNCTION constexpr + typename std::enable_if::value, int>::type + extent_int(const iType& r) const { + return static_cast(d_view.extent(r)); + } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE /* Deprecate all 'dimension' functions in favor of @@ -756,27 +860,27 @@ public: */ /* \brief return size of dimension 0 */ - size_t dimension_0() const {return d_view.extent(0);} + size_t dimension_0() const { return d_view.extent(0); } /* \brief return size of dimension 1 */ - size_t dimension_1() const {return d_view.extent(1);} + size_t dimension_1() const { return d_view.extent(1); } /* \brief return size of dimension 2 */ - size_t dimension_2() const {return d_view.extent(2);} + size_t dimension_2() const { return d_view.extent(2); } /* \brief return size of dimension 3 */ - size_t dimension_3() const {return d_view.extent(3);} + size_t dimension_3() const { return d_view.extent(3); } /* \brief return size of dimension 4 */ - size_t dimension_4() const {return d_view.extent(4);} + size_t dimension_4() const { return d_view.extent(4); } /* \brief return size of dimension 5 */ - size_t dimension_5() const {return d_view.extent(5);} + size_t dimension_5() const { return d_view.extent(5); } /* \brief return size of dimension 6 */ - size_t dimension_6() const {return d_view.extent(6);} + size_t dimension_6() const { return d_view.extent(6); } /* \brief return size of dimension 7 */ - size_t dimension_7() const {return d_view.extent(7);} + size_t dimension_7() const { return d_view.extent(7); } #endif //@} }; -} // namespace Kokkos +} // namespace Kokkos //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- @@ -787,32 +891,24 @@ public: namespace Kokkos { namespace Impl { -template< class D, class A1, class A2, class A3, class ... Args > +template struct DualViewSubview { + typedef typename Kokkos::Impl::ViewMapping< + void, Kokkos::ViewTraits, Args...>::traits_type dst_traits; - typedef typename Kokkos::Impl::ViewMapping - < void - , Kokkos::ViewTraits< D, A1, A2, A3 > - , Args ... - >::traits_type dst_traits ; - - typedef Kokkos::DualView - < typename dst_traits::data_type - , typename dst_traits::array_layout - , typename dst_traits::device_type - , typename dst_traits::memory_traits - > type ; + typedef Kokkos::DualView< + typename dst_traits::data_type, typename dst_traits::array_layout, + typename dst_traits::device_type, typename dst_traits::memory_traits> + type; }; } /* namespace Impl */ - -template< class D , class A1 , class A2 , class A3 , class ... Args > -typename Impl::DualViewSubview::type -subview( const DualView & src , Args ... args ) -{ - return typename - Impl::DualViewSubview::type( src , args ... ); +template +typename Impl::DualViewSubview::type subview( + const DualView& src, Args... args) { + return typename Impl::DualViewSubview::type(src, + args...); } } /* namespace Kokkos */ @@ -826,40 +922,58 @@ namespace Kokkos { // Partial specialization of Kokkos::deep_copy() for DualView objects. // -template< class DT , class DL , class DD , class DM , - class ST , class SL , class SD , class SM > -void -deep_copy (DualView dst, // trust me, this must not be a reference - const DualView& src ) -{ - if ( src.need_sync_device() ) { - deep_copy (dst.h_view, src.h_view); - dst.modify_host(); - } - else { - deep_copy (dst.d_view, src.d_view); - dst.modify_device(); - } -} - -template< class ExecutionSpace , - class DT , class DL , class DD , class DM , - class ST , class SL , class SD , class SM > -void -deep_copy (const ExecutionSpace& exec , - DualView dst, // trust me, this must not be a reference - const DualView& src ) -{ - if ( src.need_sync_device() ) { - deep_copy (exec, dst.h_view, src.h_view); +template +void deep_copy( + DualView dst, // trust me, this must not be a reference + const DualView& src) { + if (src.need_sync_device()) { + deep_copy(dst.h_view, src.h_view); dst.modify_host(); } else { - deep_copy (exec, dst.d_view, src.d_view); + deep_copy(dst.d_view, src.d_view); dst.modify_device(); } } -} // namespace Kokkos +template +void deep_copy( + const ExecutionSpace& exec, + DualView dst, // trust me, this must not be a reference + const DualView& src) { + if (src.need_sync_device()) { + deep_copy(exec, dst.h_view, src.h_view); + dst.modify_host(); + } else { + deep_copy(exec, dst.d_view, src.d_view); + dst.modify_device(); + } +} + +} // namespace Kokkos + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +// +// Non-member resize and realloc +// + +template +void resize(DualView& dv, Args&&... args) noexcept( + noexcept(dv.resize(std::forward(args)...))) { + dv.resize(std::forward(args)...); +} + +template +void realloc(DualView& dv, Args&&... args) noexcept( + noexcept(dv.realloc(std::forward(args)...))) { + dv.realloc(std::forward(args)...); +} + +} // end namespace Kokkos #endif - diff --git a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp index d1e6704a57..4ab212d7b9 100644 --- a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp @@ -2,10 +2,11 @@ //@HEADER // ************************************************************************ // -// Kokkos v. 2.0 -// Copyright (2014) Sandia Corporation +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). // -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +// Under the terms of Contract DE-NA0003525 with NTESS, // the U.S. Government retains certain rights in this software. // // Redistribution and use in source and binary forms, with or without @@ -23,10 +24,10 @@ // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR @@ -56,342 +57,326 @@ namespace Kokkos { -template< typename DataType , class ... Properties > -class DynRankView; //forward declare +template +class DynRankView; // forward declare namespace Impl { template struct DynRankDimTraits { - - enum : size_t{unspecified = KOKKOS_INVALID_INDEX}; + enum : size_t { unspecified = KOKKOS_INVALID_INDEX }; // Compute the rank of the view from the nonzero dimension arguments. KOKKOS_INLINE_FUNCTION - static size_t computeRank( const size_t N0 - , const size_t N1 - , const size_t N2 - , const size_t N3 - , const size_t N4 - , const size_t N5 - , const size_t N6 - , const size_t /* N7 */) - { - return - ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified && N0 == unspecified) ? 0 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified && N1 == unspecified) ? 1 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified && N2 == unspecified) ? 2 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified && N3 == unspecified) ? 3 - : ( (N6 == unspecified && N5 == unspecified && N4 == unspecified) ? 4 - : ( (N6 == unspecified && N5 == unspecified) ? 5 - : ( (N6 == unspecified) ? 6 - : 7 ) ) ) ) ) ) ); + static size_t computeRank(const size_t N0, const size_t N1, const size_t N2, + const size_t N3, const size_t N4, const size_t N5, + const size_t N6, const size_t /* N7 */) { + return ( + (N6 == unspecified && N5 == unspecified && N4 == unspecified && + N3 == unspecified && N2 == unspecified && N1 == unspecified && + N0 == unspecified) + ? 0 + : ((N6 == unspecified && N5 == unspecified && N4 == unspecified && + N3 == unspecified && N2 == unspecified && N1 == unspecified) + ? 1 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified && N3 == unspecified && + N2 == unspecified) + ? 2 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified && N3 == unspecified) + ? 3 + : ((N6 == unspecified && N5 == unspecified && + N4 == unspecified) + ? 4 + : ((N6 == unspecified && + N5 == unspecified) + ? 5 + : ((N6 == unspecified) + ? 6 + : 7))))))); } // Compute the rank of the view from the nonzero layout arguments. template - KOKKOS_INLINE_FUNCTION - static size_t computeRank( const Layout& layout ) - { - return computeRank( layout.dimension[0] - , layout.dimension[1] - , layout.dimension[2] - , layout.dimension[3] - , layout.dimension[4] - , layout.dimension[5] - , layout.dimension[6] - , layout.dimension[7] ); + KOKKOS_INLINE_FUNCTION static size_t computeRank(const Layout& layout) { + return computeRank(layout.dimension[0], layout.dimension[1], + layout.dimension[2], layout.dimension[3], + layout.dimension[4], layout.dimension[5], + layout.dimension[6], layout.dimension[7]); } // Extra overload to match that for specialize types v2 - template - KOKKOS_INLINE_FUNCTION - static size_t computeRank( const Kokkos::Impl::ViewCtorProp& /* prop */, const Layout& layout ) - { + template + KOKKOS_INLINE_FUNCTION static size_t computeRank( + const Kokkos::Impl::ViewCtorProp& /* prop */, + const Layout& layout) { return computeRank(layout); } // Create the layout for the rank-7 view. // Non-strided Layout template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value) , Layout >::type createLayout( const Layout& layout ) - { - return Layout( layout.dimension[0] != unspecified ? layout.dimension[0] : 1 - , layout.dimension[1] != unspecified ? layout.dimension[1] : 1 - , layout.dimension[2] != unspecified ? layout.dimension[2] : 1 - , layout.dimension[3] != unspecified ? layout.dimension[3] : 1 - , layout.dimension[4] != unspecified ? layout.dimension[4] : 1 - , layout.dimension[5] != unspecified ? layout.dimension[5] : 1 - , layout.dimension[6] != unspecified ? layout.dimension[6] : 1 - , layout.dimension[7] != unspecified ? layout.dimension[7] : 1 - ); + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value || + std::is_same::value), + Layout>::type + createLayout(const Layout& layout) { + return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.dimension[7] != unspecified ? layout.dimension[7] : 1); } // LayoutStride template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value) , Layout>::type createLayout( const Layout& layout ) - { - return Layout( layout.dimension[0] != unspecified ? layout.dimension[0] : 1 - , layout.stride[0] - , layout.dimension[1] != unspecified ? layout.dimension[1] : 1 - , layout.stride[1] - , layout.dimension[2] != unspecified ? layout.dimension[2] : 1 - , layout.stride[2] - , layout.dimension[3] != unspecified ? layout.dimension[3] : 1 - , layout.stride[3] - , layout.dimension[4] != unspecified ? layout.dimension[4] : 1 - , layout.stride[4] - , layout.dimension[5] != unspecified ? layout.dimension[5] : 1 - , layout.stride[5] - , layout.dimension[6] != unspecified ? layout.dimension[6] : 1 - , layout.stride[6] - , layout.dimension[7] != unspecified ? layout.dimension[7] : 1 - , layout.stride[7] - ); + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value), Layout>::type + createLayout(const Layout& layout) { + return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.stride[0], + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.stride[1], + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.stride[2], + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.stride[3], + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.stride[4], + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.stride[5], + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.stride[6], + layout.dimension[7] != unspecified ? layout.dimension[7] : 1, + layout.stride[7]); } // Extra overload to match that for specialize types - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value || std::is_same::value) , typename Traits::array_layout >::type createLayout( const Kokkos::Impl::ViewCtorProp& /* prop */, const typename Traits::array_layout& layout ) - { - return createLayout( layout ); + template + KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value || + std::is_same::value || + std::is_same::value), + typename Traits::array_layout>::type + createLayout(const Kokkos::Impl::ViewCtorProp& /* prop */, + const typename Traits::array_layout& layout) { + return createLayout(layout); } // Create a view from the given dimension arguments. // This is only necessary because the shmem constructor doesn't take a layout. - // NDE shmem View's are not compatible with the added view_alloc value_type / fad_dim deduction functionality + // NDE shmem View's are not compatible with the added view_alloc value_type + // / fad_dim deduction functionality template - static ViewType createView( const ViewArg& arg - , const size_t N0 - , const size_t N1 - , const size_t N2 - , const size_t N3 - , const size_t N4 - , const size_t N5 - , const size_t N6 - , const size_t N7 ) - { - return ViewType( arg - , N0 != unspecified ? N0 : 1 - , N1 != unspecified ? N1 : 1 - , N2 != unspecified ? N2 : 1 - , N3 != unspecified ? N3 : 1 - , N4 != unspecified ? N4 : 1 - , N5 != unspecified ? N5 : 1 - , N6 != unspecified ? N6 : 1 - , N7 != unspecified ? N7 : 1 ); + static ViewType createView(const ViewArg& arg, const size_t N0, + const size_t N1, const size_t N2, const size_t N3, + const size_t N4, const size_t N5, const size_t N6, + const size_t N7) { + return ViewType(arg, N0 != unspecified ? N0 : 1, N1 != unspecified ? N1 : 1, + N2 != unspecified ? N2 : 1, N3 != unspecified ? N3 : 1, + N4 != unspecified ? N4 : 1, N5 != unspecified ? N5 : 1, + N6 != unspecified ? N6 : 1, N7 != unspecified ? N7 : 1); } }; - // Non-strided Layout - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value || std::is_same::value) && std::is_integral::value , Layout >::type - reconstructLayout( const Layout& layout , iType dynrank ) - { - return Layout( dynrank > 0 ? layout.dimension[0] :KOKKOS_INVALID_INDEX - , dynrank > 1 ? layout.dimension[1] :KOKKOS_INVALID_INDEX - , dynrank > 2 ? layout.dimension[2] :KOKKOS_INVALID_INDEX - , dynrank > 3 ? layout.dimension[3] :KOKKOS_INVALID_INDEX - , dynrank > 4 ? layout.dimension[4] :KOKKOS_INVALID_INDEX - , dynrank > 5 ? layout.dimension[5] :KOKKOS_INVALID_INDEX - , dynrank > 6 ? layout.dimension[6] :KOKKOS_INVALID_INDEX - , dynrank > 7 ? layout.dimension[7] :KOKKOS_INVALID_INDEX - ); - } - - // LayoutStride - template - KOKKOS_INLINE_FUNCTION - static typename std::enable_if< (std::is_same::value) && std::is_integral::value , Layout >::type - reconstructLayout( const Layout& layout , iType dynrank ) - { - return Layout( dynrank > 0 ? layout.dimension[0] :KOKKOS_INVALID_INDEX - , dynrank > 0 ? layout.stride[0] : (0) - , dynrank > 1 ? layout.dimension[1] :KOKKOS_INVALID_INDEX - , dynrank > 1 ? layout.stride[1] : (0) - , dynrank > 2 ? layout.dimension[2] :KOKKOS_INVALID_INDEX - , dynrank > 2 ? layout.stride[2] : (0) - , dynrank > 3 ? layout.dimension[3] :KOKKOS_INVALID_INDEX - , dynrank > 3 ? layout.stride[3] : (0) - , dynrank > 4 ? layout.dimension[4] :KOKKOS_INVALID_INDEX - , dynrank > 4 ? layout.stride[4] : (0) - , dynrank > 5 ? layout.dimension[5] :KOKKOS_INVALID_INDEX - , dynrank > 5 ? layout.stride[5] : (0) - , dynrank > 6 ? layout.dimension[6] :KOKKOS_INVALID_INDEX - , dynrank > 6 ? layout.stride[6] : (0) - , dynrank > 7 ? layout.dimension[7] :KOKKOS_INVALID_INDEX - , dynrank > 7 ? layout.stride[7] : (0) - ); - } +// Non-strided Layout +template +KOKKOS_INLINE_FUNCTION static + typename std::enable_if<(std::is_same::value || + std::is_same::value) && + std::is_integral::value, + Layout>::type + reconstructLayout(const Layout& layout, iType dynrank) { + return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, + dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, + dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, + dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, + dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, + dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, + dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, + dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX); +} +// LayoutStride +template +KOKKOS_INLINE_FUNCTION static typename std::enable_if< + (std::is_same::value) && + std::is_integral::value, + Layout>::type +reconstructLayout(const Layout& layout, iType dynrank) { + return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, + dynrank > 0 ? layout.stride[0] : (0), + dynrank > 1 ? layout.dimension[1] : KOKKOS_INVALID_INDEX, + dynrank > 1 ? layout.stride[1] : (0), + dynrank > 2 ? layout.dimension[2] : KOKKOS_INVALID_INDEX, + dynrank > 2 ? layout.stride[2] : (0), + dynrank > 3 ? layout.dimension[3] : KOKKOS_INVALID_INDEX, + dynrank > 3 ? layout.stride[3] : (0), + dynrank > 4 ? layout.dimension[4] : KOKKOS_INVALID_INDEX, + dynrank > 4 ? layout.stride[4] : (0), + dynrank > 5 ? layout.dimension[5] : KOKKOS_INVALID_INDEX, + dynrank > 5 ? layout.stride[5] : (0), + dynrank > 6 ? layout.dimension[6] : KOKKOS_INVALID_INDEX, + dynrank > 6 ? layout.stride[6] : (0), + dynrank > 7 ? layout.dimension[7] : KOKKOS_INVALID_INDEX, + dynrank > 7 ? layout.stride[7] : (0)); +} /** \brief Debug bounds-checking routines */ // Enhanced debug checking - most infrastructure matches that of functions in // Kokkos_ViewMapping; additional checks for extra arguments beyond rank are 0 -template< unsigned , typename iType0 , class MapType > -KOKKOS_INLINE_FUNCTION -bool dyn_rank_view_verify_operator_bounds( const iType0 & , const MapType & ) -{ return true ; } +template +KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( + const iType0&, const MapType&) { + return true; +} -template< unsigned R , typename iType0 , class MapType , typename iType1 , class ... Args > -KOKKOS_INLINE_FUNCTION -bool dyn_rank_view_verify_operator_bounds - ( const iType0 & rank - , const MapType & map - , const iType1 & i - , Args ... args - ) -{ - if ( static_cast(R) < rank ) { - return ( size_t(i) < map.extent(R) ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); - } - else if ( i != 0 ) { - printf("DynRankView Debug Bounds Checking Error: at rank %u\n Extra arguments beyond the rank must be zero \n",R); - return ( false ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); - } - else { - return ( true ) - && dyn_rank_view_verify_operator_bounds( rank , map , args ... ); +template +KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( + const iType0& rank, const MapType& map, const iType1& i, Args... args) { + if (static_cast(R) < rank) { + return (size_t(i) < map.extent(R)) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); + } else if (i != 0) { + printf( + "DynRankView Debug Bounds Checking Error: at rank %u\n Extra " + "arguments beyond the rank must be zero \n", + R); + return (false) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); + } else { + return (true) && + dyn_rank_view_verify_operator_bounds(rank, map, args...); } } -template< unsigned , class MapType > -inline -void dyn_rank_view_error_operator_bounds( char * , int , const MapType & ) -{} +template +inline void dyn_rank_view_error_operator_bounds(char*, int, const MapType&) {} -template< unsigned R , class MapType , class iType , class ... Args > -inline -void dyn_rank_view_error_operator_bounds - ( char * buf - , int len - , const MapType & map - , const iType & i - , Args ... args - ) -{ - const int n = - snprintf(buf,len," %ld < %ld %c" - , static_cast(i) - , static_cast( map.extent(R) ) - , ( sizeof...(Args) ? ',' : ')' ) - ); - dyn_rank_view_error_operator_bounds(buf+n,len-n,map,args...); +template +inline void dyn_rank_view_error_operator_bounds(char* buf, int len, + const MapType& map, + const iType& i, Args... args) { + const int n = snprintf( + buf, len, " %ld < %ld %c", static_cast(i), + static_cast(map.extent(R)), (sizeof...(Args) ? ',' : ')')); + dyn_rank_view_error_operator_bounds(buf + n, len - n, map, args...); } // op_rank = rank of the operator version that was called -template< typename MemorySpace - , typename iType0 , typename iType1 , class MapType , class ... Args > -KOKKOS_INLINE_FUNCTION -void dyn_rank_view_verify_operator_bounds - ( const iType0 & op_rank , const iType1 & rank - , const Kokkos::Impl::SharedAllocationTracker & tracker - , const MapType & map , Args ... args ) -{ - if ( static_cast(rank) > op_rank ) { - Kokkos::abort( "DynRankView Bounds Checking Error: Need at least rank arguments to the operator()" ); +template +KOKKOS_INLINE_FUNCTION void dyn_rank_view_verify_operator_bounds( + const iType0& op_rank, const iType1& rank, + const Kokkos::Impl::SharedAllocationTracker& tracker, const MapType& map, + Args... args) { + if (static_cast(rank) > op_rank) { + Kokkos::abort( + "DynRankView Bounds Checking Error: Need at least rank arguments to " + "the operator()"); } - if ( ! dyn_rank_view_verify_operator_bounds<0>( rank , map , args ... ) ) { -#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST ) + if (!dyn_rank_view_verify_operator_bounds<0>(rank, map, args...)) { +#if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST) enum { LEN = 1024 }; - char buffer[ LEN ]; + char buffer[LEN]; const std::string label = tracker.template get_label(); - int n = snprintf(buffer,LEN,"DynRankView bounds error of view %s (", label.c_str()); - dyn_rank_view_error_operator_bounds<0>( buffer + n , LEN - n , map , args ... ); + int n = snprintf(buffer, LEN, "DynRankView bounds error of view %s (", + label.c_str()); + dyn_rank_view_error_operator_bounds<0>(buffer + n, LEN - n, map, args...); Kokkos::Impl::throw_runtime_exception(std::string(buffer)); #else + (void)tracker; Kokkos::abort("DynRankView bounds error"); #endif } } - /** \brief Assign compatible default mappings */ struct ViewToDynRankViewTag {}; -} // namespace Impl +} // namespace Impl namespace Impl { -template< class DstTraits , class SrcTraits > -class ViewMapping< DstTraits , SrcTraits , - typename std::enable_if<( - std::is_same< typename DstTraits::memory_space , typename SrcTraits::memory_space >::value - && - std::is_same< typename DstTraits::specialize , void >::value - && - std::is_same< typename SrcTraits::specialize , void >::value - && - ( - std::is_same< typename DstTraits::array_layout , typename SrcTraits::array_layout >::value - || - ( - ( - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutLeft >::value || - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutRight >::value || - std::is_same< typename DstTraits::array_layout , Kokkos::LayoutStride >::value - ) - && - ( - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutLeft >::value || - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutRight >::value || - std::is_same< typename SrcTraits::array_layout , Kokkos::LayoutStride >::value - ) - ) - ) - ) , Kokkos::Impl::ViewToDynRankViewTag >::type > -{ -private: +template +class ViewMapping< + DstTraits, SrcTraits, + typename std::enable_if< + (std::is_same::value && + std::is_same::value && + std::is_same::value && + (std::is_same::value || + ((std::is_same::value || + std::is_same::value || + std::is_same::value) && + (std::is_same::value || + std::is_same::value || + std::is_same::value)))), + Kokkos::Impl::ViewToDynRankViewTag>::type> { + private: + enum { + is_assignable_value_type = + std::is_same::value || + std::is_same::value + }; - enum { is_assignable_value_type = - std::is_same< typename DstTraits::value_type - , typename SrcTraits::value_type >::value || - std::is_same< typename DstTraits::value_type - , typename SrcTraits::const_value_type >::value }; + enum { + is_assignable_layout = + std::is_same::value || + std::is_same::value + }; - enum { is_assignable_layout = - std::is_same< typename DstTraits::array_layout - , typename SrcTraits::array_layout >::value || - std::is_same< typename DstTraits::array_layout - , Kokkos::LayoutStride >::value - }; + public: + enum { is_assignable = is_assignable_value_type && is_assignable_layout }; -public: + typedef ViewMapping DstType; + typedef ViewMapping SrcType; - enum { is_assignable = is_assignable_value_type && - is_assignable_layout }; + template + KOKKOS_INLINE_FUNCTION static void assign( + Kokkos::DynRankView& dst, const Kokkos::View& src) { + static_assert( + is_assignable_value_type, + "View assignment must have same value type or const = non-const"); - typedef ViewMapping< DstTraits , typename DstTraits::specialize > DstType ; - typedef ViewMapping< SrcTraits , typename SrcTraits::specialize > SrcType ; - - template < typename DT , typename ... DP , typename ST , typename ... SP > - KOKKOS_INLINE_FUNCTION - static void assign( Kokkos::DynRankView< DT , DP...> & dst , const Kokkos::View< ST , SP... > & src ) - { - static_assert( is_assignable_value_type - , "View assignment must have same value type or const = non-const" ); - - static_assert( is_assignable_layout - , "View assignment must have compatible layout or have rank <= 1" ); + static_assert( + is_assignable_layout, + "View assignment must have compatible layout or have rank <= 1"); // Removed dimension checks... - typedef typename DstType::offset_type dst_offset_type ; - dst.m_map.m_impl_offset = dst_offset_type(std::integral_constant() , src.layout() ); //Check this for integer input1 for padding, etc - dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle< DstTraits >::assign( src.m_map.m_impl_handle , src.m_track ); - dst.m_track.assign( src.m_track , DstTraits::is_managed ); - dst.m_rank = src.Rank ; - } + typedef typename DstType::offset_type dst_offset_type; + dst.m_map.m_impl_offset = dst_offset_type( + std::integral_constant(), + src.layout()); // Check this for integer input1 for padding, etc + dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle::assign( + src.m_map.m_impl_handle, src.m_track); + dst.m_track.assign(src.m_track, DstTraits::is_managed); + dst.m_rank = src.Rank; + } }; -} //end Impl +} // namespace Impl /* \class DynRankView * \brief Container that creates a Kokkos view with rank determined at runtime. @@ -400,7 +385,8 @@ public: * Changes from View * 1. The rank of the DynRankView is returned by the method rank() * 2. Max rank of a DynRankView is 7 - * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward compatibility) + * 3. subview called with 'subview(...)' or 'subdynrankview(...)' (backward + * compatibility) * 4. Every subview is returned with LayoutStride * 5. Copy and Copy-Assign View to DynRankView * 6. deep_copy between Views and DynRankViews @@ -408,93 +394,99 @@ public: * */ -template< class > struct is_dyn_rank_view : public std::false_type {}; +template +struct is_dyn_rank_view : public std::false_type {}; -template< class D, class ... P > -struct is_dyn_rank_view< Kokkos::DynRankView > : public std::true_type {}; +template +struct is_dyn_rank_view > : public std::true_type { +}; +template +class DynRankView : public ViewTraits { + static_assert(!std::is_array::value && + !std::is_pointer::value, + "Cannot template DynRankView with array or pointer datatype - " + "must be pod"); -template< typename DataType , class ... Properties > -class DynRankView : public ViewTraits< DataType , Properties ... > -{ - static_assert( !std::is_array::value && !std::is_pointer::value , "Cannot template DynRankView with array or pointer datatype - must be pod" ); + private: + template + friend class DynRankView; + template + friend class Kokkos::Impl::ViewMapping; -private: - template < class , class ... > friend class DynRankView ; - template < class , class ... > friend class Kokkos::Impl::ViewMapping ; + public: + typedef ViewTraits drvtraits; -public: - typedef ViewTraits< DataType , Properties ... > drvtraits ; + typedef View view_type; - typedef View< DataType******* , Properties...> view_type ; + typedef ViewTraits traits; - typedef ViewTraits< DataType******* , Properties ... > traits ; + private: + typedef Kokkos::Impl::ViewMapping + map_type; + typedef Kokkos::Impl::SharedAllocationTracker track_type; - -private: - typedef Kokkos::Impl::ViewMapping< traits , typename traits::specialize > map_type ; - typedef Kokkos::Impl::SharedAllocationTracker track_type ; - - track_type m_track ; - map_type m_map ; + track_type m_track; + map_type m_map; unsigned m_rank; -public: + public: KOKKOS_INLINE_FUNCTION - view_type & DownCast() const { return ( view_type & ) (*this); } + view_type& DownCast() const { return (view_type&)(*this); } KOKKOS_INLINE_FUNCTION - const view_type & ConstDownCast() const { return (const view_type & ) (*this); } + const view_type& ConstDownCast() const { return (const view_type&)(*this); } - //Types below - at least the HostMirror requires the value_type, NOT the rank 7 data_type of the traits + // Types below - at least the HostMirror requires the value_type, NOT the rank + // 7 data_type of the traits /** \brief Compatible view of array of scalar types */ - typedef DynRankView< typename drvtraits::scalar_array_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - array_type ; + typedef DynRankView< + typename drvtraits::scalar_array_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + array_type; /** \brief Compatible view of const data type */ - typedef DynRankView< typename drvtraits::const_data_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - const_type ; + typedef DynRankView< + typename drvtraits::const_data_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + const_type; /** \brief Compatible view of non-const data type */ - typedef DynRankView< typename drvtraits::non_const_data_type , - typename drvtraits::array_layout , - typename drvtraits::device_type , - typename drvtraits::memory_traits > - non_const_type ; + typedef DynRankView< + typename drvtraits::non_const_data_type, typename drvtraits::array_layout, + typename drvtraits::device_type, typename drvtraits::memory_traits> + non_const_type; /** \brief Compatible HostMirror view */ - typedef DynRankView< typename drvtraits::non_const_data_type , - typename drvtraits::array_layout , - typename drvtraits::host_mirror_space > - HostMirror ; - + typedef DynRankView + HostMirror; //---------------------------------------- // Domain rank and extents -// enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the enum? + // enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the + // enum? - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - extent( const iType & r ) const - { return m_map.extent(r); } + typename std::enable_if::value, size_t>::type + extent(const iType& r) const { + return m_map.extent(r); + } - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , int >::type - extent_int( const iType & r ) const - { return static_cast(m_map.extent(r)); } + typename std::enable_if::value, int>::type + extent_int(const iType& r) const { + return static_cast(m_map.extent(r)); + } - KOKKOS_INLINE_FUNCTION constexpr - typename traits::array_layout layout() const - { return m_map.layout(); } + KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() + const { + return m_map.layout(); + } //---------------------------------------- /* Deprecate all 'dimension' functions in favor of @@ -502,830 +494,969 @@ public: */ #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - template< typename iType > + template KOKKOS_INLINE_FUNCTION constexpr - typename std::enable_if< std::is_integral::value , size_t >::type - dimension( const iType & r ) const { return extent( r ); } + typename std::enable_if::value, size_t>::type + dimension(const iType& r) const { + return extent(r); + } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const { return m_map.dimension_0(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const { return m_map.dimension_1(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const { return m_map.dimension_2(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const { return m_map.dimension_3(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const { return m_map.dimension_4(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const { return m_map.dimension_5(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const { return m_map.dimension_6(); } - KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const { return m_map.dimension_7(); } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const { + return m_map.dimension_0(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const { + return m_map.dimension_1(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const { + return m_map.dimension_2(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const { + return m_map.dimension_3(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const { + return m_map.dimension_4(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const { + return m_map.dimension_5(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const { + return m_map.dimension_6(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const { + return m_map.dimension_7(); + } #endif //---------------------------------------- - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { return m_map.extent(0) * - m_map.extent(1) * - m_map.extent(2) * - m_map.extent(3) * - m_map.extent(4) * - m_map.extent(5) * - m_map.extent(6) * - m_map.extent(7); } + KOKKOS_INLINE_FUNCTION constexpr size_t size() const { + return m_map.extent(0) * m_map.extent(1) * m_map.extent(2) * + m_map.extent(3) * m_map.extent(4) * m_map.extent(5) * + m_map.extent(6) * m_map.extent(7); + } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { + return m_map.stride_0(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { + return m_map.stride_1(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { + return m_map.stride_2(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { + return m_map.stride_3(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { + return m_map.stride_4(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { + return m_map.stride_5(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { + return m_map.stride_6(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { + return m_map.stride_7(); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION void stride( iType * const s ) const { m_map.stride(s); } + template + KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { + m_map.stride(s); + } //---------------------------------------- // Range span is the span which contains all members. - typedef typename map_type::reference_type reference_type ; - typedef typename map_type::pointer_type pointer_type ; + typedef typename map_type::reference_type reference_type; + typedef typename map_type::pointer_type pointer_type; - enum { reference_type_is_lvalue_reference = std::is_lvalue_reference< reference_type >::value }; + enum { + reference_type_is_lvalue_reference = + std::is_lvalue_reference::value + }; KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE // Deprecated, use 'span()' instead - KOKKOS_INLINE_FUNCTION constexpr size_t capacity() const { return m_map.span(); } + KOKKOS_INLINE_FUNCTION constexpr size_t capacity() const { + return m_map.span(); + } #endif - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { return m_map.span_is_contiguous(); } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return m_map.data(); } + KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { + return m_map.span_is_contiguous(); + } + KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { + return m_map.data(); + } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE // Deprecated, use 'span_is_contigous()' instead - KOKKOS_INLINE_FUNCTION constexpr bool is_contiguous() const { return m_map.span_is_contiguous(); } + KOKKOS_INLINE_FUNCTION constexpr bool is_contiguous() const { + return m_map.span_is_contiguous(); + } // Deprecated, use 'data()' instead - KOKKOS_INLINE_FUNCTION constexpr pointer_type ptr_on_device() const { return m_map.data(); } + KOKKOS_INLINE_FUNCTION constexpr pointer_type ptr_on_device() const { + return m_map.data(); + } #endif //---------------------------------------- // Allow specializations to query their specialized map #ifdef KOKKOS_ENABLE_DEPRECATED_CODE KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping< traits , typename traits::specialize > & - implementation_map() const { return m_map ; } + const Kokkos::Impl::ViewMapping& + implementation_map() const { + return m_map; + } #endif KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping< traits , typename traits::specialize > & - impl_map() const { return m_map ; } + const Kokkos::Impl::ViewMapping& + impl_map() const { + return m_map; + } //---------------------------------------- -private: - + private: enum { - is_layout_left = std::is_same< typename traits::array_layout - , Kokkos::LayoutLeft >::value , + is_layout_left = + std::is_same::value, - is_layout_right = std::is_same< typename traits::array_layout - , Kokkos::LayoutRight >::value , + is_layout_right = + std::is_same::value, - is_layout_stride = std::is_same< typename traits::array_layout - , Kokkos::LayoutStride >::value , + is_layout_stride = std::is_same::value, - is_default_map = - std::is_same< typename traits::specialize , void >::value && - ( is_layout_left || is_layout_right || is_layout_stride ) + is_default_map = std::is_same::value && + (is_layout_left || is_layout_right || is_layout_stride) }; - template< class Space , bool = Kokkos::Impl::MemorySpaceAccess< Space , typename traits::memory_space >::accessible > struct verify_space - { KOKKOS_FORCEINLINE_FUNCTION static void check() {} }; + template ::accessible> + struct verify_space { + KOKKOS_FORCEINLINE_FUNCTION static void check() {} + }; - template< class Space > struct verify_space - { KOKKOS_FORCEINLINE_FUNCTION static void check() - { Kokkos::abort("Kokkos::DynRankView ERROR: attempt to access inaccessible memory space"); }; + template + struct verify_space { + KOKKOS_FORCEINLINE_FUNCTION static void check() { + Kokkos::abort( + "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " + "space"); }; + }; // Bounds checking macros -#if defined( KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK ) +#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) // rank of the calling operator - included as first argument in ARG -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( ARG ) \ - DynRankView::template verify_space< Kokkos::Impl::ActiveExecutionMemorySpace >::check(); \ - Kokkos::Impl::dyn_rank_view_verify_operator_bounds< typename traits::memory_space > ARG ; +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + DynRankView::template verify_space< \ + Kokkos::Impl::ActiveExecutionMemorySpace>::check(); \ + Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ + typename traits::memory_space> \ + ARG; #else -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( ARG ) \ - DynRankView::template verify_space< Kokkos::Impl::ActiveExecutionMemorySpace >::check(); +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + DynRankView::template verify_space< \ + Kokkos::Impl::ActiveExecutionMemorySpace>::check(); #endif -public: - + public: KOKKOS_INLINE_FUNCTION constexpr unsigned rank() const { return m_rank; } - - //operators () + // operators () // Rank 0 KOKKOS_INLINE_FUNCTION - reference_type operator()() const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (0 , this->rank(), m_track, m_map) ) - return impl_map().reference(); - //return m_map.reference(0,0,0,0,0,0,0); - } + reference_type operator()() const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) + return impl_map().reference(); + // return m_map.reference(0,0,0,0,0,0,0); + } // Rank 1 - // This assumes a contiguous underlying memory (i.e. no padding, no striding...) - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< std::is_same::value && std::is_integral::value, reference_type>::type - operator[](const iType & i0) const - { - //Phalanx is violating this, since they use the operator to access ALL elements in the allocation - //KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map) ) - return data()[i0]; - } + // This assumes a contiguous underlying memory (i.e. no padding, no + // striding...) + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + std::is_same::value && + std::is_integral::value, + reference_type>::type + operator[](const iType& i0) const { + // Phalanx is violating this, since they use the operator to access ALL + // elements in the allocation KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , + // this->rank(), m_track, m_map) ) + return data()[i0]; + } - // This assumes a contiguous underlying memory (i.e. no padding, no striding... - // AND a Trilinos/Sacado scalar type ) - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !std::is_same::value && std::is_integral::value, reference_type>::type - operator[](const iType & i0) const - { -// auto map = impl_map(); - const size_t dim_scalar = m_map.dimension_scalar(); - const size_t bytes = this->span() / dim_scalar; + // This assumes a contiguous underlying memory (i.e. no padding, no + // striding... AND a Trilinos/Sacado scalar type ) + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !std::is_same::value && + std::is_integral::value, + reference_type>::type + operator[](const iType& i0) const { + // auto map = impl_map(); + const size_t dim_scalar = m_map.dimension_scalar(); + const size_t bytes = this->span() / dim_scalar; - typedef Kokkos::View > tmp_view_type; - tmp_view_type rankone_view(this->data(), bytes, dim_scalar); - return rankone_view(i0); - } + typedef Kokkos::View< + DataType*, typename traits::array_layout, typename traits::device_type, + Kokkos::MemoryTraits > + tmp_view_type; + tmp_view_type rankone_view(this->data(), bytes, dim_scalar); + return rankone_view(i0); + } // Rank 1 parenthesis - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0,0,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0, 0, 0, 0, 0, 0, 0); + } // Rank 2 - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1); + } - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1, 0, 0, 0, 0, 0); + } // Rank 3 - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2); + } - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2, 0, 0, 0, 0); + } // Rank 4 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3, 0, 0, 0); + } // Rank 5 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4, 0, 0); + } // Rank 6 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5, 0); + } // Rank 7 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 , typename iType6 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - operator()(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,i6); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + operator()(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3, const iType4& i4, const iType5& i5, + const iType6& i6) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6); + } // Rank 0 KOKKOS_INLINE_FUNCTION - reference_type access() const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (0 , this->rank(), m_track, m_map) ) - return impl_map().reference(); - //return m_map.reference(0,0,0,0,0,0,0); - } + reference_type access() const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) + return impl_map().reference(); + // return m_map.reference(0,0,0,0,0,0,0); + } // Rank 1 - // Rank 1 parenthesis - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0); - } + // Rank 1 parenthesis + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0); + } - template< typename iType > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType & i0 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , this->rank(), m_track, m_map, i0) ) - return m_map.reference(i0,0,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType& i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) + return m_map.reference(i0, 0, 0, 0, 0, 0, 0); + } // Rank 2 - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1); + } - template< typename iType0 , typename iType1 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (2 , this->rank(), m_track, m_map, i0, i1) ) - return m_map.reference(i0,i1,0,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) + return m_map.reference(i0, i1, 0, 0, 0, 0, 0); + } // Rank 3 - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2); + } - template< typename iType0 , typename iType1 , typename iType2 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (3 , this->rank(), m_track, m_map, i0, i1, i2) ) - return m_map.reference(i0,i1,i2,0,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (3, this->rank(), m_track, m_map, i0, i1, i2)) + return m_map.reference(i0, i1, i2, 0, 0, 0, 0); + } // Rank 4 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (4 , this->rank(), m_track, m_map, i0, i1, i2, i3) ) - return m_map.reference(i0,i1,i2,i3,0,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, + const iType3& i3) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) + return m_map.reference(i0, i1, i2, i3, 0, 0, 0); + } // Rank 5 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (5 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4) ) - return m_map.reference(i0,i1,i2,i3,i4,0,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) + return m_map.reference(i0, i1, i2, i3, i4, 0, 0); + } // Rank 6 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_same::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_same::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5); + } - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< !(std::is_same::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (6 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,0); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + !(std::is_same::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) + return m_map.reference(i0, i1, i2, i3, i4, i5, 0); + } // Rank 7 - template< typename iType0 , typename iType1 , typename iType2 , typename iType3, typename iType4 , typename iType5 , typename iType6 > - KOKKOS_INLINE_FUNCTION - typename std::enable_if< (std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value && std::is_integral::value), reference_type>::type - access(const iType0 & i0 , const iType1 & i1 , const iType2 & i2 , const iType3 & i3 , const iType4 & i4 , const iType5 & i5 , const iType6 & i6 ) const - { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (7 , this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6) ) - return m_map.reference(i0,i1,i2,i3,i4,i5,i6); - } + template + KOKKOS_INLINE_FUNCTION typename std::enable_if< + (std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value && std::is_integral::value && + std::is_integral::value), + reference_type>::type + access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, + const iType4& i4, const iType5& i5, const iType6& i6) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( + (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6); + } #undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY //---------------------------------------- // Standard constructor, destructor, and assignment operators... - KOKKOS_INLINE_FUNCTION - ~DynRankView() {} + KOKKOS_DEFAULTED_FUNCTION + ~DynRankView() = default; KOKKOS_INLINE_FUNCTION - DynRankView() : m_track(), m_map(), m_rank() {} //Default ctor + DynRankView() : m_track(), m_map(), m_rank() {} // Default ctor KOKKOS_INLINE_FUNCTION - DynRankView( const DynRankView & rhs ) : m_track( rhs.m_track ), m_map( rhs.m_map ), m_rank(rhs.m_rank) {} + DynRankView(const DynRankView& rhs) + : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION - DynRankView( DynRankView && rhs ) : m_track( rhs.m_track ), m_map( rhs.m_map ), m_rank(rhs.m_rank) {} + DynRankView(DynRankView&& rhs) + : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( const DynRankView & rhs ) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } + DynRankView& operator=(const DynRankView& rhs) { + m_track = rhs.m_track; + m_map = rhs.m_map; + m_rank = rhs.m_rank; + return *this; + } KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( DynRankView && rhs ) { m_track = rhs.m_track; m_map = rhs.m_map; m_rank = rhs.m_rank; return *this; } + DynRankView& operator=(DynRankView&& rhs) { + m_track = rhs.m_track; + m_map = rhs.m_map; + m_rank = rhs.m_rank; + return *this; + } //---------------------------------------- // Compatible view copy constructor and assignment // may assign unmanaged from managed. - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView( const DynRankView & rhs ) - : m_track( rhs.m_track , traits::is_managed ) - , m_map() - , m_rank(rhs.m_rank) - { - typedef typename DynRankView ::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , typename traits::specialize > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible DynRankView copy construction" ); - Mapping::assign( m_map , rhs.m_map , rhs.m_track ); - } + template + KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) + : m_track(rhs.m_track, traits::is_managed), m_map(), m_rank(rhs.m_rank) { + typedef typename DynRankView::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible DynRankView copy construction"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track); + } - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView & operator = (const DynRankView & rhs ) - { - typedef typename DynRankView ::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , typename traits::specialize > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible DynRankView copy construction" ); - Mapping::assign( m_map , rhs.m_map , rhs.m_track ); - m_track.assign( rhs.m_track , traits::is_managed ); - m_rank = rhs.rank(); - return *this; - } + template + KOKKOS_INLINE_FUNCTION DynRankView& operator=( + const DynRankView& rhs) { + typedef typename DynRankView::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible DynRankView copy construction"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track); + m_track.assign(rhs.m_track, traits::is_managed); + m_rank = rhs.rank(); + return *this; + } -// Copy/Assign View to DynRankView - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView( const View & rhs ) - : m_track() - , m_map() - , m_rank( rhs.Rank ) - { - typedef typename View::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , Kokkos::Impl::ViewToDynRankViewTag > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible View to DynRankView copy construction" ); - Mapping::assign( *this , rhs ); - } + // Copy/Assign View to DynRankView + template + KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs) + : m_track(), m_map(), m_rank(rhs.Rank) { + typedef typename View::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible View to DynRankView copy construction"); + Mapping::assign(*this, rhs); + } - template< class RT , class ... RP > - KOKKOS_INLINE_FUNCTION - DynRankView & operator = ( const View & rhs ) - { - typedef typename View::traits SrcTraits ; - typedef Kokkos::Impl::ViewMapping< traits , SrcTraits , Kokkos::Impl::ViewToDynRankViewTag > Mapping ; - static_assert( Mapping::is_assignable , "Incompatible View to DynRankView copy assignment" ); - Mapping::assign( *this , rhs ); - return *this ; - } + template + KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { + typedef typename View::traits SrcTraits; + typedef Kokkos::Impl::ViewMapping + Mapping; + static_assert(Mapping::is_assignable, + "Incompatible View to DynRankView copy assignment"); + Mapping::assign(*this, rhs); + return *this; + } //---------------------------------------- // Allocation tracking properties KOKKOS_INLINE_FUNCTION - int use_count() const - { return m_track.use_count(); } + int use_count() const { return m_track.use_count(); } - inline - const std::string label() const - { return m_track.template get_label< typename traits::memory_space >(); } + inline const std::string label() const { + return m_track.template get_label(); + } //---------------------------------------- // Allocation according to allocation properties and array layout - // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that rank deduction can properly take place - template< class ... P > - explicit inline - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< ! Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , typename traits::array_layout - >::type const & arg_layout - ) - : m_track() - , m_map() - , m_rank( Impl::DynRankDimTraits::template computeRank< typename traits::array_layout, P...>(arg_prop, arg_layout) ) - { - // Append layout and spaces if not input - typedef Kokkos::Impl::ViewCtorProp< P ... > alloc_prop_input ; + // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that + // rank deduction can properly take place + template + explicit inline DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + typename traits::array_layout>::type const& + arg_layout) + : m_track(), + m_map(), + m_rank(Impl::DynRankDimTraits:: + template computeRank( + arg_prop, arg_layout)) { + // Append layout and spaces if not input + typedef Kokkos::Impl::ViewCtorProp alloc_prop_input; - // use 'std::integral_constant' for non-types - // to avoid duplicate class error. - typedef Kokkos::Impl::ViewCtorProp - < P ... - , typename std::conditional - < alloc_prop_input::has_label - , std::integral_constant - , typename std::string - >::type - , typename std::conditional - < alloc_prop_input::has_memory_space - , std::integral_constant - , typename traits::device_type::memory_space - >::type - , typename std::conditional - < alloc_prop_input::has_execution_space - , std::integral_constant - , typename traits::device_type::execution_space - >::type - > alloc_prop ; + // use 'std::integral_constant' for non-types + // to avoid duplicate class error. + typedef Kokkos::Impl::ViewCtorProp< + P..., + typename std::conditional, + typename std::string>::type, + typename std::conditional< + alloc_prop_input::has_memory_space, + std::integral_constant, + typename traits::device_type::memory_space>::type, + typename std::conditional< + alloc_prop_input::has_execution_space, + std::integral_constant, + typename traits::device_type::execution_space>::type> + alloc_prop; - static_assert( traits::is_managed - , "View allocation constructor requires managed memory" ); + static_assert(traits::is_managed, + "View allocation constructor requires managed memory"); - if ( alloc_prop::initialize && + if (alloc_prop::initialize && #ifdef KOKKOS_ENABLE_DEPRECATED_CODE - ! alloc_prop::execution_space::is_initialized() + !alloc_prop::execution_space::is_initialized() #else - ! alloc_prop::execution_space::impl_is_initialized() + !alloc_prop::execution_space::impl_is_initialized() #endif - ) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception("Constructing DynRankView and initializing data with uninitialized execution space"); - } - - // Copy the input allocation properties with possibly defaulted properties - alloc_prop prop_copy( arg_prop ); - -//------------------------------------------------------------ -#if defined( KOKKOS_ENABLE_CUDA ) - // If allocating in CudaUVMSpace must fence before and after - // the allocation to protect against possible concurrent access - // on the CPU and the GPU. - // Fence using the trait's executon space (which will be Kokkos::Cuda) - // to avoid incomplete type errors from usng Kokkos::Cuda directly. - if ( std::is_same< Kokkos::CudaUVMSpace , typename traits::device_type::memory_space >::value ) { - typename traits::device_type::memory_space::execution_space().fence(); - } -#endif -//------------------------------------------------------------ - - Kokkos::Impl::SharedAllocationRecord<> * - record = m_map.allocate_shared( prop_copy, Impl::DynRankDimTraits::template createLayout(arg_prop, arg_layout) ); - -//------------------------------------------------------------ -#if defined( KOKKOS_ENABLE_CUDA ) - if ( std::is_same< Kokkos::CudaUVMSpace , typename traits::device_type::memory_space >::value ) { - typename traits::device_type::memory_space::execution_space().fence(); - } -#endif -//------------------------------------------------------------ - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized( record ); + ) { + // If initializing view data then + // the execution space must be initialized. + Kokkos::Impl::throw_runtime_exception( + "Constructing DynRankView and initializing data with uninitialized " + "execution space"); } + // Copy the input allocation properties with possibly defaulted properties + alloc_prop prop_copy(arg_prop); + +//------------------------------------------------------------ +#if defined(KOKKOS_ENABLE_CUDA) + // If allocating in CudaUVMSpace must fence before and after + // the allocation to protect against possible concurrent access + // on the CPU and the GPU. + // Fence using the trait's executon space (which will be Kokkos::Cuda) + // to avoid incomplete type errors from usng Kokkos::Cuda directly. + if (std::is_same::value) { + typename traits::device_type::memory_space::execution_space().fence(); + } +#endif + //------------------------------------------------------------ + + Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( + prop_copy, + Impl::DynRankDimTraits:: + template createLayout(arg_prop, arg_layout)); + +//------------------------------------------------------------ +#if defined(KOKKOS_ENABLE_CUDA) + if (std::is_same::value) { + typename traits::device_type::memory_space::execution_space().fence(); + } +#endif + //------------------------------------------------------------ + + // Setup and initialization complete, start tracking + m_track.assign_allocated_record_to_uninitialized(record); + } // Wrappers - template< class ... P > - explicit KOKKOS_INLINE_FUNCTION - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , typename traits::array_layout - >::type const & arg_layout - ) - : m_track() // No memory tracking - , m_map( arg_prop , Impl::DynRankDimTraits::template createLayout(arg_prop, arg_layout) ) - , m_rank( Impl::DynRankDimTraits::template computeRank< typename traits::array_layout, P...>(arg_prop, arg_layout) ) - { - static_assert( - std::is_same< pointer_type - , typename Impl::ViewCtorProp< P... >::pointer_type - >::value , - "Constructing DynRankView to wrap user memory must supply matching pointer type" ); - } + template + explicit KOKKOS_INLINE_FUNCTION DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + typename traits::array_layout>::type const& + arg_layout) + : m_track() // No memory tracking + , + m_map(arg_prop, + Impl::DynRankDimTraits:: + template createLayout(arg_prop, arg_layout)), + m_rank(Impl::DynRankDimTraits:: + template computeRank( + arg_prop, arg_layout)) { + static_assert( + std::is_same::pointer_type>::value, + "Constructing DynRankView to wrap user memory must supply matching " + "pointer type"); + } //---------------------------------------- - //Constructor(s) + // Constructor(s) // Simple dimension-only layout - template< class ... P > - explicit inline - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< ! Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , size_t - >::type const arg_N0 =KOKKOS_INVALID_INDEX - , const size_t arg_N1 =KOKKOS_INVALID_INDEX - , const size_t arg_N2 =KOKKOS_INVALID_INDEX - , const size_t arg_N3 =KOKKOS_INVALID_INDEX - , const size_t arg_N4 =KOKKOS_INVALID_INDEX - , const size_t arg_N5 =KOKKOS_INVALID_INDEX - , const size_t arg_N6 =KOKKOS_INVALID_INDEX - , const size_t arg_N7 =KOKKOS_INVALID_INDEX - ) - : DynRankView( arg_prop - , typename traits::array_layout - ( arg_N0 , arg_N1 , arg_N2 , arg_N3 , arg_N4 , arg_N5 , arg_N6 , arg_N7 ) - ) - {} + template + explicit inline DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + size_t>::type const arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) + : DynRankView(arg_prop, typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, + arg_N5, arg_N6, arg_N7)) {} - template< class ... P > - explicit KOKKOS_INLINE_FUNCTION - DynRankView( const Kokkos::Impl::ViewCtorProp< P ... > & arg_prop - , typename std::enable_if< Kokkos::Impl::ViewCtorProp< P... >::has_pointer - , size_t - >::type const arg_N0 =KOKKOS_INVALID_INDEX - , const size_t arg_N1 =KOKKOS_INVALID_INDEX - , const size_t arg_N2 =KOKKOS_INVALID_INDEX - , const size_t arg_N3 =KOKKOS_INVALID_INDEX - , const size_t arg_N4 =KOKKOS_INVALID_INDEX - , const size_t arg_N5 =KOKKOS_INVALID_INDEX - , const size_t arg_N6 =KOKKOS_INVALID_INDEX - , const size_t arg_N7 =KOKKOS_INVALID_INDEX - ) - : DynRankView( arg_prop - , typename traits::array_layout - ( arg_N0 , arg_N1 , arg_N2 , arg_N3 , arg_N4 , arg_N5 , arg_N6 , arg_N7 ) - ) - {} + template + explicit KOKKOS_INLINE_FUNCTION DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + typename std::enable_if::has_pointer, + size_t>::type const arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) + : DynRankView(arg_prop, typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, + arg_N5, arg_N6, arg_N7)) {} // Allocate with label and layout - template< typename Label > - explicit inline - DynRankView( const Label & arg_label - , typename std::enable_if< - Kokkos::Impl::is_view_label