From 8db88b1c02a6b9e26e2eafad986707efb06ab85a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 24 Jan 2019 13:48:08 -0500 Subject: [PATCH 01/16] Added initial untested ADIOS 2.x code with cmake building problems --- cmake/CMakeLists.txt | 48 +++- runconf | 22 ++ src/USER-ADIOS/Install.sh | 89 +++++++ src/USER-ADIOS/README | 16 ++ src/USER-ADIOS/dump_atom_adios.cpp | 314 ++++++++++++++++++++++++ src/USER-ADIOS/dump_atom_adios.h | 70 ++++++ src/USER-ADIOS/dump_custom_adios.cpp | 354 +++++++++++++++++++++++++++ src/USER-ADIOS/dump_custom_adios.h | 94 +++++++ 8 files changed, 1005 insertions(+), 2 deletions(-) create mode 100644 runconf create mode 100644 src/USER-ADIOS/Install.sh create mode 100644 src/USER-ADIOS/README create mode 100644 src/USER-ADIOS/dump_atom_adios.cpp create mode 100644 src/USER-ADIOS/dump_atom_adios.h create mode 100644 src/USER-ADIOS/dump_custom_adios.cpp create mode 100644 src/USER-ADIOS/dump_custom_adios.h diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 9ad64a59ed..5010643fd8 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -2,7 +2,22 @@ # CMake build system # This file is part of LAMMPS # Created by Christoph Junghans and Richard Berger -cmake_minimum_required(VERSION 2.8.12) +if(PKG_USER-ADIOS) + message(STATUS "Force newer standards because using ADIOS") + cmake_minimum_required(VERSION 3.6) + # Force C++11 and C99 + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED True) + # Use meta-compile features if available, otherwise use specific language + # features +# if(NOT (CMAKE_VERSION VERSION_LESS 3.9)) +# set(ADIOS2_CXX11_FEATURES cxx_std_11) +# else() +# set(ADIOS2_CXX11_FEATURES cxx_auto_type cxx_nullptr) +# endif() +else() + cmake_minimum_required(VERSION 2.8.12) +endif() project(lammps CXX) set(SOVERSION 0) @@ -178,7 +193,7 @@ set(DEFAULT_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS DIPOLE GRANULAR USER-MEAMC USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-PHONON USER-PLUMED USER-PTM USER-QTB USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY USER-UEF USER-VTK - USER-QUIP USER-QMMM USER-YAFF) + USER-QUIP USER-QMMM USER-YAFF USER-ADIOS) set(ACCEL_PACKAGES USER-OMP KOKKOS OPT USER-INTEL GPU) set(OTHER_PACKAGES CORESHELL QEQ) foreach(PKG ${DEFAULT_PACKAGES}) @@ -613,6 +628,7 @@ if(PKG_USER-NETCDF) add_definitions(-DLMP_HAS_NETCDF -DNC_64BIT_DATA=0x0020) endif() + if(PKG_USER-SMD) option(DOWNLOAD_EIGEN3 "Download Eigen3 instead of using an already installed one)" OFF) if(DOWNLOAD_EIGEN3) @@ -1354,6 +1370,34 @@ if(BUILD_EXE) endif() endif() +#if(PKG_USER-ADIOS) +# cmake_minimum_required(VERSION 3.6) +# enable_language(C) +# +# find_package(ADIOS2 REQUIRED) +## find_package(MPI REQUIRED) +# include_directories(${ADIOS2_INCLUDE_DIRS}) +## include_directories(/opt/adios2/include) +## list(APPEND LAMMPS_LINK_LIBS ${ADIOS2_LIBRARIES}) +# list(APPEND LAMMPS_LINK_LIBS ${MPI_C_LIBRARIES}) +# if(BUILD_LIB) +## target_link_libraries(lammps adios2::adios2 MPI::MPI_C) +## #target_include_directories(lammps PRIVATE ${ADIOS_INCLUDE_DIRS}) +# elseif(BUILD_EXE) +## target_link_libraries(lmp adios2::adios2 MPI::MPI_C) +# endif() +#endif(PKG_USER-ADIOS) + +if(PKG_USER-ADIOS) + enable_language(C) + find_package(ADIOS2 REQUIRED) + if(BUILD_LIB) + target_link_libraries(lammps adios2::adios2) + elseif(BUILD_EXE) + target_link_libraries(lmp adios2::adios2) + endif() +endif() + ############################################################################### # Build documentation ############################################################################### diff --git a/runconf b/runconf new file mode 100644 index 0000000000..bb313981c5 --- /dev/null +++ b/runconf @@ -0,0 +1,22 @@ + +# -D LAMMPS_SIZES=value # smallbig (default) or bigbig or smallsmall + +export HDF5_ROOT=/opt/hdf5-serial +export ADIOS2_DIR=/opt/adios2 + +cmake -D CMAKE_INSTALL_PREFIX=/opt/lammps \ + -D CMAKE_BUILD_TYPE=Debug \ + -D BUILD_MPI=yes \ + -D LAMMPS_MACHINE=adiosvm \ + -D BUILD_EXE=yes \ + -D BUILD_LIB=no \ + -D BUILD_SHARED_LIBS=no \ + -D BUILD_DOC=no \ + -D LAMMPS_SIZES=smallbig \ + -D PKG_USER-H5MD=yes \ + -D PKG_USER-ADIOS=yes \ + ../cmake + + + + diff --git a/src/USER-ADIOS/Install.sh b/src/USER-ADIOS/Install.sh new file mode 100644 index 0000000000..cae755406b --- /dev/null +++ b/src/USER-ADIOS/Install.sh @@ -0,0 +1,89 @@ +# Install/unInstall package files in LAMMPS +# mode = 0/1/2 for uninstall/install/update + +mode=$1 + +# arg1 = file, arg2 = file it depends on + +action () { + if (test $mode = 0) then + rm -f ../$1 + elif (! cmp -s $1 ../$1) then + if (test -z "$2" || test -e ../$2) then + cp $1 .. + if (test $mode = 2) then + echo " updating src/$1" + fi + fi + elif (test -n "$2") then + if (test ! -e ../$2) then + rm -f ../$1 + fi + fi +} + +for file in *.cpp *.h; do + action $file +done + +# edit 2 Makefile.package files to include/exclude package info + +if (test $1 = 1) then + +# if (test -z "$ADIOS_DIR") then +# if command -v adios_config; then +# ADIOS_DIR=`adios_config -d` +# else +# echo "ERROR: ADIOS_DIR environment variable needs to point to ADIOS" \ +# " installation directory or adios_config should be in PATH" +# fi +# fi +# ADIOS_INC=-I${ADIOS_DIR}/include +# ADIOS_LIB=`${ADIOS_DIR}/bin/adios_config -l` +# +# echo "adios_SYSINC=${ADIOS_INC} +#adios_SYSLIB=${ADIOS_LIB} +#adios_SYSPATH=${ADIOS_DIR} +#" > ../Makefile.adios + + + if (test -e ../Makefile.package) then + sed -i -e 's/[^ \t]*adios[^ \t]* //g' ../Makefile.package + sed -i -e 's/-DLMP_ADIOS //g' ../Makefile.package + sed -i -e '/^adios_SYS.*$/d' ../Makefile.package +# sed -i -e '4 i \ +#adios_SYSINC='"${ADIOS_INC}"' +#' ../Makefile.package +# sed -i -e '5 i \ +#adios_SYSLIB='"${ADIOS_LIB}"' +#' ../Makefile.package +# sed -i -e '6 i \ +#adios_SYSPATH='"${ADIOS_DIR}"' +#' ../Makefile.package + sed -i -e 's|^PKG_INC =[ \t]*|&-DLMP_ADIOS |' ../Makefile.package + sed -i -e 's|^PKG_SYSINC =[ \t]*|&$(adios_SYSINC) |' ../Makefile.package + sed -i -e 's|^PKG_SYSLIB =[ \t]*|&$(adios_SYSLIB) |' ../Makefile.package + sed -i -e 's|^PKG_SYSPATH =[ \t]*|&$(adios_SYSPATH) |' ../Makefile.package + fi + + if (test -e ../Makefile.package.settings) then + sed -i -e '/^include.*adios.*$/d' ../Makefile.package.settings + # multiline form needed for BSD sed on Macs + sed -i -e '4 i \ +include ..\/..\/lib\/adios\/Makefile.lammps +' ../Makefile.package.settings + fi + +elif (test $1 = 0) then + + if (test -e ../Makefile.package) then + sed -i -e 's/[^ \t]*adios[^ \t]* //g' ../Makefile.package + sed -i -e 's/-DLMP_ADIOS //g' ../Makefile.package + sed -i -e '/^adios_SYS.*$/d' ../Makefile.package + fi + + if (test -e ../Makefile.package.settings) then + sed -i -e '/^include.*adios.*$/d' ../Makefile.package.settings + fi + +fi diff --git a/src/USER-ADIOS/README b/src/USER-ADIOS/README new file mode 100644 index 0000000000..49717a741f --- /dev/null +++ b/src/USER-ADIOS/README @@ -0,0 +1,16 @@ +This package provides the adios dump and restart styles. + +See the doc page for the "dump adios" and "restart adios" commands. +These styles require having ADIOS 2.x itself installed on your system. + +Configure LAMMPS with CMake + a. set the environment variable + ADIOS2_DIR + to the ADIOS 2.x installation path + b. use the cmake option + -D PKG_USER-ADIOS=yes + +The person who created this package is Norbert Podhorszki (Oak Ridge National Laboratory); +If you need help, please submit a ticket at the OLCF ticket user support mentioning his name in the ticket. +https://www.olcf.ornl.gov/support/submit-ticket + diff --git a/src/USER-ADIOS/dump_atom_adios.cpp b/src/USER-ADIOS/dump_atom_adios.cpp new file mode 100644 index 0000000000..b6ace357ac --- /dev/null +++ b/src/USER-ADIOS/dump_atom_adios.cpp @@ -0,0 +1,314 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Norbert Podhorszki (ORNL) +------------------------------------------------------------------------- */ + +#include +#include "dump_atom_adios.h" +#include "domain.h" +#include "atom.h" +#include "update.h" +#include "group.h" +#include "memory.h" +#include "universe.h" +#include "error.h" + + +using namespace LAMMPS_NS; + +#define MAX_TEXT_HEADER_SIZE 4096 +#define DUMP_BUF_CHUNK_SIZE 16384 +#define DUMP_BUF_INCREMENT_SIZE 4096 + +/* ---------------------------------------------------------------------- */ + +DumpAtomADIOS::DumpAtomADIOS(LAMMPS *lmp, int narg, char **arg) : + DumpAtom(lmp, narg, arg) +{ + ad = new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); + groupSize = 0; +} + +/* ---------------------------------------------------------------------- */ + +DumpAtomADIOS::~DumpAtomADIOS() +{ + if (fh) + { + fh.Close(); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomADIOS::openfile() +{ + if (multifile) { + // if one file per timestep, replace '*' with current timestep + char *filestar = strdup(filename); + char *filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + fh = io.Open(filecurrent, adios2::Mode::Write, world); + if (!fh) { + char str[128]; + sprintf(str,"Cannot open dump file %s",filecurrent); + error->one(FLERR,str); + } + free(filestar); + delete [] filecurrent; + } + else + { + if (!singlefile_opened) + { + fh = io.Open(filename, adios2::Mode::Write, world); + if (!fh) { + char str[128]; + sprintf(str,"Cannot open dump file %s",filename); + error->one(FLERR,str); + } + singlefile_opened = 1; + } + + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomADIOS::write() +{ + if (domain->triclinic == 0) { + boxxlo = domain->boxlo[0]; + boxxhi = domain->boxhi[0]; + boxylo = domain->boxlo[1]; + boxyhi = domain->boxhi[1]; + boxzlo = domain->boxlo[2]; + boxzhi = domain->boxhi[2]; + } else { + boxxlo = domain->boxlo_bound[0]; + boxxhi = domain->boxhi_bound[0]; + boxylo = domain->boxlo_bound[1]; + boxyhi = domain->boxhi_bound[1]; + boxzlo = domain->boxlo_bound[2]; + boxzhi = domain->boxhi_bound[2]; + boxxy = domain->xy; + boxxz = domain->xz; + boxyz = domain->yz; + } + + // nme = # of dump lines this proc contributes to dump + + nme = count(); + + // ntotal = total # of atoms in snapshot + // atomOffset = sum of # of atoms up to this proc (exclusive prefix sum) + + bigint bnme = nme; + MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); + + bigint atomOffset; // sum of all atoms on processes 0..me-1 + MPI_Scan (&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); + atomOffset -= nme; // exclusive prefix sum needed + + // Now we know the global size and the local subset size and offset + // of the atoms table + size_t nAtomsGlobal = static_cast(ntotal); + size_t startRow = static_cast(atomOffset); + size_t nAtomsLocal = static_cast(nme); + size_t nColumns = static_cast(size_one); + varAtoms.SetShape({nAtomsGlobal,nColumns}); + varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal,nColumns}}); + + // insure buf is sized for packing + // adios does not limit per-process data size so nme*size_one is not constrained to int + // if sorting on IDs also request ID list from pack() + // sort buf as needed + + if (nme > maxbuf) { + maxbuf = nme; + memory->destroy(buf); + memory->create(buf,(maxbuf*size_one),"dump:buf"); + } + if (sort_flag && sortcol == 0 && nme > maxids) { + maxids = nme; + memory->destroy(ids); + memory->create(ids,maxids,"dump:ids"); + } + + if (sort_flag && sortcol == 0) pack(ids); + else pack(NULL); + if (sort_flag) sort(); + + // Calculate data size written by this process + groupSize = nme * size_one * sizeof(double); // size of atoms data on this process + groupSize += 3*sizeof(uint64_t) + 1*sizeof(int); // scalars written by each process + if (me == 0) { + groupSize += 1*sizeof(uint64_t) + 1*sizeof(int) + 6*sizeof(double); // scalars + if (domain->triclinic) { + groupSize += 3*sizeof(double); // boxxy, boxxz, boxyz + } + } + + openfile(); + fh.BeginStep(); + // write info on data as scalars (by me==0) + if (me == 0) { + fh.Put("ntimestep", update->ntimestep); + fh.Put("nprocs", nprocs); + + fh.Put("boxxlo", boxxlo); + fh.Put("boxxhi", boxxhi); + fh.Put("boxylo", boxylo); + fh.Put("boxyhi", boxyhi); + fh.Put("boxzlo", boxzlo); + fh.Put("boxzhi", boxzhi); + + if (domain->triclinic) { + fh.Put("boxxy", boxxy); + fh.Put("boxxz", boxxz); + fh.Put("boxyz", boxyz); + } + } + // Everyone needs to write scalar variables that are used as dimensions and offsets of arrays + fh.Put("natoms", ntotal); + fh.Put("ncolumns", size_one); + fh.Put("nme", bnme); + fh.Put("offset", atomOffset); + // now write the atoms + fh.Put("atoms", buf); + fh.EndStep();// I/O will happen now... + + if (multifile) + { + fh.Close(); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomADIOS::init_style() +{ + if (image_flag == 0) size_one = 5; + else size_one = 8; + + // setup boundary string + + domain->boundary_string(boundstr); + + // remove % from filename since ADIOS always writes a global file with data/metadata + int len = strlen(filename); + char *ptr = strchr(filename,'%'); + if (ptr) { + *ptr = '\0'; + char *s = new char[len-1]; + sprintf(s,"%s%s",filename,ptr+1); + strncpy(filename,s,len); + } + + // setup column string + + if (scale_flag == 0 && image_flag == 0) + columns = (char *) "id type x y z"; + else if (scale_flag == 0 && image_flag == 1) + columns = (char *) "id type x y z ix iy iz"; + else if (scale_flag == 1 && image_flag == 0) + columns = (char *) "id type xs ys zs"; + else if (scale_flag == 1 && image_flag == 1) + columns = (char *) "id type xs ys zs ix iy iz"; + + // setup function ptrs + + if (scale_flag == 1 && image_flag == 0 && domain->triclinic == 0) + pack_choice = &DumpAtomADIOS::pack_scale_noimage; + else if (scale_flag == 1 && image_flag == 1 && domain->triclinic == 0) + pack_choice = &DumpAtomADIOS::pack_scale_image; + else if (scale_flag == 1 && image_flag == 0 && domain->triclinic == 1) + pack_choice = &DumpAtomADIOS::pack_scale_noimage_triclinic; + else if (scale_flag == 1 && image_flag == 1 && domain->triclinic == 1) + pack_choice = &DumpAtomADIOS::pack_scale_image_triclinic; + else if (scale_flag == 0 && image_flag == 0) + pack_choice = &DumpAtomADIOS::pack_noscale_noimage; + else if (scale_flag == 0 && image_flag == 1) + pack_choice = &DumpAtomADIOS::pack_noscale_image; + + /* Define the group of variables for the atom style here since it's a fixed set */ + adios2::IO io = ad->DeclareIO(ioName); + if (!io.InConfigFile()) + { + // if not defined by user, we can change the default settings + // BPFile is the default writer + io.SetEngine("BPFile"); + int num_aggregators = multiproc; + if (num_aggregators == 0) + num_aggregators = 1; + char nstreams[128]; + sprintf (nstreams, "%d", num_aggregators); + io.SetParameters({{"substreams", nstreams}}); + if (me==0 && screen) fprintf(screen, "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", filename, nstreams); + } + + + io.DefineVariable("ntimestep"); + io.DefineVariable("natoms"); + + io.DefineVariable("nprocs"); + io.DefineVariable("ncolumns"); + + io.DefineVariable("boxxlo"); + io.DefineVariable("boxxhi"); + io.DefineVariable("boxylo"); + io.DefineVariable("boxyhi"); + io.DefineVariable("boxzlo"); + io.DefineVariable("boxzhi"); + + io.DefineVariable("boxxy"); + io.DefineVariable("boxxz"); + io.DefineVariable("boxyz"); + + io.DefineAttribute("triclinic", domain->triclinic); + io.DefineAttribute("scaled", scale_flag); + io.DefineAttribute("image", image_flag); + + int *boundaryptr = reinterpret_cast(domain->boundary); + io.DefineAttribute("boundary", boundaryptr, 6); + + io.DefineAttribute("columns", columns); + io.DefineAttribute("boundarystr", boundstr); + io.DefineAttribute("LAMMPS/dump_style", "atom"); + io.DefineAttribute("LAMMPS/version", universe->version); + io.DefineAttribute("LAMMPS/num_ver", universe->num_ver); + + io.DefineVariable("nme", {adios2::LocalValueDim}); // local dimension variable + io.DefineVariable("offset", {adios2::LocalValueDim}); // local dimension variable + + // atom table size is not known at the moment + // it will be correctly defined at the moment of write + size_t UnknownSizeYet = 1; + size_t nColumns = static_cast(size_one); + varAtoms = io.DefineVariable("atoms", + {UnknownSizeYet,nColumns}, + {UnknownSizeYet, 0}, + {UnknownSizeYet,nColumns}); +} diff --git a/src/USER-ADIOS/dump_atom_adios.h b/src/USER-ADIOS/dump_atom_adios.h new file mode 100644 index 0000000000..a3999bea76 --- /dev/null +++ b/src/USER-ADIOS/dump_atom_adios.h @@ -0,0 +1,70 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(atom/adios,DumpAtomADIOS) + +#else + +#ifndef LMP_DUMP_ATOM_ADIOS_H +#define LMP_DUMP_ATOM_ADIOS_H + +#include "dump_atom.h" +#include +#include +#include "adios2.h" + +namespace LAMMPS_NS { + +class DumpAtomADIOS : public DumpAtom { + + public: + DumpAtomADIOS(class LAMMPS *, int, char **); + virtual ~DumpAtomADIOS(); + + protected: + + const std::string ioName="atom"; // name of adios group, referrable in adios2_config.xml + adios2::ADIOS *ad = nullptr; // adios object + adios2::IO io; // adios group of variables and attributes in this dump + adios2::Engine fh; // adios file/stream handle object + adios2::Variable varAtoms; // one ADIOS output variable we need to change + uint64_t groupSize; // pre-calculate # of bytes written per processor in a step before writing anything + uint64_t groupTotalSize; // ADIOS buffer size returned by adios_group_size(), valid only if size is > default 16MB ADIOS buffer + std::string filecurrent; // name of file for this round (with % and * replaced) + + virtual void openfile(); + virtual void write(); + virtual void init_style(); + +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Cannot open dump file %s + +The output file for the dump command cannot be opened. Check that the +path and name are correct. + +E: Too much per-proc info for dump + +Number of local atoms times number of columns must fit in a 32-bit +integer for dump. + +*/ diff --git a/src/USER-ADIOS/dump_custom_adios.cpp b/src/USER-ADIOS/dump_custom_adios.cpp new file mode 100644 index 0000000000..738e6dc076 --- /dev/null +++ b/src/USER-ADIOS/dump_custom_adios.cpp @@ -0,0 +1,354 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Paul Coffman (IBM) +------------------------------------------------------------------------- */ + +#include +#include +#include +#include "dump_custom_adios.h" +#include "atom.h" +#include "force.h" +#include "domain.h" +#include "region.h" +#include "group.h" +#include "input.h" +#include "variable.h" +#include "update.h" +#include "modify.h" +#include "compute.h" +#include "fix.h" +#include "universe.h" +#include "memory.h" +#include "error.h" +#include + +using namespace LAMMPS_NS; + +#define MAX_TEXT_HEADER_SIZE 4096 +#define DUMP_BUF_CHUNK_SIZE 16384 +#define DUMP_BUF_INCREMENT_SIZE 4096 + +enum{ID,MOL,TYPE,ELEMENT,MASS, + X,Y,Z,XS,YS,ZS,XSTRI,YSTRI,ZSTRI,XU,YU,ZU,XUTRI,YUTRI,ZUTRI, + XSU,YSU,ZSU,XSUTRI,YSUTRI,ZSUTRI, + IX,IY,IZ, + VX,VY,VZ,FX,FY,FZ, + Q,MUX,MUY,MUZ,MU,RADIUS,DIAMETER, + OMEGAX,OMEGAY,OMEGAZ,ANGMOMX,ANGMOMY,ANGMOMZ, + TQX,TQY,TQZ,SPIN,ERADIUS,ERVEL,ERFORCE, + COMPUTE,FIX,VARIABLE}; +enum{LT,LE,GT,GE,EQ,NEQ}; +enum{INT,DOUBLE,STRING,BIGINT}; // same as in DumpCustom + +/* ---------------------------------------------------------------------- */ + +DumpCustomADIOS::DumpCustomADIOS(LAMMPS *lmp, int narg, char **arg) : + DumpCustom(lmp, narg, arg) +{ + ad = new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); + groupsize = 0; + //if (screen) fprintf(screen, "DumpCustomADIOS constructor: nvariable=%d id_variable=%p, variables=%p, nfield=%d, earg=%p\n", nvariable, id_variable, variable, nfield, earg); + columnNames.reserve(nfield); + for (int i = 0; i < nfield; ++i) { + columnNames[i]=std::string(earg[i]); + //if (screen) fprintf(screen, "earg[%d] = '%s'\n", i, earg[i]); + } +} + +/* ---------------------------------------------------------------------- */ + +DumpCustomADIOS::~DumpCustomADIOS() +{ + columnNames.clear(); + if (fh) + { + fh.Close(); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomADIOS::openfile() +{ + if (multifile) { + // if one file per timestep, replace '*' with current timestep + char *filestar = strdup(filename); + char *filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + fh = io.Open(filecurrent, adios2::Mode::Write, world); + if (!fh) { + char str[128]; + sprintf(str,"Cannot open dump file %s",filecurrent); + error->one(FLERR,str); + } + free(filestar); + delete [] filecurrent; + } + else + { + if (!singlefile_opened) + { + fh = io.Open(filename, adios2::Mode::Write, world); + if (!fh) { + char str[128]; + sprintf(str,"Cannot open dump file %s",filename); + error->one(FLERR,str); + } + singlefile_opened = 1; + } + + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomADIOS::write() +{ + if (domain->triclinic == 0) { + boxxlo = domain->boxlo[0]; + boxxhi = domain->boxhi[0]; + boxylo = domain->boxlo[1]; + boxyhi = domain->boxhi[1]; + boxzlo = domain->boxlo[2]; + boxzhi = domain->boxhi[2]; + } else { + boxxlo = domain->boxlo_bound[0]; + boxxhi = domain->boxhi_bound[0]; + boxylo = domain->boxlo_bound[1]; + boxyhi = domain->boxhi_bound[1]; + boxzlo = domain->boxlo_bound[2]; + boxzhi = domain->boxhi_bound[2]; + boxxy = domain->xy; + boxxz = domain->xz; + boxyz = domain->yz; + } + + // nme = # of dump lines this proc contributes to dump + + nme = count(); + + // ntotal = total # of atoms in snapshot + // atomOffset = sum of # of atoms up to this proc (exclusive prefix sum) + + bigint bnme = nme; + MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); + + bigint atomOffset; // sum of all atoms on processes 0..me-1 + MPI_Scan (&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); + atomOffset -= nme; // exclusive prefix sum needed + + // Now we know the global size and the local subset size and offset + // of the atoms table + size_t nAtomsGlobal = static_cast(ntotal); + size_t startRow = static_cast(atomOffset); + size_t nAtomsLocal = static_cast(nme); + size_t nColumns = static_cast(size_one); + varAtoms.SetShape({nAtomsGlobal,nColumns}); + varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal,nColumns}}); + + // insure filewriter proc can receive everyone's info + // limit nmax*size_one to int since used as arg in MPI_Rsend() below + // pack my data into buf + // if sorting on IDs also request ID list from pack() + // sort buf as needed + + if (nme > maxbuf) { + if ((bigint) nme * size_one > MAXSMALLINT) + error->all(FLERR,"Too much per-proc info for dump"); + maxbuf = nme; + memory->destroy(buf); + memory->create(buf,(maxbuf*size_one),"dump:buf"); + } + if (sort_flag && sortcol == 0 && nme > maxids) { + maxids = nme; + memory->destroy(ids); + memory->create(ids,maxids,"dump:ids"); + } + + if (sort_flag && sortcol == 0) pack(ids); + else pack(NULL); + if (sort_flag) sort(); + + // Calculate data size written by this process + groupsize = nme * size_one * sizeof(double); // size of atoms data on this process + groupsize += 3*sizeof(uint64_t) + 1*sizeof(int); // scalars written by each process + if (me == 0) { + groupsize += 1*sizeof(uint64_t) + 1*sizeof(int) + 6*sizeof(double); // scalars + if (domain->triclinic) { + groupsize += 3*sizeof(double); // boxxy, boxxz, boxyz + } + } + + openfile(); + fh.BeginStep(); + // write info on data as scalars (by me==0) + if (me == 0) { + fh.Put("ntimestep", update->ntimestep); + fh.Put("nprocs", nprocs); + + fh.Put("boxxlo", boxxlo); + fh.Put("boxxhi", boxxhi); + fh.Put("boxylo", boxylo); + fh.Put("boxyhi", boxyhi); + fh.Put("boxzlo", boxzlo); + fh.Put("boxzhi", boxzhi); + + if (domain->triclinic) { + fh.Put("boxxy", boxxy); + fh.Put("boxxz", boxxz); + fh.Put("boxyz", boxyz); + } + } + // Everyone needs to write scalar variables that are used as dimensions and offsets of arrays + fh.Put("natoms", ntotal); + fh.Put("ncolumns", size_one); + fh.Put("nme", bnme); + fh.Put("offset", atomOffset); + // now write the atoms + fh.Put("atoms", buf); + fh.EndStep();// I/O will happen now... + + if (multifile) + { + fh.Close(); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomADIOS::init_style() +{ + + // setup boundary string + + domain->boundary_string(boundstr); + + // remove % from filename since ADIOS always writes a global file with data/metadata + int len = strlen(filename); + char *ptr = strchr(filename,'%'); + if (ptr) { + *ptr = '\0'; + char *s = new char[len-1]; + sprintf(s,"%s%s",filename,ptr+1); + strncpy(filename,s,len); + } + + /* The next four loops are copied from dump_custom_mpiio, but nothing is done with them. + * It is unclear why we need them here. + * For metadata, variable[] will be written out as an ADIOS attribute if nvariable>0 + */ + // find current ptr for each compute,fix,variable + // check that fix frequency is acceptable + int icompute; + for (int i = 0; i < ncompute; i++) { + icompute = modify->find_compute(id_compute[i]); + if (icompute < 0) error->all(FLERR,"Could not find dump custom compute ID"); + compute[i] = modify->compute[icompute]; + } + + int ifix; + for (int i = 0; i < nfix; i++) { + ifix = modify->find_fix(id_fix[i]); + if (ifix < 0) error->all(FLERR,"Could not find dump custom fix ID"); + fix[i] = modify->fix[ifix]; + if (nevery % modify->fix[ifix]->peratom_freq) + error->all(FLERR,"Dump custom and fix not computed at compatible times"); + } + + int ivariable; + for (int i = 0; i < nvariable; i++) { + ivariable = input->variable->find(id_variable[i]); + if (ivariable < 0) + error->all(FLERR,"Could not find dump custom variable name"); + variable[i] = ivariable; + } + + // set index and check validity of region + if (iregion >= 0) { + iregion = domain->find_region(idregion); + if (iregion == -1) + error->all(FLERR,"Region ID for dump custom does not exist"); + } + + /* Define the group of variables for the atom style here since it's a fixed set */ + adios2::IO io = ad->DeclareIO(ioName); + if (!io.InConfigFile()) + { + // if not defined by user, we can change the default settings + // BPFile is the default writer + io.SetEngine("BPFile"); + int num_aggregators = multiproc; + if (num_aggregators == 0) + num_aggregators = 1; + char nstreams[128]; + sprintf (nstreams, "%d", num_aggregators); + io.SetParameters({{"substreams", nstreams}}); + if (me==0 && screen) fprintf(screen, "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", filename, nstreams); + } + + + io.DefineVariable("ntimestep"); + io.DefineVariable("natoms"); + + io.DefineVariable("nprocs"); + io.DefineVariable("ncolumns"); + + io.DefineVariable("boxxlo"); + io.DefineVariable("boxxhi"); + io.DefineVariable("boxylo"); + io.DefineVariable("boxyhi"); + io.DefineVariable("boxzlo"); + io.DefineVariable("boxzhi"); + + io.DefineVariable("boxxy"); + io.DefineVariable("boxxz"); + io.DefineVariable("boxyz"); + + io.DefineAttribute("triclinic", domain->triclinic); + + int *boundaryptr = reinterpret_cast(domain->boundary); + io.DefineAttribute("boundary", boundaryptr, 6); + + size_t nColumns = static_cast(size_one); + io.DefineAttribute("columns", columnNames.data(), nColumns); + io.DefineAttribute("columnstr", columns); + io.DefineAttribute("boundarystr", boundstr); + io.DefineAttribute("LAMMPS/dump_style", "atom"); + io.DefineAttribute("LAMMPS/version", universe->version); + io.DefineAttribute("LAMMPS/num_ver", universe->num_ver); + + io.DefineVariable("nme", {adios2::LocalValueDim}); // local dimension variable + io.DefineVariable("offset", {adios2::LocalValueDim}); // local dimension variable + + // atom table size is not known at the moment + // it will be correctly defined at the moment of write + size_t UnknownSizeYet = 1; + varAtoms = io.DefineVariable("atoms", + {UnknownSizeYet,nColumns}, + {UnknownSizeYet, 0}, + {UnknownSizeYet,nColumns}); + +} diff --git a/src/USER-ADIOS/dump_custom_adios.h b/src/USER-ADIOS/dump_custom_adios.h new file mode 100644 index 0000000000..2461e20b2e --- /dev/null +++ b/src/USER-ADIOS/dump_custom_adios.h @@ -0,0 +1,94 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(custom/adios,DumpCustomADIOS) + +#else + +#ifndef LMP_DUMP_CUSTOM_ADIOS_H +#define LMP_DUMP_CUSTOM_ADIOS_H + +#include "dump_custom.h" +#include +#include +#include "adios2.h" + +namespace LAMMPS_NS { + +class DumpCustomADIOS : public DumpCustom { + public: + DumpCustomADIOS(class LAMMPS *, int, char **); + virtual ~DumpCustomADIOS(); + + protected: + + const std::string ioName="custom"; // name of adios group, referrable in adios2_config.xml + adios2::ADIOS *ad = nullptr; // adios object + adios2::IO io; // adios group of variables and attributes in this dump + adios2::Engine fh; // adios file/stream handle object + adios2::Variable varAtoms; // one ADIOS output variable we need to change + uint64_t groupsize; // pre-calculate # of bytes written per processor in a step before writing anything + std::vectorcolumnNames; // list of column names for the atom table (individual list of 'columns' string) + + virtual void openfile(); + virtual void write(); + virtual void init_style(); + +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Cannot open dump file %s + +The output file for the dump command cannot be opened. Check that the +path and name are correct. + +E: Too much per-proc info for dump + +Number of local atoms times number of columns must fit in a 32-bit +integer for dump. + +E: Dump_modify format string is too short + +There are more fields to be dumped in a line of output than your +format string specifies. + +E: Could not find dump custom compute ID + +Self-explanatory. + +E: Could not find dump custom fix ID + +Self-explanatory. + +E: Dump custom and fix not computed at compatible times + +The fix must produce per-atom quantities on timesteps that dump custom +needs them. + +E: Could not find dump custom variable name + +Self-explanatory. + +E: Region ID for dump custom does not exist + +Self-explanatory. + +*/ From 24234d3a0bc43ba1ba22800c0fb9fafd24464309 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 24 Jan 2019 14:14:50 -0500 Subject: [PATCH 02/16] Fixed cmake build for ADIOS 2.x --- cmake/CMakeLists.txt | 38 +++++++++++--------------------------- runconf | 2 +- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5010643fd8..bf2a176d58 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -216,6 +216,17 @@ endif() include_directories(${LAMMPS_SOURCE_DIR}) + +if(PKG_USER-ADIOS) + # The search for ADIOS2 must come before MPI because + # it includes its own MPI search with the latest FindMPI.cmake + # script that defines the MPI::MPI_C target + enable_language(C) + find_package(ADIOS2 REQUIRED) + list(APPEND LAMMPS_LINK_LIBS adios2::adios2) +endif() + + # do MPI detection after language activation, if MPI for these language is required find_package(MPI QUIET) option(BUILD_MPI "Build MPI version" ${MPI_FOUND}) @@ -1370,33 +1381,6 @@ if(BUILD_EXE) endif() endif() -#if(PKG_USER-ADIOS) -# cmake_minimum_required(VERSION 3.6) -# enable_language(C) -# -# find_package(ADIOS2 REQUIRED) -## find_package(MPI REQUIRED) -# include_directories(${ADIOS2_INCLUDE_DIRS}) -## include_directories(/opt/adios2/include) -## list(APPEND LAMMPS_LINK_LIBS ${ADIOS2_LIBRARIES}) -# list(APPEND LAMMPS_LINK_LIBS ${MPI_C_LIBRARIES}) -# if(BUILD_LIB) -## target_link_libraries(lammps adios2::adios2 MPI::MPI_C) -## #target_include_directories(lammps PRIVATE ${ADIOS_INCLUDE_DIRS}) -# elseif(BUILD_EXE) -## target_link_libraries(lmp adios2::adios2 MPI::MPI_C) -# endif() -#endif(PKG_USER-ADIOS) - -if(PKG_USER-ADIOS) - enable_language(C) - find_package(ADIOS2 REQUIRED) - if(BUILD_LIB) - target_link_libraries(lammps adios2::adios2) - elseif(BUILD_EXE) - target_link_libraries(lmp adios2::adios2) - endif() -endif() ############################################################################### # Build documentation diff --git a/runconf b/runconf index bb313981c5..18f05f38a5 100644 --- a/runconf +++ b/runconf @@ -1,7 +1,7 @@ # -D LAMMPS_SIZES=value # smallbig (default) or bigbig or smallsmall -export HDF5_ROOT=/opt/hdf5-serial +export HDF5_ROOT=/opt/hdf5-parallel export ADIOS2_DIR=/opt/adios2 cmake -D CMAKE_INSTALL_PREFIX=/opt/lammps \ From 200fed9d4c484318002779e728d33ad234d7cb55 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 24 Jan 2019 15:00:32 -0500 Subject: [PATCH 03/16] Fix basic errors in USER-ADIOS code --- src/USER-ADIOS/dump_atom_adios.cpp | 2 +- src/USER-ADIOS/dump_custom_adios.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/USER-ADIOS/dump_atom_adios.cpp b/src/USER-ADIOS/dump_atom_adios.cpp index b6ace357ac..e7a5f05e9d 100644 --- a/src/USER-ADIOS/dump_atom_adios.cpp +++ b/src/USER-ADIOS/dump_atom_adios.cpp @@ -254,7 +254,7 @@ void DumpAtomADIOS::init_style() pack_choice = &DumpAtomADIOS::pack_noscale_image; /* Define the group of variables for the atom style here since it's a fixed set */ - adios2::IO io = ad->DeclareIO(ioName); + io = ad->DeclareIO(ioName); if (!io.InConfigFile()) { // if not defined by user, we can change the default settings diff --git a/src/USER-ADIOS/dump_custom_adios.cpp b/src/USER-ADIOS/dump_custom_adios.cpp index 738e6dc076..5f54b987b1 100644 --- a/src/USER-ADIOS/dump_custom_adios.cpp +++ b/src/USER-ADIOS/dump_custom_adios.cpp @@ -63,7 +63,7 @@ DumpCustomADIOS::DumpCustomADIOS(LAMMPS *lmp, int narg, char **arg) : //if (screen) fprintf(screen, "DumpCustomADIOS constructor: nvariable=%d id_variable=%p, variables=%p, nfield=%d, earg=%p\n", nvariable, id_variable, variable, nfield, earg); columnNames.reserve(nfield); for (int i = 0; i < nfield; ++i) { - columnNames[i]=std::string(earg[i]); + columnNames.push_back(earg[i]); //if (screen) fprintf(screen, "earg[%d] = '%s'\n", i, earg[i]); } } @@ -294,7 +294,7 @@ void DumpCustomADIOS::init_style() } /* Define the group of variables for the atom style here since it's a fixed set */ - adios2::IO io = ad->DeclareIO(ioName); + io = ad->DeclareIO(ioName); if (!io.InConfigFile()) { // if not defined by user, we can change the default settings From 410d79d8b7d8ce4f01db038e10bd2a043a8dcdfd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 24 Jan 2019 15:01:15 -0500 Subject: [PATCH 04/16] Add an example for USER/adios based on the balance example --- examples/USER/adios/adios2_config.xml | 45 ++++++++++ examples/USER/adios/in.adios_balance | 56 +++++++++++++ examples/USER/adios/log.balance | 114 ++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) create mode 100644 examples/USER/adios/adios2_config.xml create mode 100644 examples/USER/adios/in.adios_balance create mode 100644 examples/USER/adios/log.balance diff --git a/examples/USER/adios/adios2_config.xml b/examples/USER/adios/adios2_config.xml new file mode 100644 index 0000000000..5e1908eb52 --- /dev/null +++ b/examples/USER/adios/adios2_config.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/USER/adios/in.adios_balance b/examples/USER/adios/in.adios_balance new file mode 100644 index 0000000000..b44870afbb --- /dev/null +++ b/examples/USER/adios/in.adios_balance @@ -0,0 +1,56 @@ +# 2d circle of particles inside a box with LJ walls + +variable b index 0 + +variable x index 50 +variable y index 20 +variable d index 20 +variable v index 5 +variable w index 2 + +units lj +dimension 2 +atom_style atomic +boundary f f p + +lattice hex 0.85 +region box block 0 $x 0 $y -0.5 0.5 +create_box 1 box +region circle sphere $(v_d/2+1) $(v_d/2/sqrt(3.0)+1) 0.0 $(v_d/2) +create_atoms 1 region circle +mass 1 1.0 + +velocity all create 0.5 87287 loop geom +velocity all set $v $w 0 sum yes + +pair_style lj/cut 2.5 +pair_coeff 1 1 10.0 1.0 2.5 + +neighbor 0.3 bin +neigh_modify delay 0 every 1 check yes + +fix 1 all nve + +fix 2 all wall/lj93 xlo 0.0 1 1 2.5 xhi $x 1 1 2.5 +fix 3 all wall/lj93 ylo 0.0 1 1 2.5 yhi $y 1 1 2.5 + +comm_style tiled +fix 10 all balance 50 0.9 rcb + +compute 1 all property/atom proc +variable p atom c_1%10 +dump 2 all custom 50 balance.dump id v_p x y z +dump 3 all custom/adios 50 balance_custom.bp id v_p x y z +dump 4 all atom/adios 50 balance_atom.bp + +#dump 3 all image 50 image.*.jpg v_p type & +# adiam 1.0 view 0 0 zoom 1.8 subbox yes 0.02 +#variable colors string & +# "red green blue yellow white & +# purple pink orange lime gray" +#dump_modify 3 pad 5 amap 0 10 sa 1 10 ${colors} + +thermo_style custom step temp epair press f_10[3] f_10 +thermo 100 + +run 200 diff --git a/examples/USER/adios/log.balance b/examples/USER/adios/log.balance new file mode 100644 index 0000000000..162ecf7741 --- /dev/null +++ b/examples/USER/adios/log.balance @@ -0,0 +1,114 @@ +LAMMPS (4 Jan 2019) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:87) + using 1 OpenMP thread(s) per MPI task +# 2d circle of particles inside a box with LJ walls + +variable b index 0 + +variable x index 50 +variable y index 20 +variable d index 20 +variable v index 5 +variable w index 2 + +units lj +dimension 2 +atom_style atomic +boundary f f p + +lattice hex 0.85 +Lattice spacing in x,y,z = 1.16553 2.01877 1.16553 +region box block 0 $x 0 $y -0.5 0.5 +region box block 0 50 0 $y -0.5 0.5 +region box block 0 50 0 20 -0.5 0.5 +create_box 1 box +Created orthogonal box = (0 0 -0.582767) to (58.2767 40.3753 0.582767) + 2 by 2 by 1 MPI processor grid +region circle sphere $(v_d/2+1) $(v_d/2/sqrt(3.0)+1) 0.0 $(v_d/2) +region circle sphere 11 $(v_d/2/sqrt(3.0)+1) 0.0 $(v_d/2) +region circle sphere 11 6.7735026918962581988 0.0 $(v_d/2) +region circle sphere 11 6.7735026918962581988 0.0 10 +create_atoms 1 region circle +Created 361 atoms + Time spent = 0.00171804 secs +mass 1 1.0 + +velocity all create 0.5 87287 loop geom +velocity all set $v $w 0 sum yes +velocity all set 5 $w 0 sum yes +velocity all set 5 2 0 sum yes + +pair_style lj/cut 2.5 +pair_coeff 1 1 10.0 1.0 2.5 + +neighbor 0.3 bin +neigh_modify delay 0 every 1 check yes + +fix 1 all nve + +fix 2 all wall/lj93 xlo 0.0 1 1 2.5 xhi $x 1 1 2.5 +fix 2 all wall/lj93 xlo 0.0 1 1 2.5 xhi 50 1 1 2.5 +fix 3 all wall/lj93 ylo 0.0 1 1 2.5 yhi $y 1 1 2.5 +fix 3 all wall/lj93 ylo 0.0 1 1 2.5 yhi 20 1 1 2.5 + +comm_style tiled +fix 10 all balance 50 0.9 rcb + +compute 1 all property/atom proc +variable p atom c_1%10 +dump 2 all custom 50 balance.dump id v_p x y z +dump 3 all custom/adios 50 balance_custom.bp id v_p x y z +dump 4 all atom/adios 50 balance_atom.bp + +#dump 3 all image 50 image.*.jpg v_p type # adiam 1.0 view 0 0 zoom 1.8 subbox yes 0.02 +#variable colors string # "red green blue yellow white # purple pink orange lime gray" +#dump_modify 3 pad 5 amap 0 10 sa 1 10 ${colors} + +thermo_style custom step temp epair press f_10[3] f_10 +thermo 100 + +run 200 +Neighbor list info ... + update every 1 steps, delay 0 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.8 + ghost atom cutoff = 2.8 + binsize = 1.4, bins = 42 29 1 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/2d/newton + bin: standard +Per MPI rank memory allocation (min/avg/max) = 4.926 | 4.933 | 4.944 Mbytes +Step Temp E_pair Press f_10[3] f_10 + 0 25.701528 -29.143179 -1.2407285 3.2354571 1.0526316 + 100 26.269576 -29.713313 7.9052334 1.2742382 1.0304709 + 200 26.368336 -29.809962 1.6412462 1.2520776 1.0083102 +Loop time of 0.0992351 on 4 procs for 200 steps with 361 atoms + +Performance: 870660.046 tau/day, 2015.417 timesteps/s +32.2% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.0078368 | 0.0081607 | 0.0085468 | 0.3 | 8.22 +Neigh | 0.002804 | 0.0045915 | 0.0092173 | 3.9 | 4.63 +Comm | 0.044407 | 0.05352 | 0.062051 | 3.0 | 53.93 +Output | 0.011406 | 0.012025 | 0.01342 | 0.7 | 12.12 +Modify | 0.006305 | 0.0064294 | 0.0066617 | 0.2 | 6.48 +Other | | 0.01451 | | | 14.62 + +Nlocal: 90.25 ave 91 max 90 min +Histogram: 3 0 0 0 0 0 0 0 0 1 +Nghost: 58.25 ave 64 max 51 min +Histogram: 1 0 0 0 0 0 2 0 0 1 +Neighs: 730.75 ave 801 max 671 min +Histogram: 1 0 1 0 0 1 0 0 0 1 + +Total # of neighbors = 2923 +Ave neighs/atom = 8.09695 +Neighbor list builds = 60 +Dangerous builds = 0 +Total wall time: 0:00:00 From a4b9542ba95581a299c421e8d0b20545af06df68 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 6 Feb 2019 14:32:01 -0500 Subject: [PATCH 05/16] remove personal runconf file. cleanup ADIOS2 xml config file for example --- examples/USER/adios/adios2_config.xml | 17 ++--------------- runconf | 22 ---------------------- 2 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 runconf diff --git a/examples/USER/adios/adios2_config.xml b/examples/USER/adios/adios2_config.xml index 5e1908eb52..7462731d43 100644 --- a/examples/USER/adios/adios2_config.xml +++ b/examples/USER/adios/adios2_config.xml @@ -4,25 +4,12 @@ - - diff --git a/runconf b/runconf deleted file mode 100644 index 18f05f38a5..0000000000 --- a/runconf +++ /dev/null @@ -1,22 +0,0 @@ - -# -D LAMMPS_SIZES=value # smallbig (default) or bigbig or smallsmall - -export HDF5_ROOT=/opt/hdf5-parallel -export ADIOS2_DIR=/opt/adios2 - -cmake -D CMAKE_INSTALL_PREFIX=/opt/lammps \ - -D CMAKE_BUILD_TYPE=Debug \ - -D BUILD_MPI=yes \ - -D LAMMPS_MACHINE=adiosvm \ - -D BUILD_EXE=yes \ - -D BUILD_LIB=no \ - -D BUILD_SHARED_LIBS=no \ - -D BUILD_DOC=no \ - -D LAMMPS_SIZES=smallbig \ - -D PKG_USER-H5MD=yes \ - -D PKG_USER-ADIOS=yes \ - ../cmake - - - - From 9c52a3754681686500412345c4dad983745cc19e Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 7 Feb 2019 16:38:36 -0500 Subject: [PATCH 06/16] PIMPL implementation of USER-ADIOS to hide internal data from style header file --- cmake/CMakeLists.txt | 9 +- src/USER-ADIOS/dump_atom_adios.cpp | 275 ++++++++++--------- src/USER-ADIOS/dump_atom_adios.h | 58 ++-- src/USER-ADIOS/dump_custom_adios.cpp | 384 ++++++++++++++++----------- src/USER-ADIOS/dump_custom_adios.h | 85 +++--- 5 files changed, 447 insertions(+), 364 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index bf2a176d58..e5d362fe52 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -3,18 +3,11 @@ # This file is part of LAMMPS # Created by Christoph Junghans and Richard Berger if(PKG_USER-ADIOS) - message(STATUS "Force newer standards because using ADIOS") + message(STATUS "Force newer standards because of using ADIOS") cmake_minimum_required(VERSION 3.6) # Force C++11 and C99 set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) - # Use meta-compile features if available, otherwise use specific language - # features -# if(NOT (CMAKE_VERSION VERSION_LESS 3.9)) -# set(ADIOS2_CXX11_FEATURES cxx_std_11) -# else() -# set(ADIOS2_CXX11_FEATURES cxx_auto_type cxx_nullptr) -# endif() else() cmake_minimum_required(VERSION 2.8.12) endif() diff --git a/src/USER-ADIOS/dump_atom_adios.cpp b/src/USER-ADIOS/dump_atom_adios.cpp index e7a5f05e9d..55d6c5fbf9 100644 --- a/src/USER-ADIOS/dump_atom_adios.cpp +++ b/src/USER-ADIOS/dump_atom_adios.cpp @@ -15,16 +15,17 @@ Contributing author: Norbert Podhorszki (ORNL) ------------------------------------------------------------------------- */ -#include #include "dump_atom_adios.h" -#include "domain.h" #include "atom.h" -#include "update.h" +#include "domain.h" +#include "error.h" #include "group.h" #include "memory.h" #include "universe.h" -#include "error.h" +#include "update.h" +#include +#include "adios2.h" using namespace LAMMPS_NS; @@ -32,23 +33,44 @@ using namespace LAMMPS_NS; #define DUMP_BUF_CHUNK_SIZE 16384 #define DUMP_BUF_INCREMENT_SIZE 4096 +namespace LAMMPS_NS +{ +class DumpAtomADIOSInternal +{ + +public: + DumpAtomADIOSInternal(){}; + ~DumpAtomADIOSInternal() = default; + + // name of adios group, referrable in adios2_config.xml + const std::string ioName = "atom"; + adios2::ADIOS *ad = nullptr; // adios object + adios2::IO io; // adios group of variables and attributes in this dump + adios2::Engine fh; // adios file/stream handle object + // one ADIOS output variable we need to change every step + adios2::Variable varAtoms; +}; +} + /* ---------------------------------------------------------------------- */ -DumpAtomADIOS::DumpAtomADIOS(LAMMPS *lmp, int narg, char **arg) : - DumpAtom(lmp, narg, arg) +DumpAtomADIOS::DumpAtomADIOS(LAMMPS *lmp, int narg, char **arg) +: DumpAtom(lmp, narg, arg) { - ad = new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); - groupSize = 0; + internal = new DumpAtomADIOSInternal(); + internal->ad = + new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); } /* ---------------------------------------------------------------------- */ DumpAtomADIOS::~DumpAtomADIOS() { - if (fh) - { - fh.Close(); + if (internal->fh) { + internal->fh.Close(); } + delete internal->ad; + delete internal; } /* ---------------------------------------------------------------------- */ @@ -59,39 +81,39 @@ void DumpAtomADIOS::openfile() // if one file per timestep, replace '*' with current timestep char *filestar = strdup(filename); char *filecurrent = new char[strlen(filestar) + 16]; - char *ptr = strchr(filestar,'*'); + char *ptr = strchr(filestar, '*'); *ptr = '\0'; if (padflag == 0) - sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", - filestar,update->ntimestep,ptr+1); + snprintf(filecurrent, sizeof(filecurrent), "%s" BIGINT_FORMAT "%s", + filestar, update->ntimestep, ptr + 1); else { - char bif[8],pad[16]; - strcpy(bif,BIGINT_FORMAT); - sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); - sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + char bif[8], pad[16]; + strcpy(bif, BIGINT_FORMAT); + snprintf(pad, sizeof(pad), "%%s%%0%d%s%%s", padflag, &bif[1]); + snprintf(filecurrent, sizeof(filecurrent), pad, filestar, + update->ntimestep, ptr + 1); } - fh = io.Open(filecurrent, adios2::Mode::Write, world); - if (!fh) { + internal->fh = + internal->io.Open(filecurrent, adios2::Mode::Write, world); + if (!internal->fh) { char str[128]; - sprintf(str,"Cannot open dump file %s",filecurrent); - error->one(FLERR,str); + snprintf(str, sizeof(str), "Cannot open dump file %s", filecurrent); + error->one(FLERR, str); } free(filestar); - delete [] filecurrent; - } - else - { - if (!singlefile_opened) - { - fh = io.Open(filename, adios2::Mode::Write, world); - if (!fh) { + delete[] filecurrent; + } else { + if (!singlefile_opened) { + internal->fh = + internal->io.Open(filename, adios2::Mode::Write, world); + if (!internal->fh) { char str[128]; - sprintf(str,"Cannot open dump file %s",filename); - error->one(FLERR,str); + snprintf(str, sizeof(str), "Cannot open dump file %s", + filename); + error->one(FLERR, str); } singlefile_opened = 1; } - } } @@ -126,83 +148,77 @@ void DumpAtomADIOS::write() // atomOffset = sum of # of atoms up to this proc (exclusive prefix sum) bigint bnme = nme; - MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); + MPI_Allreduce(&bnme, &ntotal, 1, MPI_LMP_BIGINT, MPI_SUM, world); bigint atomOffset; // sum of all atoms on processes 0..me-1 - MPI_Scan (&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); + MPI_Scan(&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); atomOffset -= nme; // exclusive prefix sum needed // Now we know the global size and the local subset size and offset - // of the atoms table + // of the atoms table size_t nAtomsGlobal = static_cast(ntotal); size_t startRow = static_cast(atomOffset); size_t nAtomsLocal = static_cast(nme); size_t nColumns = static_cast(size_one); - varAtoms.SetShape({nAtomsGlobal,nColumns}); - varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal,nColumns}}); + internal->varAtoms.SetShape({nAtomsGlobal, nColumns}); + internal->varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal, nColumns}}); // insure buf is sized for packing - // adios does not limit per-process data size so nme*size_one is not constrained to int + // adios does not limit per-process data size so nme*size_one is not + // constrained to int // if sorting on IDs also request ID list from pack() // sort buf as needed if (nme > maxbuf) { maxbuf = nme; memory->destroy(buf); - memory->create(buf,(maxbuf*size_one),"dump:buf"); + memory->create(buf, (maxbuf * size_one), "dump:buf"); } if (sort_flag && sortcol == 0 && nme > maxids) { maxids = nme; memory->destroy(ids); - memory->create(ids,maxids,"dump:ids"); + memory->create(ids, maxids, "dump:ids"); } - if (sort_flag && sortcol == 0) pack(ids); - else pack(NULL); - if (sort_flag) sort(); - - // Calculate data size written by this process - groupSize = nme * size_one * sizeof(double); // size of atoms data on this process - groupSize += 3*sizeof(uint64_t) + 1*sizeof(int); // scalars written by each process - if (me == 0) { - groupSize += 1*sizeof(uint64_t) + 1*sizeof(int) + 6*sizeof(double); // scalars - if (domain->triclinic) { - groupSize += 3*sizeof(double); // boxxy, boxxz, boxyz - } - } + if (sort_flag && sortcol == 0) + pack(ids); + else + pack(NULL); + if (sort_flag) + sort(); openfile(); - fh.BeginStep(); + internal->fh.BeginStep(); // write info on data as scalars (by me==0) if (me == 0) { - fh.Put("ntimestep", update->ntimestep); - fh.Put("nprocs", nprocs); + internal->fh.Put("ntimestep", update->ntimestep); + internal->fh.Put("nprocs", nprocs); - fh.Put("boxxlo", boxxlo); - fh.Put("boxxhi", boxxhi); - fh.Put("boxylo", boxylo); - fh.Put("boxyhi", boxyhi); - fh.Put("boxzlo", boxzlo); - fh.Put("boxzhi", boxzhi); + internal->fh.Put("boxxlo", boxxlo); + internal->fh.Put("boxxhi", boxxhi); + internal->fh.Put("boxylo", boxylo); + internal->fh.Put("boxyhi", boxyhi); + internal->fh.Put("boxzlo", boxzlo); + internal->fh.Put("boxzhi", boxzhi); if (domain->triclinic) { - fh.Put("boxxy", boxxy); - fh.Put("boxxz", boxxz); - fh.Put("boxyz", boxyz); + internal->fh.Put("boxxy", boxxy); + internal->fh.Put("boxxz", boxxz); + internal->fh.Put("boxyz", boxyz); } } - // Everyone needs to write scalar variables that are used as dimensions and offsets of arrays - fh.Put("natoms", ntotal); - fh.Put("ncolumns", size_one); - fh.Put("nme", bnme); - fh.Put("offset", atomOffset); + // Everyone needs to write scalar variables that are used as dimensions and + // offsets of arrays + internal->fh.Put("natoms", ntotal); + internal->fh.Put("ncolumns", size_one); + internal->fh.Put("nme", bnme); + internal->fh.Put("offset", atomOffset); // now write the atoms - fh.Put("atoms", buf); - fh.EndStep();// I/O will happen now... + internal->fh.Put(internal->varAtoms, buf); + internal->fh.EndStep(); // I/O will happen now... - if (multifile) - { - fh.Close(); + if (multifile) { + internal->fh.Close(); } } @@ -210,33 +226,36 @@ void DumpAtomADIOS::write() void DumpAtomADIOS::init_style() { - if (image_flag == 0) size_one = 5; - else size_one = 8; + if (image_flag == 0) + size_one = 5; + else + size_one = 8; // setup boundary string domain->boundary_string(boundstr); - // remove % from filename since ADIOS always writes a global file with data/metadata + // remove % from filename since ADIOS always writes a global file with + // data/metadata int len = strlen(filename); - char *ptr = strchr(filename,'%'); + char *ptr = strchr(filename, '%'); if (ptr) { *ptr = '\0'; - char *s = new char[len-1]; - sprintf(s,"%s%s",filename,ptr+1); - strncpy(filename,s,len); + char *s = new char[len - 1]; + snprintf(s, sizeof(s), "%s%s", filename, ptr + 1); + strncpy(filename, s, len); } // setup column string if (scale_flag == 0 && image_flag == 0) - columns = (char *) "id type x y z"; + columns = (char *)"id type x y z"; else if (scale_flag == 0 && image_flag == 1) - columns = (char *) "id type x y z ix iy iz"; + columns = (char *)"id type x y z ix iy iz"; else if (scale_flag == 1 && image_flag == 0) - columns = (char *) "id type xs ys zs"; + columns = (char *)"id type xs ys zs"; else if (scale_flag == 1 && image_flag == 1) - columns = (char *) "id type xs ys zs ix iy iz"; + columns = (char *)"id type xs ys zs ix iy iz"; // setup function ptrs @@ -253,62 +272,68 @@ void DumpAtomADIOS::init_style() else if (scale_flag == 0 && image_flag == 1) pack_choice = &DumpAtomADIOS::pack_noscale_image; - /* Define the group of variables for the atom style here since it's a fixed set */ - io = ad->DeclareIO(ioName); - if (!io.InConfigFile()) - { + /* Define the group of variables for the atom style here since it's a fixed + * set */ + internal->io = internal->ad->DeclareIO(internal->ioName); + if (!internal->io.InConfigFile()) { // if not defined by user, we can change the default settings // BPFile is the default writer - io.SetEngine("BPFile"); + internal->io.SetEngine("BPFile"); int num_aggregators = multiproc; if (num_aggregators == 0) num_aggregators = 1; char nstreams[128]; - sprintf (nstreams, "%d", num_aggregators); - io.SetParameters({{"substreams", nstreams}}); - if (me==0 && screen) fprintf(screen, "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", filename, nstreams); + snprintf(nstreams, sizeof(nstreams), "%d", num_aggregators); + internal->io.SetParameters({{"substreams", nstreams}}); + if (me == 0 && screen) + fprintf( + screen, + "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", + filename, nstreams); } + internal->io.DefineVariable("ntimestep"); + internal->io.DefineVariable("natoms"); - io.DefineVariable("ntimestep"); - io.DefineVariable("natoms"); + internal->io.DefineVariable("nprocs"); + internal->io.DefineVariable("ncolumns"); - io.DefineVariable("nprocs"); - io.DefineVariable("ncolumns"); + internal->io.DefineVariable("boxxlo"); + internal->io.DefineVariable("boxxhi"); + internal->io.DefineVariable("boxylo"); + internal->io.DefineVariable("boxyhi"); + internal->io.DefineVariable("boxzlo"); + internal->io.DefineVariable("boxzhi"); - io.DefineVariable("boxxlo"); - io.DefineVariable("boxxhi"); - io.DefineVariable("boxylo"); - io.DefineVariable("boxyhi"); - io.DefineVariable("boxzlo"); - io.DefineVariable("boxzhi"); + internal->io.DefineVariable("boxxy"); + internal->io.DefineVariable("boxxz"); + internal->io.DefineVariable("boxyz"); - io.DefineVariable("boxxy"); - io.DefineVariable("boxxz"); - io.DefineVariable("boxyz"); + internal->io.DefineAttribute("triclinic", domain->triclinic); + internal->io.DefineAttribute("scaled", scale_flag); + internal->io.DefineAttribute("image", image_flag); - io.DefineAttribute("triclinic", domain->triclinic); - io.DefineAttribute("scaled", scale_flag); - io.DefineAttribute("image", image_flag); + int *boundaryptr = reinterpret_cast(domain->boundary); + internal->io.DefineAttribute("boundary", boundaryptr, 6); - int *boundaryptr = reinterpret_cast(domain->boundary); - io.DefineAttribute("boundary", boundaryptr, 6); + internal->io.DefineAttribute("columns", columns); + internal->io.DefineAttribute("boundarystr", boundstr); + internal->io.DefineAttribute("LAMMPS/dump_style", "atom"); + internal->io.DefineAttribute("LAMMPS/version", + universe->version); + internal->io.DefineAttribute("LAMMPS/num_ver", + universe->num_ver); - io.DefineAttribute("columns", columns); - io.DefineAttribute("boundarystr", boundstr); - io.DefineAttribute("LAMMPS/dump_style", "atom"); - io.DefineAttribute("LAMMPS/version", universe->version); - io.DefineAttribute("LAMMPS/num_ver", universe->num_ver); - - io.DefineVariable("nme", {adios2::LocalValueDim}); // local dimension variable - io.DefineVariable("offset", {adios2::LocalValueDim}); // local dimension variable + internal->io.DefineVariable( + "nme", {adios2::LocalValueDim}); // local dimension variable + internal->io.DefineVariable( + "offset", {adios2::LocalValueDim}); // local dimension variable // atom table size is not known at the moment // it will be correctly defined at the moment of write size_t UnknownSizeYet = 1; size_t nColumns = static_cast(size_one); - varAtoms = io.DefineVariable("atoms", - {UnknownSizeYet,nColumns}, - {UnknownSizeYet, 0}, - {UnknownSizeYet,nColumns}); + internal->varAtoms = internal->io.DefineVariable( + "atoms", {UnknownSizeYet, nColumns}, {UnknownSizeYet, 0}, + {UnknownSizeYet, nColumns}); } diff --git a/src/USER-ADIOS/dump_atom_adios.h b/src/USER-ADIOS/dump_atom_adios.h index a3999bea76..dc6bc519bb 100644 --- a/src/USER-ADIOS/dump_atom_adios.h +++ b/src/USER-ADIOS/dump_atom_adios.h @@ -12,59 +12,51 @@ ------------------------------------------------------------------------- */ #ifdef DUMP_CLASS - -DumpStyle(atom/adios,DumpAtomADIOS) - +// clang-format off +DumpStyle(atom/adios, DumpAtomADIOS) +// clang-format on #else #ifndef LMP_DUMP_ATOM_ADIOS_H #define LMP_DUMP_ATOM_ADIOS_H #include "dump_atom.h" -#include -#include -#include "adios2.h" -namespace LAMMPS_NS { +namespace LAMMPS_NS +{ -class DumpAtomADIOS : public DumpAtom { +class DumpAtomADIOSInternal; - public: - DumpAtomADIOS(class LAMMPS *, int, char **); - virtual ~DumpAtomADIOS(); +class DumpAtomADIOS : public DumpAtom +{ - protected: +public: + DumpAtomADIOS(class LAMMPS *, int, char **); + virtual ~DumpAtomADIOS(); - const std::string ioName="atom"; // name of adios group, referrable in adios2_config.xml - adios2::ADIOS *ad = nullptr; // adios object - adios2::IO io; // adios group of variables and attributes in this dump - adios2::Engine fh; // adios file/stream handle object - adios2::Variable varAtoms; // one ADIOS output variable we need to change - uint64_t groupSize; // pre-calculate # of bytes written per processor in a step before writing anything - uint64_t groupTotalSize; // ADIOS buffer size returned by adios_group_size(), valid only if size is > default 16MB ADIOS buffer - std::string filecurrent; // name of file for this round (with % and * replaced) - - virtual void openfile(); - virtual void write(); - virtual void init_style(); +protected: + virtual void openfile(); + virtual void write(); + virtual void init_style(); +private: + DumpAtomADIOSInternal *internal; }; - } #endif #endif -/* ERROR/WARNING messages: + /* ERROR/WARNING messages: -E: Cannot open dump file %s + E: Cannot open dump file %s -The output file for the dump command cannot be opened. Check that the -path and name are correct. + The output file for the dump command cannot be opened. Check that the + path and name are correct. -E: Too much per-proc info for dump + E: Too much per-proc info for dump -Number of local atoms times number of columns must fit in a 32-bit -integer for dump. + Number of local atoms times number of columns must fit in a 32-bit + integer for dump. -*/ + */ diff --git a/src/USER-ADIOS/dump_custom_adios.cpp b/src/USER-ADIOS/dump_custom_adios.cpp index 5f54b987b1..5b76688098 100644 --- a/src/USER-ADIOS/dump_custom_adios.cpp +++ b/src/USER-ADIOS/dump_custom_adios.cpp @@ -15,25 +15,25 @@ Contributing author: Paul Coffman (IBM) ------------------------------------------------------------------------- */ -#include -#include -#include #include "dump_custom_adios.h" #include "atom.h" -#include "force.h" +#include "compute.h" #include "domain.h" -#include "region.h" +#include "error.h" +#include "fix.h" +#include "force.h" #include "group.h" #include "input.h" -#include "variable.h" -#include "update.h" -#include "modify.h" -#include "compute.h" -#include "fix.h" -#include "universe.h" #include "memory.h" -#include "error.h" -#include +#include "modify.h" +#include "region.h" +#include "universe.h" +#include "update.h" +#include "variable.h" +#include +#include + +#include "adios2.h" using namespace LAMMPS_NS; @@ -41,30 +41,107 @@ using namespace LAMMPS_NS; #define DUMP_BUF_CHUNK_SIZE 16384 #define DUMP_BUF_INCREMENT_SIZE 4096 -enum{ID,MOL,TYPE,ELEMENT,MASS, - X,Y,Z,XS,YS,ZS,XSTRI,YSTRI,ZSTRI,XU,YU,ZU,XUTRI,YUTRI,ZUTRI, - XSU,YSU,ZSU,XSUTRI,YSUTRI,ZSUTRI, - IX,IY,IZ, - VX,VY,VZ,FX,FY,FZ, - Q,MUX,MUY,MUZ,MU,RADIUS,DIAMETER, - OMEGAX,OMEGAY,OMEGAZ,ANGMOMX,ANGMOMY,ANGMOMZ, - TQX,TQY,TQZ,SPIN,ERADIUS,ERVEL,ERFORCE, - COMPUTE,FIX,VARIABLE}; -enum{LT,LE,GT,GE,EQ,NEQ}; -enum{INT,DOUBLE,STRING,BIGINT}; // same as in DumpCustom +enum { + ID, + MOL, + TYPE, + ELEMENT, + MASS, + X, + Y, + Z, + XS, + YS, + ZS, + XSTRI, + YSTRI, + ZSTRI, + XU, + YU, + ZU, + XUTRI, + YUTRI, + ZUTRI, + XSU, + YSU, + ZSU, + XSUTRI, + YSUTRI, + ZSUTRI, + IX, + IY, + IZ, + VX, + VY, + VZ, + FX, + FY, + FZ, + Q, + MUX, + MUY, + MUZ, + MU, + RADIUS, + DIAMETER, + OMEGAX, + OMEGAY, + OMEGAZ, + ANGMOMX, + ANGMOMY, + ANGMOMZ, + TQX, + TQY, + TQZ, + SPIN, + ERADIUS, + ERVEL, + ERFORCE, + COMPUTE, + FIX, + VARIABLE +}; +enum { LT, LE, GT, GE, EQ, NEQ }; +enum { INT, DOUBLE, STRING, BIGINT }; // same as in DumpCustom + +namespace LAMMPS_NS +{ +class DumpCustomADIOSInternal +{ + +public: + DumpCustomADIOSInternal(){}; + ~DumpCustomADIOSInternal() = default; + + // name of adios group, referrable in adios2_config.xml + const std::string ioName = "custom"; + adios2::ADIOS *ad = nullptr; // adios object + adios2::IO io; // adios group of variables and attributes in this dump + adios2::Engine fh; // adios file/stream handle object + // one ADIOS output variable we need to change every step + adios2::Variable varAtoms; + // list of column names for the atom table + // (individual list of 'columns' string) + std::vector columnNames; +}; +} /* ---------------------------------------------------------------------- */ -DumpCustomADIOS::DumpCustomADIOS(LAMMPS *lmp, int narg, char **arg) : - DumpCustom(lmp, narg, arg) +DumpCustomADIOS::DumpCustomADIOS(LAMMPS *lmp, int narg, char **arg) +: DumpCustom(lmp, narg, arg) { - ad = new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); - groupsize = 0; - //if (screen) fprintf(screen, "DumpCustomADIOS constructor: nvariable=%d id_variable=%p, variables=%p, nfield=%d, earg=%p\n", nvariable, id_variable, variable, nfield, earg); - columnNames.reserve(nfield); + internal = new DumpCustomADIOSInternal(); + internal->ad = + new adios2::ADIOS("adios2_config.xml", world, adios2::DebugON); + + // if (screen) fprintf(screen, "DumpCustomADIOS constructor: nvariable=%d + // id_variable=%p, variables=%p, nfield=%d, earg=%p\n", nvariable, + // id_variable, variable, nfield, earg); + internal->columnNames.reserve(nfield); for (int i = 0; i < nfield; ++i) { - columnNames.push_back(earg[i]); - //if (screen) fprintf(screen, "earg[%d] = '%s'\n", i, earg[i]); + internal->columnNames.push_back(earg[i]); + // if (screen) fprintf(screen, "earg[%d] = '%s'\n", i, earg[i]); } } @@ -72,11 +149,11 @@ DumpCustomADIOS::DumpCustomADIOS(LAMMPS *lmp, int narg, char **arg) : DumpCustomADIOS::~DumpCustomADIOS() { - columnNames.clear(); - if (fh) - { - fh.Close(); + internal->columnNames.clear(); + if (internal->fh) { + internal->fh.Close(); } + delete internal->ad; } /* ---------------------------------------------------------------------- */ @@ -87,39 +164,37 @@ void DumpCustomADIOS::openfile() // if one file per timestep, replace '*' with current timestep char *filestar = strdup(filename); char *filecurrent = new char[strlen(filestar) + 16]; - char *ptr = strchr(filestar,'*'); + char *ptr = strchr(filestar, '*'); *ptr = '\0'; if (padflag == 0) - sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", - filestar,update->ntimestep,ptr+1); + sprintf(filecurrent, "%s" BIGINT_FORMAT "%s", filestar, + update->ntimestep, ptr + 1); else { - char bif[8],pad[16]; - strcpy(bif,BIGINT_FORMAT); - sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); - sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + char bif[8], pad[16]; + strcpy(bif, BIGINT_FORMAT); + sprintf(pad, "%%s%%0%d%s%%s", padflag, &bif[1]); + sprintf(filecurrent, pad, filestar, update->ntimestep, ptr + 1); } - fh = io.Open(filecurrent, adios2::Mode::Write, world); - if (!fh) { + internal->fh = + internal->io.Open(filecurrent, adios2::Mode::Write, world); + if (!internal->fh) { char str[128]; - sprintf(str,"Cannot open dump file %s",filecurrent); - error->one(FLERR,str); + sprintf(str, "Cannot open dump file %s", filecurrent); + error->one(FLERR, str); } free(filestar); - delete [] filecurrent; - } - else - { - if (!singlefile_opened) - { - fh = io.Open(filename, adios2::Mode::Write, world); - if (!fh) { + delete[] filecurrent; + } else { + if (!singlefile_opened) { + internal->fh = + internal->io.Open(filename, adios2::Mode::Write, world); + if (!internal->fh) { char str[128]; - sprintf(str,"Cannot open dump file %s",filename); - error->one(FLERR,str); + sprintf(str, "Cannot open dump file %s", filename); + error->one(FLERR, str); } singlefile_opened = 1; } - } } @@ -154,20 +229,20 @@ void DumpCustomADIOS::write() // atomOffset = sum of # of atoms up to this proc (exclusive prefix sum) bigint bnme = nme; - MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); + MPI_Allreduce(&bnme, &ntotal, 1, MPI_LMP_BIGINT, MPI_SUM, world); bigint atomOffset; // sum of all atoms on processes 0..me-1 - MPI_Scan (&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); + MPI_Scan(&bnme, &atomOffset, 1, MPI_LMP_BIGINT, MPI_SUM, world); atomOffset -= nme; // exclusive prefix sum needed // Now we know the global size and the local subset size and offset - // of the atoms table + // of the atoms table size_t nAtomsGlobal = static_cast(ntotal); size_t startRow = static_cast(atomOffset); size_t nAtomsLocal = static_cast(nme); size_t nColumns = static_cast(size_one); - varAtoms.SetShape({nAtomsGlobal,nColumns}); - varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal,nColumns}}); + internal->varAtoms.SetShape({nAtomsGlobal, nColumns}); + internal->varAtoms.SetSelection({{startRow, 0}, {nAtomsLocal, nColumns}}); // insure filewriter proc can receive everyone's info // limit nmax*size_one to int since used as arg in MPI_Rsend() below @@ -176,64 +251,57 @@ void DumpCustomADIOS::write() // sort buf as needed if (nme > maxbuf) { - if ((bigint) nme * size_one > MAXSMALLINT) - error->all(FLERR,"Too much per-proc info for dump"); + if ((bigint)nme * size_one > MAXSMALLINT) + error->all(FLERR, "Too much per-proc info for dump"); maxbuf = nme; memory->destroy(buf); - memory->create(buf,(maxbuf*size_one),"dump:buf"); + memory->create(buf, (maxbuf * size_one), "dump:buf"); } if (sort_flag && sortcol == 0 && nme > maxids) { maxids = nme; memory->destroy(ids); - memory->create(ids,maxids,"dump:ids"); + memory->create(ids, maxids, "dump:ids"); } - if (sort_flag && sortcol == 0) pack(ids); - else pack(NULL); - if (sort_flag) sort(); - - // Calculate data size written by this process - groupsize = nme * size_one * sizeof(double); // size of atoms data on this process - groupsize += 3*sizeof(uint64_t) + 1*sizeof(int); // scalars written by each process - if (me == 0) { - groupsize += 1*sizeof(uint64_t) + 1*sizeof(int) + 6*sizeof(double); // scalars - if (domain->triclinic) { - groupsize += 3*sizeof(double); // boxxy, boxxz, boxyz - } - } + if (sort_flag && sortcol == 0) + pack(ids); + else + pack(NULL); + if (sort_flag) + sort(); openfile(); - fh.BeginStep(); + internal->fh.BeginStep(); // write info on data as scalars (by me==0) if (me == 0) { - fh.Put("ntimestep", update->ntimestep); - fh.Put("nprocs", nprocs); + internal->fh.Put("ntimestep", update->ntimestep); + internal->fh.Put("nprocs", nprocs); - fh.Put("boxxlo", boxxlo); - fh.Put("boxxhi", boxxhi); - fh.Put("boxylo", boxylo); - fh.Put("boxyhi", boxyhi); - fh.Put("boxzlo", boxzlo); - fh.Put("boxzhi", boxzhi); + internal->fh.Put("boxxlo", boxxlo); + internal->fh.Put("boxxhi", boxxhi); + internal->fh.Put("boxylo", boxylo); + internal->fh.Put("boxyhi", boxyhi); + internal->fh.Put("boxzlo", boxzlo); + internal->fh.Put("boxzhi", boxzhi); if (domain->triclinic) { - fh.Put("boxxy", boxxy); - fh.Put("boxxz", boxxz); - fh.Put("boxyz", boxyz); + internal->fh.Put("boxxy", boxxy); + internal->fh.Put("boxxz", boxxz); + internal->fh.Put("boxyz", boxyz); } } - // Everyone needs to write scalar variables that are used as dimensions and offsets of arrays - fh.Put("natoms", ntotal); - fh.Put("ncolumns", size_one); - fh.Put("nme", bnme); - fh.Put("offset", atomOffset); + // Everyone needs to write scalar variables that are used as dimensions and + // offsets of arrays + internal->fh.Put("natoms", ntotal); + internal->fh.Put("ncolumns", size_one); + internal->fh.Put("nme", bnme); + internal->fh.Put("offset", atomOffset); // now write the atoms - fh.Put("atoms", buf); - fh.EndStep();// I/O will happen now... + internal->fh.Put("atoms", buf); + internal->fh.EndStep(); // I/O will happen now... - if (multifile) - { - fh.Close(); + if (multifile) { + internal->fh.Close(); } } @@ -246,43 +314,49 @@ void DumpCustomADIOS::init_style() domain->boundary_string(boundstr); - // remove % from filename since ADIOS always writes a global file with data/metadata + // remove % from filename since ADIOS always writes a global file with + // data/metadata int len = strlen(filename); - char *ptr = strchr(filename,'%'); + char *ptr = strchr(filename, '%'); if (ptr) { *ptr = '\0'; - char *s = new char[len-1]; - sprintf(s,"%s%s",filename,ptr+1); - strncpy(filename,s,len); + char *s = new char[len - 1]; + sprintf(s, "%s%s", filename, ptr + 1); + strncpy(filename, s, len); } - /* The next four loops are copied from dump_custom_mpiio, but nothing is done with them. + /* The next four loops are copied from dump_custom_mpiio, but nothing is + * done with them. * It is unclear why we need them here. - * For metadata, variable[] will be written out as an ADIOS attribute if nvariable>0 + * For metadata, variable[] will be written out as an ADIOS attribute if + * nvariable>0 */ // find current ptr for each compute,fix,variable // check that fix frequency is acceptable int icompute; for (int i = 0; i < ncompute; i++) { icompute = modify->find_compute(id_compute[i]); - if (icompute < 0) error->all(FLERR,"Could not find dump custom compute ID"); + if (icompute < 0) + error->all(FLERR, "Could not find dump custom compute ID"); compute[i] = modify->compute[icompute]; } int ifix; for (int i = 0; i < nfix; i++) { ifix = modify->find_fix(id_fix[i]); - if (ifix < 0) error->all(FLERR,"Could not find dump custom fix ID"); + if (ifix < 0) + error->all(FLERR, "Could not find dump custom fix ID"); fix[i] = modify->fix[ifix]; if (nevery % modify->fix[ifix]->peratom_freq) - error->all(FLERR,"Dump custom and fix not computed at compatible times"); + error->all(FLERR, + "Dump custom and fix not computed at compatible times"); } int ivariable; for (int i = 0; i < nvariable; i++) { ivariable = input->variable->find(id_variable[i]); if (ivariable < 0) - error->all(FLERR,"Could not find dump custom variable name"); + error->all(FLERR, "Could not find dump custom variable name"); variable[i] = ivariable; } @@ -290,65 +364,71 @@ void DumpCustomADIOS::init_style() if (iregion >= 0) { iregion = domain->find_region(idregion); if (iregion == -1) - error->all(FLERR,"Region ID for dump custom does not exist"); + error->all(FLERR, "Region ID for dump custom does not exist"); } - /* Define the group of variables for the atom style here since it's a fixed set */ - io = ad->DeclareIO(ioName); - if (!io.InConfigFile()) - { + /* Define the group of variables for the atom style here since it's a fixed + * set */ + internal->io = internal->ad->DeclareIO(internal->ioName); + if (!internal->io.InConfigFile()) { // if not defined by user, we can change the default settings // BPFile is the default writer - io.SetEngine("BPFile"); + internal->io.SetEngine("BPFile"); int num_aggregators = multiproc; if (num_aggregators == 0) num_aggregators = 1; char nstreams[128]; - sprintf (nstreams, "%d", num_aggregators); - io.SetParameters({{"substreams", nstreams}}); - if (me==0 && screen) fprintf(screen, "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", filename, nstreams); + sprintf(nstreams, "%d", num_aggregators); + internal->io.SetParameters({{"substreams", nstreams}}); + if (me == 0 && screen) + fprintf( + screen, + "ADIOS method for %s is n-to-m (aggregation with %s writers)\n", + filename, nstreams); } + internal->io.DefineVariable("ntimestep"); + internal->io.DefineVariable("natoms"); - io.DefineVariable("ntimestep"); - io.DefineVariable("natoms"); + internal->io.DefineVariable("nprocs"); + internal->io.DefineVariable("ncolumns"); - io.DefineVariable("nprocs"); - io.DefineVariable("ncolumns"); + internal->io.DefineVariable("boxxlo"); + internal->io.DefineVariable("boxxhi"); + internal->io.DefineVariable("boxylo"); + internal->io.DefineVariable("boxyhi"); + internal->io.DefineVariable("boxzlo"); + internal->io.DefineVariable("boxzhi"); - io.DefineVariable("boxxlo"); - io.DefineVariable("boxxhi"); - io.DefineVariable("boxylo"); - io.DefineVariable("boxyhi"); - io.DefineVariable("boxzlo"); - io.DefineVariable("boxzhi"); + internal->io.DefineVariable("boxxy"); + internal->io.DefineVariable("boxxz"); + internal->io.DefineVariable("boxyz"); - io.DefineVariable("boxxy"); - io.DefineVariable("boxxz"); - io.DefineVariable("boxyz"); + internal->io.DefineAttribute("triclinic", domain->triclinic); - io.DefineAttribute("triclinic", domain->triclinic); - - int *boundaryptr = reinterpret_cast(domain->boundary); - io.DefineAttribute("boundary", boundaryptr, 6); + int *boundaryptr = reinterpret_cast(domain->boundary); + internal->io.DefineAttribute("boundary", boundaryptr, 6); size_t nColumns = static_cast(size_one); - io.DefineAttribute("columns", columnNames.data(), nColumns); - io.DefineAttribute("columnstr", columns); - io.DefineAttribute("boundarystr", boundstr); - io.DefineAttribute("LAMMPS/dump_style", "atom"); - io.DefineAttribute("LAMMPS/version", universe->version); - io.DefineAttribute("LAMMPS/num_ver", universe->num_ver); + internal->io.DefineAttribute( + "columns", internal->columnNames.data(), nColumns); + internal->io.DefineAttribute("columnstr", columns); + internal->io.DefineAttribute("boundarystr", boundstr); + internal->io.DefineAttribute("LAMMPS/dump_style", "atom"); + internal->io.DefineAttribute("LAMMPS/version", + universe->version); + internal->io.DefineAttribute("LAMMPS/num_ver", + universe->num_ver); - io.DefineVariable("nme", {adios2::LocalValueDim}); // local dimension variable - io.DefineVariable("offset", {adios2::LocalValueDim}); // local dimension variable + internal->io.DefineVariable( + "nme", {adios2::LocalValueDim}); // local dimension variable + internal->io.DefineVariable( + "offset", {adios2::LocalValueDim}); // local dimension variable // atom table size is not known at the moment // it will be correctly defined at the moment of write size_t UnknownSizeYet = 1; - varAtoms = io.DefineVariable("atoms", - {UnknownSizeYet,nColumns}, - {UnknownSizeYet, 0}, - {UnknownSizeYet,nColumns}); - + internal->varAtoms = internal->io.DefineVariable( + "atoms", {UnknownSizeYet, nColumns}, {UnknownSizeYet, 0}, + {UnknownSizeYet, nColumns}); } diff --git a/src/USER-ADIOS/dump_custom_adios.h b/src/USER-ADIOS/dump_custom_adios.h index 2461e20b2e..d5d41eb8c7 100644 --- a/src/USER-ADIOS/dump_custom_adios.h +++ b/src/USER-ADIOS/dump_custom_adios.h @@ -12,83 +12,76 @@ ------------------------------------------------------------------------- */ #ifdef DUMP_CLASS - -DumpStyle(custom/adios,DumpCustomADIOS) - +// clang-format off +DumpStyle(custom/adios, DumpCustomADIOS) +// clang-format on #else #ifndef LMP_DUMP_CUSTOM_ADIOS_H #define LMP_DUMP_CUSTOM_ADIOS_H #include "dump_custom.h" -#include -#include -#include "adios2.h" -namespace LAMMPS_NS { +namespace LAMMPS_NS +{ -class DumpCustomADIOS : public DumpCustom { - public: - DumpCustomADIOS(class LAMMPS *, int, char **); - virtual ~DumpCustomADIOS(); +class DumpCustomADIOSInternal; - protected: +class DumpCustomADIOS : public DumpCustom +{ +public: + DumpCustomADIOS(class LAMMPS *, int, char **); + virtual ~DumpCustomADIOS(); - const std::string ioName="custom"; // name of adios group, referrable in adios2_config.xml - adios2::ADIOS *ad = nullptr; // adios object - adios2::IO io; // adios group of variables and attributes in this dump - adios2::Engine fh; // adios file/stream handle object - adios2::Variable varAtoms; // one ADIOS output variable we need to change - uint64_t groupsize; // pre-calculate # of bytes written per processor in a step before writing anything - std::vectorcolumnNames; // list of column names for the atom table (individual list of 'columns' string) - - virtual void openfile(); - virtual void write(); - virtual void init_style(); +protected: + virtual void openfile(); + virtual void write(); + virtual void init_style(); +private: + DumpCustomADIOSInternal *internal; }; - } #endif #endif -/* ERROR/WARNING messages: + /* ERROR/WARNING messages: -E: Cannot open dump file %s + E: Cannot open dump file %s -The output file for the dump command cannot be opened. Check that the -path and name are correct. + The output file for the dump command cannot be opened. Check that the + path and name are correct. -E: Too much per-proc info for dump + E: Too much per-proc info for dump -Number of local atoms times number of columns must fit in a 32-bit -integer for dump. + Number of local atoms times number of columns must fit in a 32-bit + integer for dump. -E: Dump_modify format string is too short + E: Dump_modify format string is too short -There are more fields to be dumped in a line of output than your -format string specifies. + There are more fields to be dumped in a line of output than your + format string specifies. -E: Could not find dump custom compute ID + E: Could not find dump custom compute ID -Self-explanatory. + Self-explanatory. -E: Could not find dump custom fix ID + E: Could not find dump custom fix ID -Self-explanatory. + Self-explanatory. -E: Dump custom and fix not computed at compatible times + E: Dump custom and fix not computed at compatible times -The fix must produce per-atom quantities on timesteps that dump custom -needs them. + The fix must produce per-atom quantities on timesteps that dump custom + needs them. -E: Could not find dump custom variable name + E: Could not find dump custom variable name -Self-explanatory. + Self-explanatory. -E: Region ID for dump custom does not exist + E: Region ID for dump custom does not exist -Self-explanatory. + Self-explanatory. -*/ + */ From 75fe967876cfc49538624693383cb5b8c575a469 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Sun, 10 Feb 2019 19:19:49 -0500 Subject: [PATCH 07/16] remove fluff about adios in CMakeLists.txt. --- cmake/CMakeLists.txt | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e5d362fe52..69f56e0216 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -2,15 +2,7 @@ # CMake build system # This file is part of LAMMPS # Created by Christoph Junghans and Richard Berger -if(PKG_USER-ADIOS) - message(STATUS "Force newer standards because of using ADIOS") - cmake_minimum_required(VERSION 3.6) - # Force C++11 and C99 - set(CMAKE_CXX_STANDARD 11) - set(CMAKE_CXX_STANDARD_REQUIRED True) -else() - cmake_minimum_required(VERSION 2.8.12) -endif() +cmake_minimum_required(VERSION 2.8.12) project(lammps CXX) set(SOVERSION 0) From 1179beb54330b061d84ecaea61e2586ae09b72e7 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 20 Feb 2019 14:42:28 -0500 Subject: [PATCH 08/16] Makefile build support for USER-ADIOS and build documentation --- doc/src/Build_extras.txt | 23 ++++++++++ doc/src/Build_package.txt | 1 + doc/src/Packages_details.txt | 26 +++++++++++ doc/src/Packages_user.txt | 1 + src/Makefile | 2 +- src/USER-ADIOS/Install.sh | 86 +++++++++++++++++++----------------- 6 files changed, 97 insertions(+), 42 deletions(-) diff --git a/doc/src/Build_extras.txt b/doc/src/Build_extras.txt index 09841bd9d4..4a12930304 100644 --- a/doc/src/Build_extras.txt +++ b/doc/src/Build_extras.txt @@ -37,6 +37,7 @@ This is the list of packages that may require additional steps. "POEMS"_#poems, "PYTHON"_#python, "VORONOI"_#voronoi, +"USER-ADIOS"_#user-adios, "USER-ATC"_#user-atc, "USER-AWPMD"_#user-awpmd, "USER-COLVARS"_#user-colvars, @@ -576,6 +577,28 @@ the lib/voronoi/Makefile.lammps file. :line +USER-ADIOS package :h4,link(user-adios) + +The USER-ADIOS package requires the "ADIOS I/O library"_https://github.com/ornladios/ADIOS2, version 2.3.1 or newer. +Make sure that you have ADIOS built with/without MPI if you build LAMMPS with/without MPI, respectively. + +[CMake build]: + +-D ADIOS2_DIR=path # path is where ADIOS 2.x is installed +-D PKG_USER-ADIOS=yes :pre + +[Traditional make]: + +Turn on the USER-ADIOS package before building LAMMPS. If the ADIOS 2.x software is installed in PATH, there is nothing else to do: + +make yes-user-adios :pre + +otherwise, set ADIOS2_DIR environment variable when turning on the package: + +ADIOS2_DIR=path make yes-user-adios # path is where ADIOS 2.x is installed :pre + +:line + USER-ATC package :h4,link(user-atc) The USER-ATC package requires the MANYBODY package also be installed. diff --git a/doc/src/Build_package.txt b/doc/src/Build_package.txt index 01c82d724e..c5eca96edb 100644 --- a/doc/src/Build_package.txt +++ b/doc/src/Build_package.txt @@ -48,6 +48,7 @@ packages: "POEMS"_Build_extras.html#poems, "PYTHON"_Build_extras.html#python, "VORONOI"_Build_extras.html#voronoi, +"USER-ADIOS"_Build_extras.html#user-adios, "USER-ATC"_Build_extras.html#user-atc, "USER-AWPMD"_Build_extras.html#user-awpmd, "USER-COLVARS"_Build_extras.html#user-colvars, diff --git a/doc/src/Packages_details.txt b/doc/src/Packages_details.txt index 7d3d6d13e2..4e3fd1968f 100644 --- a/doc/src/Packages_details.txt +++ b/doc/src/Packages_details.txt @@ -63,6 +63,7 @@ as contained in the file name. "SRD"_#PKG-SRD, "VORONOI"_#PKG-VORONOI :tb(c=6,ea=c) +"USER-ADIOS"_#PKG-USER-ADIOS, "USER-ATC"_#PKG-USER-ATC, "USER-AWPMD"_#PKG-USER-AWPMD, "USER-BOCS"_#PKG-USER-BOCS, @@ -975,6 +976,31 @@ examples/voronoi :ul :line +USER-ADIOS package :link(PKG-USER-ADIOS),h4 + +[Contents:] + +ADIOS is a high-performance I/O library. This package implements the +dump "atom/adios" and dump "custom/adios" commands to write data using +the ADIOS library. + +[Authors:] Norbert Podhorszki (ORNL) from the ADIOS developer team. + +[Install:] + +This package has "specific installation +instructions"_Build_extras.html#user-adios on the "Build +extras"_Build_extras.html doc page. + +[Supporting info:] + +src/USER-ADIOS: filenames -> commands +src/USER-ADIOS/README +examples/USER/adios +https://github.com/ornladios/ADIOS2 :ul + +:line + USER-ATC package :link(PKG-USER-ATC),h4 [Contents:] diff --git a/doc/src/Packages_user.txt b/doc/src/Packages_user.txt index 4a702e971f..4210f617ef 100644 --- a/doc/src/Packages_user.txt +++ b/doc/src/Packages_user.txt @@ -38,6 +38,7 @@ int = internal library: provided with LAMMPS, but you may need to build it ext = external library: you will need to download and install it on your machine :ul Package, Description, Doc page, Example, Library +"USER-ADIOS"_Packages_details.html#PKG-USER-ADIOS, dump output via ADIOS, "dump adios"_dump_adios.html, USER/adios, ext "USER-ATC"_Packages_details.html#PKG-USER-ATC, Atom-to-Continuum coupling, "fix atc"_fix_atc.html, USER/atc, int "USER-AWPMD"_Packages_details.html#PKG-USER-AWPMD, wave packet MD, "pair_style awpmd/cut"_pair_awpmd.html, USER/awpmd, int "USER-BOCS"_Packages_details.html#PKG-USER-BOCS, BOCS bottom up coarse graining, "fix bocs"_fix_bocs.html, USER/bocs, no diff --git a/src/Makefile b/src/Makefile index 1f0a294e8a..3e224cd6ab 100644 --- a/src/Makefile +++ b/src/Makefile @@ -58,7 +58,7 @@ PACKAGE = asphere body class2 colloid compress coreshell dipole gpu \ molecule mpiio mscg opt peri poems \ python qeq replica rigid shock snap spin srd voronoi -PACKUSER = user-atc user-awpmd user-bocs user-cgdna user-cgsdk user-colvars \ +PACKUSER = user-adios user-atc user-awpmd user-bocs user-cgdna user-cgsdk user-colvars \ user-diffraction user-dpd user-drude user-eff user-fep user-h5md \ user-intel user-lb user-manifold user-meamc user-meso \ user-mgpt user-misc user-mofff user-molfile \ diff --git a/src/USER-ADIOS/Install.sh b/src/USER-ADIOS/Install.sh index cae755406b..8ea971c233 100644 --- a/src/USER-ADIOS/Install.sh +++ b/src/USER-ADIOS/Install.sh @@ -30,60 +30,64 @@ done if (test $1 = 1) then -# if (test -z "$ADIOS_DIR") then -# if command -v adios_config; then -# ADIOS_DIR=`adios_config -d` -# else -# echo "ERROR: ADIOS_DIR environment variable needs to point to ADIOS" \ -# " installation directory or adios_config should be in PATH" -# fi -# fi -# ADIOS_INC=-I${ADIOS_DIR}/include -# ADIOS_LIB=`${ADIOS_DIR}/bin/adios_config -l` -# -# echo "adios_SYSINC=${ADIOS_INC} -#adios_SYSLIB=${ADIOS_LIB} -#adios_SYSPATH=${ADIOS_DIR} -#" > ../Makefile.adios - - - if (test -e ../Makefile.package) then - sed -i -e 's/[^ \t]*adios[^ \t]* //g' ../Makefile.package - sed -i -e 's/-DLMP_ADIOS //g' ../Makefile.package - sed -i -e '/^adios_SYS.*$/d' ../Makefile.package -# sed -i -e '4 i \ -#adios_SYSINC='"${ADIOS_INC}"' -#' ../Makefile.package -# sed -i -e '5 i \ -#adios_SYSLIB='"${ADIOS_LIB}"' -#' ../Makefile.package -# sed -i -e '6 i \ -#adios_SYSPATH='"${ADIOS_DIR}"' -#' ../Makefile.package - sed -i -e 's|^PKG_INC =[ \t]*|&-DLMP_ADIOS |' ../Makefile.package - sed -i -e 's|^PKG_SYSINC =[ \t]*|&$(adios_SYSINC) |' ../Makefile.package - sed -i -e 's|^PKG_SYSLIB =[ \t]*|&$(adios_SYSLIB) |' ../Makefile.package - sed -i -e 's|^PKG_SYSPATH =[ \t]*|&$(adios_SYSPATH) |' ../Makefile.package + CONFIGSCRIPT=none + if ( test `which adios2-config` ) then + CONFIGSCRIPT=adios2-config + elif ( ! test -z "$ADIOS2_DIR" ) then + if ( test `which $ADIOS2_DIR/bin/adios2-config` ) then + CONFIGSCRIPT=$ADIOS2_DIR/bin/adios2-config + else + echo "ERROR: ADIOS2_DIR environment variable is set but" \ + "\$ADIOS2_DIR/bin/adios2-config does not exist" + fi + elif ( ! test -z "$ADIOS_DIR" ) then + if ( test `which $ADIOS_DIR/bin/adios2-config` ) then + CONFIGSCRIPT=$ADIOS_DIR/bin/adios2-config + else + echo "ERROR: ADIOS_DIR environment variable is set but" \ + "\$ADIOS_DIR/bin/adios2-config does not exist" + fi + else + echo "ERROR: ADIOS2_DIR environment variable must point to ADIOS 2.x" \ + "installation directory or adios2-config should be in PATH" fi - if (test -e ../Makefile.package.settings) then - sed -i -e '/^include.*adios.*$/d' ../Makefile.package.settings - # multiline form needed for BSD sed on Macs - sed -i -e '4 i \ -include ..\/..\/lib\/adios\/Makefile.lammps + if [ "$CONFIGSCRIPT" != "none" ]; then + ADIOS2_INC=`$CONFIGSCRIPT --cxx-flags` + ADIOS2_LIB=`$CONFIGSCRIPT --cxx-libs` + + echo "adios_SYSINC=${ADIOS2_INC} +adios_SYSLIB=${ADIOS2_LIB} +" > Makefile.lammps + + + if (test -e ../Makefile.package) then + sed -i -e 's/[^ \t]*adios[^ \t]* //g' ../Makefile.package + sed -i -e '/^adios_SYS.*$/d' ../Makefile.package + sed -i -e 's|^PKG_SYSINC =[ \t]*|&$(adios_SYSINC) |' ../Makefile.package + sed -i -e 's|^PKG_SYSLIB =[ \t]*|&$(adios_SYSLIB) |' ../Makefile.package + fi + + if (test -e ../Makefile.package.settings) then + sed -i -e '/^include.*ADIOS.*$/d' ../Makefile.package.settings + # multiline form needed for BSD sed on Macs + sed -i -e '4 i \ +include ../USER-ADIOS/Makefile.lammps ' ../Makefile.package.settings + fi fi elif (test $1 = 0) then if (test -e ../Makefile.package) then sed -i -e 's/[^ \t]*adios[^ \t]* //g' ../Makefile.package - sed -i -e 's/-DLMP_ADIOS //g' ../Makefile.package sed -i -e '/^adios_SYS.*$/d' ../Makefile.package fi if (test -e ../Makefile.package.settings) then - sed -i -e '/^include.*adios.*$/d' ../Makefile.package.settings + sed -i -e '/^include.*ADIOS.*$/d' ../Makefile.package.settings fi + rm -f Makefile.lammps + fi From 2de01cef45519adbc57edad6dd038aaa42fecef0 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 20 Feb 2019 16:21:07 -0500 Subject: [PATCH 09/16] Dump atom/adios and dump custom/adios command notifications --- doc/src/Commands_all.txt | 1 + doc/src/commands_list.txt | 1 + doc/src/dump.txt | 5 +- doc/src/dump_adios.txt | 73 ++++++++++++++++++++++++++++ examples/USER/adios/in.adios_balance | 1 + 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 doc/src/dump_adios.txt diff --git a/doc/src/Commands_all.txt b/doc/src/Commands_all.txt index f137ccffd9..e6f364f2c9 100644 --- a/doc/src/Commands_all.txt +++ b/doc/src/Commands_all.txt @@ -48,6 +48,7 @@ An alphabetic list of all general LAMMPS commands. "dimension"_dimension.html, "displace_atoms"_displace_atoms.html, "dump"_dump.html, +"dump adios"_dump_adios.html, "dump image"_dump_image.html, "dump_modify"_dump_modify.html, "dump movie"_dump_image.html, diff --git a/doc/src/commands_list.txt b/doc/src/commands_list.txt index 78fa9fbf87..9b3185bda8 100644 --- a/doc/src/commands_list.txt +++ b/doc/src/commands_list.txt @@ -32,6 +32,7 @@ Commands :h1 dimension displace_atoms dump + dump_adios dump_cfg_uef dump_h5md dump_image diff --git a/doc/src/dump.txt b/doc/src/dump.txt index 9999f5bbff..a776ff70fc 100644 --- a/doc/src/dump.txt +++ b/doc/src/dump.txt @@ -13,6 +13,7 @@ dump command :h3 "dump netcdf"_dump_netcdf.html command :h3 "dump image"_dump_image.html command :h3 "dump movie"_dump_image.html command :h3 +"dump adios"_dump_adios.html command :h3 [Syntax:] @@ -27,10 +28,12 @@ args = list of arguments for a particular style :l {atom} args = none {atom/gz} args = none {atom/mpiio} args = none + {atom/adios} args = none, discussed on "dump adios"_dump_adios.html doc page {cfg} args = same as {custom} args, see below {cfg/gz} args = same as {custom} args, see below {cfg/mpiio} args = same as {custom} args, see below {custom}, {custom/gz}, {custom/mpiio} args = see below + {custom/adios} args = same as {custom} args, discussed on "dump adios"_dump_adios.html doc page {dcd} args = none {h5md} args = discussed on "dump h5md"_dump_h5md.html doc page {image} args = discussed on "dump image"_dump_image.html doc page @@ -653,7 +656,7 @@ package"_Build_package.html doc page for more info. [Related commands:] -"dump h5md"_dump_h5md.html, "dump image"_dump_image.html, +"dump adios"_dump_adios.html "dump h5md"_dump_h5md.html, "dump image"_dump_image.html, "dump molfile"_dump_molfile.html, "dump_modify"_dump_modify.html, "undump"_undump.html diff --git a/doc/src/dump_adios.txt b/doc/src/dump_adios.txt new file mode 100644 index 0000000000..d6424d664a --- /dev/null +++ b/doc/src/dump_adios.txt @@ -0,0 +1,73 @@ +"LAMMPS WWW Site"_lws - "LAMMPS Documentation"_ld - "LAMMPS Commands"_lc :c + +:link(lws,http://lammps.sandia.gov) +:link(ld,Manual.html) +:link(lc,Commands_all.html) + +:line + +dump atoms/adios command :h3 +dump custom/adios command :h3 + +[Syntax:] + +dump ID group-ID atoms/adios N file.bp :pre +dump ID group-ID custom/adios N file.bp args :pre + +ID = user-assigned name for the dump :ulb,l +group-ID = ID of the group of atoms to be imaged :l +adios = style of dump command (other styles {atom} or {cfg} or {dcd} or {xtc} or {xyz} or {local} or {custom} are discussed on the "dump"_dump.html doc page) :l +N = dump every this many timesteps :l +file.bp = name of file/stream to write to :l +args = same options as in "{dump custom}"_dump.html command :l +:ule + + +[Examples:] + +dump adios1 all atom/adios 100 atoms.bp +dump 4a all custom/adios 100 dump_adios.bp id v_p x y z +dump 2 subgroup custom/adios 100 dump_adios.bp mass type xs ys zs vx vy vz :pre + +[Description:] + +Dump a snapshot of atom coordinates every N timesteps in the +"ADIOS"_adios based "BP" file format, or using different I/O solutions in ADIOS, +to a stream that can be read on-line by another program. +ADIOS-BP files are binary, portable and self-describing. + +:link(adios,https://github.com/ornladios/ADIOS2) + + +[Use from write_dump:] + +It is possible to use these dump styles with the +"write_dump"_write_dump.html command. In this case, the sub-intervals +must not be set at all. The write_dump command can be used to +create a new file at each individual dump. + +dump 4 all atom/adios 100 dump.bp +write_dump all atom/adios singledump.bp :pre + +:line + +[Restrictions:] + +The number of atoms per snapshot CAN change with the adios style. +When using the ADIOS tool 'bpls' to list the content of a .bp file, +bpls will print "__" for the size of the output table indicating that +its size is changing every step. + +The {atom/adios} and {custom/adios} dump styles are part of the USER-adios package. +They are only enabled if LAMMPS was built with that package. See the +"Build package"_Build_package.html doc page for more info. + + +:line + +[Related commands:] + +"dump"_dump.html, "dump_modify"_dump_modify.html, "undump"_undump.html + +:line + diff --git a/examples/USER/adios/in.adios_balance b/examples/USER/adios/in.adios_balance index b44870afbb..d89ae52b41 100644 --- a/examples/USER/adios/in.adios_balance +++ b/examples/USER/adios/in.adios_balance @@ -54,3 +54,4 @@ thermo_style custom step temp epair press f_10[3] f_10 thermo 100 run 200 +write_dump all atom/adios balance_atom_final.bp From a5f20305d49667d0815eed056064a55142694e96 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 20 Feb 2019 16:28:51 -0500 Subject: [PATCH 10/16] Add user-adios to PACKLIB and PACKEXT --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3e224cd6ab..f954d84e5d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -68,7 +68,7 @@ PACKUSER = user-adios user-atc user-awpmd user-bocs user-cgdna user-cgsdk user-c PACKLIB = compress gpu kim kokkos latte message mpiio mscg poems \ python voronoi \ - user-atc user-awpmd user-colvars user-h5md user-lb user-molfile \ + user-adios user-atc user-awpmd user-colvars user-h5md user-lb user-molfile \ user-netcdf user-plumed user-qmmm user-quip user-scafacos \ user-smd user-vtk @@ -77,7 +77,7 @@ PACKSYS = compress mpiio python user-lb PACKINT = gpu kokkos message poems user-atc user-awpmd user-colvars PACKEXT = kim latte mscg voronoi \ - user-h5md user-molfile user-netcdf user-plumed user-qmmm user-quip \ + user-adios user-h5md user-molfile user-netcdf user-plumed user-qmmm user-quip \ user-smd user-vtk PACKALL = $(PACKAGE) $(PACKUSER) From 3ad268739b003e6168eeb89920707a0b5c0393d6 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 20 Feb 2019 17:02:48 -0500 Subject: [PATCH 11/16] Add dump adios to PDF build. Update example config with better documentation --- doc/src/dump_adios.txt | 2 +- doc/src/lammps.book | 1 + examples/USER/adios/adios2_config.xml | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/src/dump_adios.txt b/doc/src/dump_adios.txt index d6424d664a..3d08d6c13a 100644 --- a/doc/src/dump_adios.txt +++ b/doc/src/dump_adios.txt @@ -55,7 +55,7 @@ write_dump all atom/adios singledump.bp :pre The number of atoms per snapshot CAN change with the adios style. When using the ADIOS tool 'bpls' to list the content of a .bp file, -bpls will print "__" for the size of the output table indicating that +bpls will print {__} for the size of the output table indicating that its size is changing every step. The {atom/adios} and {custom/adios} dump styles are part of the USER-adios package. diff --git a/doc/src/lammps.book b/doc/src/lammps.book index 02a55e3810..d15d287742 100644 --- a/doc/src/lammps.book +++ b/doc/src/lammps.book @@ -150,6 +150,7 @@ dielectric.html dimension.html displace_atoms.html dump.html +dump_adios.html dump_h5md.html dump_image.html dump_modify.html diff --git a/examples/USER/adios/adios2_config.xml b/examples/USER/adios/adios2_config.xml index 7462731d43..4b9cd9b9b5 100644 --- a/examples/USER/adios/adios2_config.xml +++ b/examples/USER/adios/adios2_config.xml @@ -6,8 +6,15 @@ + + + The 'substreams' parameter in BPFile controls how many + files on disk are created. This number should be proportional + to the number of servers in the parallel file system, + NOT to the number of processes. + substreams=1 is generally a very inefficient setting for large parallel runs. -->