From 0d8454ac25ab6337e91e4f097bbaa686d33127c3 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 25 Aug 2020 15:21:38 -0400 Subject: [PATCH 01/26] First version of Zstd dump atom --- cmake/Modules/FindZstd.cmake | 28 ++++ cmake/Modules/Packages/COMPRESS.cmake | 3 + src/COMPRESS/dump_atom_zstd.cpp | 221 ++++++++++++++++++++++++++ src/COMPRESS/dump_atom_zstd.h | 65 ++++++++ src/dump.cpp | 3 + 5 files changed, 320 insertions(+) create mode 100644 cmake/Modules/FindZstd.cmake create mode 100644 src/COMPRESS/dump_atom_zstd.cpp create mode 100644 src/COMPRESS/dump_atom_zstd.h diff --git a/cmake/Modules/FindZstd.cmake b/cmake/Modules/FindZstd.cmake new file mode 100644 index 0000000000..252767cb24 --- /dev/null +++ b/cmake/Modules/FindZstd.cmake @@ -0,0 +1,28 @@ +# Find Zstd library +# +# Zstd_INCLUDE_DIRS - where to find zstd.h, etc. +# Zstd_LIBRARIES - List of libraries when using libzstd +# Zstd_FOUND - True if libzstd is found. + +find_path(Zstd_INCLUDE_DIR NAMES zstd.h) +find_library(Zstd_LIBRARY NAMES zstd) + +# handle the QUIET and REQUIRED arguments and +# set Zstd_FOUND to TRUE if all variables are non-zero +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Zstd DEFAULT_MSG Zstd_LIBRARY Zstd_INCLUDE_DIR) + +# Copy the results to the output variables and target. +if(Zstd_FOUND) + set(Zstd_LIBRARIES ${Zstd_LIBRARY}) + set(Zstd_INCLUDE_DIRS ${Zstd_INCLUDE_DIR}) + + if(NOT TARGET Zstd::Zstd) + add_library(Zstd::Zstd UNKNOWN IMPORTED) + set_target_properties(Zstd::Zstd PROPERTIES + IMPORTED_LOCATION "${Zstd_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${Zstd_INCLUDE_DIR}") + endif() +endif() + +mark_as_advanced(Zstd_INCLUDE_DIR Zstd_LIBRARY) diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index 801a38a215..c92767692a 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -1,2 +1,5 @@ find_package(ZLIB REQUIRED) target_link_libraries(lammps PRIVATE ZLIB::ZLIB) + +find_package(Zstd REQUIRED) +target_link_libraries(lammps PRIVATE Zstd::Zstd) diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp new file mode 100644 index 0000000000..92021b860f --- /dev/null +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -0,0 +1,221 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "dump_atom_zstd.h" +#include "domain.h" +#include "error.h" +#include "update.h" + +#include +#include + +using namespace LAMMPS_NS; + +DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : + DumpAtom(lmp, narg, arg) +{ + cctx = nullptr; + zstdFp = nullptr; + fp = nullptr; + out_buffer_size = ZSTD_CStreamOutSize(); + out_buffer = new char[out_buffer_size]; + + if (!compressed) + error->all(FLERR,"Dump atom/zstd only writes compressed files"); +} + +/* ---------------------------------------------------------------------- */ + +DumpAtomZstd::~DumpAtomZstd() +{ + if(cctx && zstdFp) zstd_close(); + + delete [] out_buffer; + out_buffer = nullptr; + out_buffer_size = 0; +} + +/* ---------------------------------------------------------------------- + generic opening of a dump file + ASCII or binary or zstdipped + some derived classes override this function +------------------------------------------------------------------------- */ + +void DumpAtomZstd::openfile() +{ + // single file, already opened, so just return + + if (singlefile_opened) return; + if (multifile == 0) singlefile_opened = 1; + + // if one file per timestep, replace '*' with current timestep + + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + *ptr = '*'; + if (maxfiles > 0) { + if (numfiles < maxfiles) { + nameslist[numfiles] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[numfiles],filecurrent); + ++numfiles; + } else { + remove(nameslist[fileidx]); + delete[] nameslist[fileidx]; + nameslist[fileidx] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[fileidx],filecurrent); + fileidx = (fileidx + 1) % maxfiles; + } + } + } + + // each proc with filewriter = 1 opens a file + + if (filewriter) { + if (append_flag) { + zstdFp = fopen(filecurrent,"ab9"); + } else { + zstdFp = fopen(filecurrent,"wb9"); + } + + if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); + + cctx = ZSTD_createCCtx(); + + if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); + } else zstdFp = nullptr; + + // delete string with timestep replaced + + if (multifile) delete [] filecurrent; +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomZstd::write_header(bigint ndump) +{ + std::string header; + + if ((multiproc) || (!multiproc && me == 0)) { + if (unit_flag && !unit_count) { + ++unit_count; + header = fmt::format("ITEM: UNITS\n%s\n",update->unit_style); + } + + if (time_flag) { + header += fmt::format("ITEM: TIME\n%.16g\n", compute_time()); + } + + header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); + header += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); + if (domain->triclinic == 0) { + header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi); + } else { + header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz); + } + header += fmt::format("ITEM: ATOMS {}\n", columns); + + zstd_write(header.c_str(), header.length()); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomZstd::write_data(int n, double *mybuf) +{ + ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(input.pos < input.size); +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomZstd::write() +{ + DumpAtom::write(); + if (filewriter) { + if (multifile) { + zstd_close(); + } else { + if (flush_flag && zstdFp) { + zstd_flush(); + fflush(zstdFp); + } + } + } +} + +void DumpAtomZstd::zstd_write(const void * buffer, size_t length) { + ZSTD_inBuffer input = { buffer, length, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(input.pos < input.size); +} + +void DumpAtomZstd::zstd_flush() { + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_flush; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(remaining); +} + +void DumpAtomZstd::zstd_close() { + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_end; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(remaining); + + ZSTD_freeCCtx(cctx); + cctx = nullptr; + if (zstdFp) fclose(zstdFp); + zstdFp = nullptr; +} diff --git a/src/COMPRESS/dump_atom_zstd.h b/src/COMPRESS/dump_atom_zstd.h new file mode 100644 index 0000000000..b6cb252f13 --- /dev/null +++ b/src/COMPRESS/dump_atom_zstd.h @@ -0,0 +1,65 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(atom/zstd,DumpAtomZstd) + +#else + +#ifndef LMP_DUMP_ATOM_ZSTD_H +#define LMP_DUMP_ATOM_ZSTD_H + +#include "dump_atom.h" +#include +#include + +namespace LAMMPS_NS { + +class DumpAtomZstd : public DumpAtom { + public: + DumpAtomZstd(class LAMMPS *, int, char **); + virtual ~DumpAtomZstd(); + + protected: + ZSTD_CCtx * cctx; + FILE * zstdFp; + char * out_buffer; + size_t out_buffer_size; + + virtual void openfile(); + virtual void write_header(bigint); + virtual void write_data(int, double *); + virtual void write(); + + void zstd_write(const void * buffer, size_t length); + void zstd_flush(); + void zstd_close(); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Dump atom/zstd only writes compressed files + +The dump atom/zstd output file name must have a .zst suffix. + +E: Cannot open dump file + +Self-explanatory. + +*/ diff --git a/src/dump.cpp b/src/dump.cpp index 83b74f1bbc..ccf71c5a38 100644 --- a/src/dump.cpp +++ b/src/dump.cpp @@ -116,6 +116,7 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : Pointers(lmp) // check file suffixes // if ends in .bin = binary file // else if ends in .gz = gzipped text file + // else if ends in .zst = Zstd compressed text file // else ASCII text file fp = NULL; @@ -153,6 +154,8 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : Pointers(lmp) if (suffix > filename && strcmp(suffix,".bin") == 0) binary = 1; suffix = filename + strlen(filename) - strlen(".gz"); if (suffix > filename && strcmp(suffix,".gz") == 0) compressed = 1; + suffix = filename + strlen(filename) - strlen(".zst"); + if (suffix > filename && strcmp(suffix,".zst") == 0) compressed = 1; } /* ---------------------------------------------------------------------- */ From 5cb8e73655b7437729763499306c1089129debaf Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 25 Aug 2020 23:59:17 -0400 Subject: [PATCH 02/26] Add checksum and compression_level as dump atom/zstd options --- src/COMPRESS/dump_atom_zstd.cpp | 42 +++++++++++++++++++++++++++++---- src/COMPRESS/dump_atom_zstd.h | 5 ++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index 92021b860f..fb738b9fb7 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -15,6 +15,7 @@ #include "domain.h" #include "error.h" #include "update.h" +#include "force.h" #include #include @@ -30,6 +31,9 @@ DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : out_buffer_size = ZSTD_CStreamOutSize(); out_buffer = new char[out_buffer_size]; + checksum_flag = 1; + compression_level = 0; // = default + if (!compressed) error->all(FLERR,"Dump atom/zstd only writes compressed files"); } @@ -97,14 +101,16 @@ void DumpAtomZstd::openfile() if (filewriter) { if (append_flag) { - zstdFp = fopen(filecurrent,"ab9"); + zstdFp = fopen(filecurrent,"ab"); } else { - zstdFp = fopen(filecurrent,"wb9"); + zstdFp = fopen(filecurrent,"wb"); } if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); } else zstdFp = nullptr; @@ -180,7 +186,32 @@ void DumpAtomZstd::write() } } -void DumpAtomZstd::zstd_write(const void * buffer, size_t length) { +/* ---------------------------------------------------------------------- */ + +int DumpAtomZstd::modify_param(int narg, char **arg) +{ + int consumed = DumpAtom::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; + else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level <= 0) error->all(FLERR,"Illegal dump_modify command"); + return 2; + } + } + return consumed; +} + +/* ---------------------------------------------------------------------- */ + +void DumpAtomZstd::zstd_write(const void * buffer, size_t length) +{ ZSTD_inBuffer input = { buffer, length, 0 }; ZSTD_EndDirective mode = ZSTD_e_continue; @@ -203,7 +234,10 @@ void DumpAtomZstd::zstd_flush() { } while(remaining); } -void DumpAtomZstd::zstd_close() { +/* ---------------------------------------------------------------------- */ + +void DumpAtomZstd::zstd_close() +{ size_t remaining; ZSTD_inBuffer input = { nullptr, 0, 0 }; ZSTD_EndDirective mode = ZSTD_e_end; diff --git a/src/COMPRESS/dump_atom_zstd.h b/src/COMPRESS/dump_atom_zstd.h index b6cb252f13..feb1cc912c 100644 --- a/src/COMPRESS/dump_atom_zstd.h +++ b/src/COMPRESS/dump_atom_zstd.h @@ -32,6 +32,9 @@ class DumpAtomZstd : public DumpAtom { virtual ~DumpAtomZstd(); protected: + int compression_level; + int checksum_flag; + ZSTD_CCtx * cctx; FILE * zstdFp; char * out_buffer; @@ -42,6 +45,8 @@ class DumpAtomZstd : public DumpAtom { virtual void write_data(int, double *); virtual void write(); + virtual int modify_param(int, char **); + void zstd_write(const void * buffer, size_t length); void zstd_flush(); void zstd_close(); From e0439ac94f4b4941239e3219faf22dba5aa30e12 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 00:14:32 -0400 Subject: [PATCH 03/26] Add compression_level parameter to dump atom gz --- src/COMPRESS/dump_atom_gz.cpp | 26 ++++++++++++++++++++++++-- src/COMPRESS/dump_atom_gz.h | 3 +++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 9be8d17b06..870bf62cd0 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -15,8 +15,10 @@ #include "domain.h" #include "error.h" #include "update.h" +#include "force.h" #include +#include using namespace LAMMPS_NS; @@ -25,6 +27,8 @@ DumpAtomGZ::DumpAtomGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; + compression_level = 9; + if (!compressed) error->all(FLERR,"Dump atom/gz only writes compressed files"); } @@ -89,12 +93,15 @@ void DumpAtomGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { + std::string mode; if (append_flag) { - gzFp = gzopen(filecurrent,"ab9"); + mode = fmt::format("ab{}", compression_level); } else { - gzFp = gzopen(filecurrent,"wb9"); + mode = fmt::format("wb{}", compression_level); } + gzFp = gzopen(filecurrent, mode.c_str()); + if (gzFp == NULL) error->one(FLERR,"Cannot open dump file"); } else gzFp = NULL; @@ -156,3 +163,18 @@ void DumpAtomGZ::write() } } +/* ---------------------------------------------------------------------- */ + +int DumpAtomGZ::modify_param(int narg, char **arg) +{ + int consumed = DumpAtom::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level <= 0) error->all(FLERR,"Illegal dump_modify command"); + return 2; + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_atom_gz.h b/src/COMPRESS/dump_atom_gz.h index bd0ed792f8..0c0b95974f 100644 --- a/src/COMPRESS/dump_atom_gz.h +++ b/src/COMPRESS/dump_atom_gz.h @@ -31,12 +31,15 @@ class DumpAtomGZ : public DumpAtom { virtual ~DumpAtomGZ(); protected: + int compression_level; gzFile gzFp; // file pointer for the compressed output stream virtual void openfile(); virtual void write_header(bigint); virtual void write_data(int, double *); virtual void write(); + + virtual int modify_param(int, char **); }; } From 046fd3d14b3c5117c24ed08b264bd4b3d2753da4 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 16:03:18 -0400 Subject: [PATCH 04/26] Verify valid compression level ranges --- src/COMPRESS/dump_atom_gz.cpp | 8 +++++--- src/COMPRESS/dump_atom_zstd.cpp | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 870bf62cd0..792f6d414b 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -27,7 +27,7 @@ DumpAtomGZ::DumpAtomGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; - compression_level = 9; + compression_level = Z_BEST_COMPRESSION; if (!compressed) error->all(FLERR,"Dump atom/gz only writes compressed files"); @@ -171,8 +171,10 @@ int DumpAtomGZ::modify_param(int narg, char **arg) if(consumed == 0) { if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - compression_level = force->inumeric(FLERR,arg[1]); - if (compression_level <= 0) error->all(FLERR,"Illegal dump_modify command"); + int min_level = Z_DEFAULT_COMPRESSION; + int max_level = Z_BEST_COMPRESSION; + if (compression_level < 0 || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } } diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index fb738b9fb7..f4660bec84 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -201,7 +201,10 @@ int DumpAtomZstd::modify_param(int narg, char **arg) } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); compression_level = force->inumeric(FLERR,arg[1]); - if (compression_level <= 0) error->all(FLERR,"Illegal dump_modify command"); + int min_level = ZSTD_minCLevel(); + int max_level = ZSTD_maxCLevel(); + if (compression_level < min_level || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } } From 32aa35009b9d8a360dc307e316d1a11dec631353 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 16:04:19 -0400 Subject: [PATCH 05/26] Start tests for compressed dump styles --- src/COMPRESS/dump_atom_gz.cpp | 12 +-- unittest/formats/CMakeLists.txt | 9 ++ unittest/formats/test_dump_atom_gz.cpp | 123 +++++++++++++++++++++++++ unittest/testing/core.h | 1 + 4 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 unittest/formats/test_dump_atom_gz.cpp diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 792f6d414b..575b6a9f44 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -127,14 +127,14 @@ void DumpAtomGZ::write_header(bigint ndump) gzprintf(gzFp,BIGINT_FORMAT "\n",ndump); if (domain->triclinic == 0) { gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); - gzprintf(gzFp,"%g %g\n",boxxlo,boxxhi); - gzprintf(gzFp,"%g %g\n",boxylo,boxyhi); - gzprintf(gzFp,"%g %g\n",boxzlo,boxzhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi); } else { gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - gzprintf(gzFp,"%g %g %g\n",boxxlo,boxxhi,boxxy); - gzprintf(gzFp,"%g %g %g\n",boxylo,boxyhi,boxxz); - gzprintf(gzFp,"%g %g %g\n",boxzlo,boxzhi,boxyz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz); } gzprintf(gzFp,"ITEM: ATOMS %s\n",columns); } diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index d2b5bccf20..fa09a69149 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -29,6 +29,15 @@ target_link_libraries(test_dump_atom PRIVATE lammps GTest::GMock GTest::GTest) add_test(NAME DumpAtom COMMAND test_dump_atom WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpAtom PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") +if (PKG_COMPRESS) + find_program(GZIP_BINARY NAMES gzip REQUIRED) + add_executable(test_dump_atom_gz test_dump_atom_gz.cpp) + target_link_libraries(test_dump_atom_gz PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpAtomGZ COMMAND test_dump_atom_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") +endif() + add_executable(test_dump_custom test_dump_custom.cpp) target_link_libraries(test_dump_custom PRIVATE lammps GTest::GMock GTest::GTest) add_test(NAME DumpCustom COMMAND test_dump_custom WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/unittest/formats/test_dump_atom_gz.cpp b/unittest/formats/test_dump_atom_gz.cpp new file mode 100644 index 0000000000..c4ff89cf9f --- /dev/null +++ b/unittest/formats/test_dump_atom_gz.cpp @@ -0,0 +1,123 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +#include + +char * GZIP_BINARY = nullptr; + +using ::testing::Eq; + +class DumpAtomGZTest : public MeltTest { + std::string dump_style = "atom"; +public: + void enable_triclinic() { + if (!verbose) ::testing::internal::CaptureStdout(); + command("change_box all triclinic"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_dump(std::string dump_file, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id all {} 1 {}", dump_style, dump_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {}", dump_style, text_file)); + command(fmt::format("dump id1 all {} 1 {}", compression_style, compressed_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", GZIP_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + + +//------------------------------------------------------------------------------------------------- +// GZ compressed files +//------------------------------------------------------------------------------------------------- + +TEST_F(DumpAtomGZTest, compressed_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_run0.melt"; + auto compressed_file = "dump_compressed_run0.melt.gz"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + GZIP_BINARY = getenv("GZIP_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} diff --git a/unittest/testing/core.h b/unittest/testing/core.h index 73e5cfbbb9..aaaf65c44f 100644 --- a/unittest/testing/core.h +++ b/unittest/testing/core.h @@ -71,6 +71,7 @@ protected: { if (!verbose) ::testing::internal::CaptureStdout(); delete lmp; + lmp = nullptr; if (!verbose) ::testing::internal::GetCapturedStdout(); } }; From 007a43a5b2b05d4e4787ad4e8d7fa387502823f1 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 16:20:02 -0400 Subject: [PATCH 06/26] Add more atom/gz tests --- unittest/formats/test_dump_atom_gz.cpp | 129 +++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/unittest/formats/test_dump_atom_gz.cpp b/unittest/formats/test_dump_atom_gz.cpp index c4ff89cf9f..438098ce4c 100644 --- a/unittest/formats/test_dump_atom_gz.cpp +++ b/unittest/formats/test_dump_atom_gz.cpp @@ -97,6 +97,135 @@ TEST_F(DumpAtomGZTest, compressed_run0) delete_file(converted_file); } +TEST_F(DumpAtomGZTest, compressed_with_units_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_with_units_run0.melt"; + auto compressed_file = "dump_compressed_with_units_run0.melt.gz"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomGZTest, compressed_with_time_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_with_time_run0.melt"; + auto compressed_file = "dump_compressed_with_time_run0.melt.gz"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomGZTest, compressed_triclinic_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_run0.melt"; + auto compressed_file = "dump_compressed_tri_run0.melt.gz"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomGZTest, compressed_triclinic_with_units_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_units_run0.melt.gz"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomGZTest, compressed_triclinic_with_time_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_time_run0.melt.gz"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomGZTest, compressed_triclinic_with_image_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_image_run0.melt.gz"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "image yes", 0); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} int main(int argc, char **argv) { From 78a1b92503f01d8cc440fcd17a9c7e951305bec5 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 16:35:35 -0400 Subject: [PATCH 07/26] Add dump atom zstd tests --- src/COMPRESS/dump_atom_zstd.cpp | 4 +- unittest/formats/CMakeLists.txt | 10 + unittest/formats/test_dump_atom_zstd.cpp | 273 +++++++++++++++++++++++ 3 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 unittest/formats/test_dump_atom_zstd.cpp diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index f4660bec84..427aede016 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -129,11 +129,11 @@ void DumpAtomZstd::write_header(bigint ndump) if ((multiproc) || (!multiproc && me == 0)) { if (unit_flag && !unit_count) { ++unit_count; - header = fmt::format("ITEM: UNITS\n%s\n",update->unit_style); + header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style); } if (time_flag) { - header += fmt::format("ITEM: TIME\n%.16g\n", compute_time()); + header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time()); } header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index fa09a69149..27c8a449d0 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -36,6 +36,16 @@ if (PKG_COMPRESS) add_test(NAME DumpAtomGZ COMMAND test_dump_atom_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + + find_program(ZSTD_BINARY NAMES zstd) + + if (ZSTD_BINARY) + add_executable(test_dump_atom_zstd test_dump_atom_zstd.cpp) + target_link_libraries(test_dump_atom_zstd PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpAtomZstd COMMAND test_dump_atom_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpAtomZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpAtomZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") + endif() endif() add_executable(test_dump_custom test_dump_custom.cpp) diff --git a/unittest/formats/test_dump_atom_zstd.cpp b/unittest/formats/test_dump_atom_zstd.cpp new file mode 100644 index 0000000000..470634a185 --- /dev/null +++ b/unittest/formats/test_dump_atom_zstd.cpp @@ -0,0 +1,273 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +#include + +char * ZSTD_BINARY = nullptr; + +using ::testing::Eq; + +class DumpAtomZSTDTest : public MeltTest { + std::string dump_style = "atom"; +public: + void enable_triclinic() { + if (!verbose) ::testing::internal::CaptureStdout(); + command("change_box all triclinic"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_dump(std::string dump_file, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id all {} 1 {}", dump_style, dump_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {}", dump_style, text_file)); + command(fmt::format("dump id1 all {} 1 {}", compression_style, compressed_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", ZSTD_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + + +//------------------------------------------------------------------------------------------------- +// ZSTD compressed files +//------------------------------------------------------------------------------------------------- + +TEST_F(DumpAtomZSTDTest, compressed_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_run0.melt"; + auto compressed_file = "dump_compressed_run0.melt.zst"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_with_units_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_with_units_run0.melt"; + auto compressed_file = "dump_compressed_with_units_run0.melt.zst"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_with_time_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_with_time_run0.melt"; + auto compressed_file = "dump_compressed_with_time_run0.melt.zst"; + + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_triclinic_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_run0.melt"; + auto compressed_file = "dump_compressed_tri_run0.melt.zst"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_units_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_units_run0.melt.zst"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_time_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_time_run0.melt.zst"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_image_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_compressed_tri_with_image_run0.melt.zst"; + + enable_triclinic(); + generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "image yes", 0); + + // make sure file is closed + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + ZSTD_BINARY = getenv("ZSTD_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From e9fd8b3ec6f582f837eeb53417f619718984b580 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 19:44:55 -0400 Subject: [PATCH 08/26] Migrate changes to dump custom/gz and add tests --- src/COMPRESS/dump_custom_gz.cpp | 40 +++++-- src/COMPRESS/dump_custom_gz.h | 3 + unittest/formats/CMakeLists.txt | 7 ++ unittest/formats/test_dump_custom_gz.cpp | 140 +++++++++++++++++++++++ 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100644 unittest/formats/test_dump_custom_gz.cpp diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 58ce98ad06..13fa564e52 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -15,8 +15,10 @@ #include "domain.h" #include "error.h" #include "update.h" +#include "force.h" #include +#include using namespace LAMMPS_NS; @@ -25,6 +27,8 @@ DumpCustomGZ::DumpCustomGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; + compression_level = Z_BEST_COMPRESSION; + if (!compressed) error->all(FLERR,"Dump custom/gz only writes compressed files"); } @@ -91,12 +95,15 @@ void DumpCustomGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { + std::string mode; if (append_flag) { - gzFp = gzopen(filecurrent,"ab9"); + mode = fmt::format("ab{}", compression_level); } else { - gzFp = gzopen(filecurrent,"wb9"); + mode = fmt::format("wb{}", compression_level); } + gzFp = gzopen(filecurrent, mode.c_str()); + if (gzFp == NULL) error->one(FLERR,"Cannot open dump file"); } else gzFp = NULL; @@ -120,14 +127,14 @@ void DumpCustomGZ::write_header(bigint ndump) gzprintf(gzFp,BIGINT_FORMAT "\n",ndump); if (domain->triclinic == 0) { gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxylo,boxyhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi); } else { gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxxlo,boxxhi,boxxy); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxylo,boxyhi,boxxz); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxzlo,boxzhi,boxyz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz); } gzprintf(gzFp,"ITEM: ATOMS %s\n",columns); } @@ -156,3 +163,20 @@ void DumpCustomGZ::write() } } +/* ---------------------------------------------------------------------- */ + +int DumpCustomGZ::modify_param(int narg, char **arg) +{ + int consumed = DumpCustom::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int min_level = Z_DEFAULT_COMPRESSION; + int max_level = Z_BEST_COMPRESSION; + if (compression_level < 0 || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); + return 2; + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_custom_gz.h b/src/COMPRESS/dump_custom_gz.h index 3a75673ab5..184f3563f1 100644 --- a/src/COMPRESS/dump_custom_gz.h +++ b/src/COMPRESS/dump_custom_gz.h @@ -31,12 +31,15 @@ class DumpCustomGZ : public DumpCustom { virtual ~DumpCustomGZ(); protected: + int compression_level; gzFile gzFp; // file pointer for the compressed output stream virtual void openfile(); virtual void write_header(bigint); virtual void write_data(int, double *); virtual void write(); + + virtual int modify_param(int, char **); }; } diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index 27c8a449d0..ffab3e7d9d 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -31,12 +31,19 @@ set_tests_properties(DumpAtom PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS if (PKG_COMPRESS) find_program(GZIP_BINARY NAMES gzip REQUIRED) + add_executable(test_dump_atom_gz test_dump_atom_gz.cpp) target_link_libraries(test_dump_atom_gz PRIVATE lammps GTest::GMock GTest::GTest) add_test(NAME DumpAtomGZ COMMAND test_dump_atom_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpAtomGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + add_executable(test_dump_custom_gz test_dump_custom_gz.cpp) + target_link_libraries(test_dump_custom_gz PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpCustomGZ COMMAND test_dump_custom_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpCustomGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpCustomGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + find_program(ZSTD_BINARY NAMES zstd) if (ZSTD_BINARY) diff --git a/unittest/formats/test_dump_custom_gz.cpp b/unittest/formats/test_dump_custom_gz.cpp new file mode 100644 index 0000000000..c0be5f76c8 --- /dev/null +++ b/unittest/formats/test_dump_custom_gz.cpp @@ -0,0 +1,140 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +using ::testing::Eq; + +char * GZIP_BINARY = nullptr; + +class DumpCustomGZTest : public MeltTest { + std::string dump_style = "custom"; +public: + void enable_triclinic() { + if (!verbose) ::testing::internal::CaptureStdout(); + command("change_box all triclinic"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_dump(std::string dump_file, std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id all {} 1 {} {}", dump_style, dump_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", GZIP_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpCustomGZTest, compressed_run1) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_custom_text_run1.melt"; + auto compressed_file = "dump_custom_compressed_run1.melt.gz"; + auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpCustomGZTest, compressed_triclinic_run1) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_file = "dump_custom_tri_text_run1.melt"; + auto compressed_file = "dump_custom_tri_compressed_run1.melt.gz"; + auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; + + enable_triclinic(); + + generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + GZIP_BINARY = getenv("GZIP_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From ced78a72eb7d1cb5b99e51599fca7b5d92beecfa Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Wed, 26 Aug 2020 19:59:28 -0400 Subject: [PATCH 09/26] Add dump custom/zstd and tests --- src/COMPRESS/dump_custom_zstd.cpp | 258 +++++++++++++++++++++ src/COMPRESS/dump_custom_zstd.h | 70 ++++++ unittest/formats/CMakeLists.txt | 6 + unittest/formats/test_dump_custom_zstd.cpp | 140 +++++++++++ 4 files changed, 474 insertions(+) create mode 100644 src/COMPRESS/dump_custom_zstd.cpp create mode 100644 src/COMPRESS/dump_custom_zstd.h create mode 100644 unittest/formats/test_dump_custom_zstd.cpp diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp new file mode 100644 index 0000000000..16794bcafd --- /dev/null +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -0,0 +1,258 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "dump_custom_zstd.h" +#include "domain.h" +#include "error.h" +#include "update.h" +#include "force.h" + +#include +#include + +using namespace LAMMPS_NS; + +DumpCustomZstd::DumpCustomZstd(LAMMPS *lmp, int narg, char **arg) : + DumpCustom(lmp, narg, arg) +{ + cctx = nullptr; + zstdFp = nullptr; + fp = nullptr; + out_buffer_size = ZSTD_CStreamOutSize(); + out_buffer = new char[out_buffer_size]; + + checksum_flag = 1; + compression_level = 0; // = default + + if (!compressed) + error->all(FLERR,"Dump custom/zstd only writes compressed files"); +} + + +/* ---------------------------------------------------------------------- */ + +DumpCustomZstd::~DumpCustomZstd() +{ + if(cctx && zstdFp) zstd_close(); + + delete [] out_buffer; + out_buffer = nullptr; + out_buffer_size = 0; +} + + +/* ---------------------------------------------------------------------- + generic opening of a dump file + ASCII or binary or gzipped + some derived classes override this function +------------------------------------------------------------------------- */ + +void DumpCustomZstd::openfile() +{ + // single file, already opened, so just return + + if (singlefile_opened) return; + if (multifile == 0) singlefile_opened = 1; + + // if one file per timestep, replace '*' with current timestep + + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + *ptr = '*'; + if (maxfiles > 0) { + if (numfiles < maxfiles) { + nameslist[numfiles] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[numfiles],filecurrent); + ++numfiles; + } else { + remove(nameslist[fileidx]); + delete[] nameslist[fileidx]; + nameslist[fileidx] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[fileidx],filecurrent); + fileidx = (fileidx + 1) % maxfiles; + } + } + } + + // each proc with filewriter = 1 opens a file + + if (filewriter) { + if (append_flag) { + zstdFp = fopen(filecurrent,"ab"); + } else { + zstdFp = fopen(filecurrent,"wb"); + } + + if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); + + cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); + + if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); + } else zstdFp = nullptr; + + // delete string with timestep replaced + + if (multifile) delete [] filecurrent; +} + +void DumpCustomZstd::write_header(bigint ndump) +{ + std::string header; + + if ((multiproc) || (!multiproc && me == 0)) { + if (unit_flag && !unit_count) { + ++unit_count; + header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style); + } + + if (time_flag) { + header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time()); + } + + header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); + header += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); + if (domain->triclinic == 0) { + header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi); + } else { + header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz); + } + header += fmt::format("ITEM: ATOMS {}\n", columns); + + zstd_write(header.c_str(), header.length()); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomZstd::write_data(int n, double *mybuf) +{ + ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(input.pos < input.size); +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomZstd::write() +{ + DumpCustom::write(); + if (filewriter) { + if (multifile) { + zstd_close(); + } else { + if (flush_flag && zstdFp) { + zstd_flush(); + fflush(zstdFp); + } + } + } +} + +/* ---------------------------------------------------------------------- */ + +int DumpCustomZstd::modify_param(int narg, char **arg) +{ + int consumed = DumpCustom::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; + else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + compression_level = force->inumeric(FLERR,arg[1]); + int min_level = ZSTD_minCLevel(); + int max_level = ZSTD_maxCLevel(); + if (compression_level < min_level || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); + return 2; + } + } + return consumed; +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomZstd::zstd_write(const void * buffer, size_t length) +{ + ZSTD_inBuffer input = { buffer, length, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(input.pos < input.size); +} + +void DumpCustomZstd::zstd_flush() { + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_flush; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(remaining); +} + +/* ---------------------------------------------------------------------- */ + +void DumpCustomZstd::zstd_close() +{ + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_end; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, zstdFp); + } while(remaining); + + ZSTD_freeCCtx(cctx); + cctx = nullptr; + if (zstdFp) fclose(zstdFp); + zstdFp = nullptr; +} diff --git a/src/COMPRESS/dump_custom_zstd.h b/src/COMPRESS/dump_custom_zstd.h new file mode 100644 index 0000000000..4736567dec --- /dev/null +++ b/src/COMPRESS/dump_custom_zstd.h @@ -0,0 +1,70 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(custom/zstd,DumpCustomZstd) + +#else + +#ifndef LMP_DUMP_CUSTOM_ZSTD_H +#define LMP_DUMP_CUSTOM_ZSTD_H + +#include "dump_custom.h" +#include +#include + +namespace LAMMPS_NS { + +class DumpCustomZstd : public DumpCustom { + public: + DumpCustomZstd(class LAMMPS *, int, char **); + virtual ~DumpCustomZstd(); + + protected: + int compression_level; + int checksum_flag; + + ZSTD_CCtx * cctx; + FILE * zstdFp; + char * out_buffer; + size_t out_buffer_size; + + virtual void openfile(); + virtual void write_header(bigint); + virtual void write_data(int, double *); + virtual void write(); + + virtual int modify_param(int, char **); + + void zstd_write(const void * buffer, size_t length); + void zstd_flush(); + void zstd_close(); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Dump custom/zstd only writes compressed files + +The dump custom/zstd output file name must have a .zst suffix. + +E: Cannot open dump file + +Self-explanatory. + +*/ diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index ffab3e7d9d..039913ce71 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -52,6 +52,12 @@ if (PKG_COMPRESS) add_test(NAME DumpAtomZstd COMMAND test_dump_atom_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpAtomZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpAtomZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") + + add_executable(test_dump_custom_zstd test_dump_custom_zstd.cpp) + target_link_libraries(test_dump_custom_zstd PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpCustomZstd COMMAND test_dump_custom_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpCustomZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpCustomZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") endif() endif() diff --git a/unittest/formats/test_dump_custom_zstd.cpp b/unittest/formats/test_dump_custom_zstd.cpp new file mode 100644 index 0000000000..1b156b3f20 --- /dev/null +++ b/unittest/formats/test_dump_custom_zstd.cpp @@ -0,0 +1,140 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +using ::testing::Eq; + +char * ZSTD_BINARY = nullptr; + +class DumpCustomZstdTest : public MeltTest { + std::string dump_style = "custom"; +public: + void enable_triclinic() { + if (!verbose) ::testing::internal::CaptureStdout(); + command("change_box all triclinic"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_dump(std::string dump_file, std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id all {} 1 {} {}", dump_style, dump_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", ZSTD_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpCustomZstdTest, compressed_run1) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_custom_text_run1.melt"; + auto compressed_file = "dump_custom_compressed_run1.melt.zst"; + auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +TEST_F(DumpCustomZstdTest, compressed_triclinic_run1) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_file = "dump_custom_tri_text_run1.melt"; + auto compressed_file = "dump_custom_tri_compressed_run1.melt.zst"; + auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; + + enable_triclinic(); + + generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + ZSTD_BINARY = getenv("ZSTD_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From 2f629db3d43619cfdda58a4fa7818015e5e865b4 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Thu, 27 Aug 2020 11:59:57 -0400 Subject: [PATCH 10/26] Refactor Zstd dump styles --- src/COMPRESS/dump_atom_zstd.cpp | 131 +++++------------ src/COMPRESS/dump_atom_zstd.h | 19 +-- src/COMPRESS/dump_custom_zstd.cpp | 133 +++++------------ src/COMPRESS/dump_custom_zstd.h | 18 +-- src/COMPRESS/zstd_file_writer.cpp | 158 +++++++++++++++++++++ src/COMPRESS/zstd_file_writer.h | 52 +++++++ src/file_writer.h | 53 +++++++ unittest/formats/test_dump_atom_gz.cpp | 44 ++++-- unittest/formats/test_dump_atom_zstd.cpp | 30 ++-- unittest/formats/test_dump_custom_gz.cpp | 12 +- unittest/formats/test_dump_custom_zstd.cpp | 12 +- 11 files changed, 400 insertions(+), 262 deletions(-) create mode 100644 src/COMPRESS/zstd_file_writer.cpp create mode 100644 src/COMPRESS/zstd_file_writer.h create mode 100644 src/file_writer.h diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index 427aede016..30862ab854 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -11,29 +11,24 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #include "dump_atom_zstd.h" #include "domain.h" #include "error.h" #include "update.h" #include "force.h" -#include #include +#include using namespace LAMMPS_NS; DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : DumpAtom(lmp, narg, arg) { - cctx = nullptr; - zstdFp = nullptr; - fp = nullptr; - out_buffer_size = ZSTD_CStreamOutSize(); - out_buffer = new char[out_buffer_size]; - - checksum_flag = 1; - compression_level = 0; // = default - if (!compressed) error->all(FLERR,"Dump atom/zstd only writes compressed files"); } @@ -42,11 +37,6 @@ DumpAtomZstd::DumpAtomZstd(LAMMPS *lmp, int narg, char **arg) : DumpAtomZstd::~DumpAtomZstd() { - if(cctx && zstdFp) zstd_close(); - - delete [] out_buffer; - out_buffer = nullptr; - out_buffer_size = 0; } /* ---------------------------------------------------------------------- @@ -101,19 +91,15 @@ void DumpAtomZstd::openfile() if (filewriter) { if (append_flag) { - zstdFp = fopen(filecurrent,"ab"); - } else { - zstdFp = fopen(filecurrent,"wb"); + error->one(FLERR, "dump/zstd currently doesn't support append"); } - if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); - - cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); - - if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); - } else zstdFp = nullptr; + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } // delete string with timestep replaced @@ -151,7 +137,7 @@ void DumpAtomZstd::write_header(bigint ndump) } header += fmt::format("ITEM: ATOMS {}\n", columns); - zstd_write(header.c_str(), header.length()); + writer.write(header.c_str(), header.length()); } } @@ -159,14 +145,7 @@ void DumpAtomZstd::write_header(bigint ndump) void DumpAtomZstd::write_data(int n, double *mybuf) { - ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); + writer.write(mybuf, n); } /* ---------------------------------------------------------------------- */ @@ -176,11 +155,10 @@ void DumpAtomZstd::write() DumpAtom::write(); if (filewriter) { if (multifile) { - zstd_close(); + writer.close(); } else { - if (flush_flag && zstdFp) { - zstd_flush(); - fflush(zstdFp); + if (flush_flag && writer.isopen()) { + writer.flush(); } } } @@ -192,67 +170,22 @@ int DumpAtomZstd::modify_param(int narg, char **arg) { int consumed = DumpAtom::modify_param(narg, arg); if(consumed == 0) { - if (strcmp(arg[0],"checksum") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; - else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; - else error->all(FLERR,"Illegal dump_modify command"); - return 2; - } else if (strcmp(arg[0],"compression_level") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - compression_level = force->inumeric(FLERR,arg[1]); - int min_level = ZSTD_minCLevel(); - int max_level = ZSTD_maxCLevel(); - if (compression_level < min_level || compression_level > max_level) - error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); - return 2; + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); } } return consumed; } - -/* ---------------------------------------------------------------------- */ - -void DumpAtomZstd::zstd_write(const void * buffer, size_t length) -{ - ZSTD_inBuffer input = { buffer, length, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); -} - -void DumpAtomZstd::zstd_flush() { - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_flush; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomZstd::zstd_close() -{ - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_end; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); - - ZSTD_freeCCtx(cctx); - cctx = nullptr; - if (zstdFp) fclose(zstdFp); - zstdFp = nullptr; -} diff --git a/src/COMPRESS/dump_atom_zstd.h b/src/COMPRESS/dump_atom_zstd.h index feb1cc912c..d1bfd9d00f 100644 --- a/src/COMPRESS/dump_atom_zstd.h +++ b/src/COMPRESS/dump_atom_zstd.h @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #ifdef DUMP_CLASS DumpStyle(atom/zstd,DumpAtomZstd) @@ -21,8 +25,7 @@ DumpStyle(atom/zstd,DumpAtomZstd) #define LMP_DUMP_ATOM_ZSTD_H #include "dump_atom.h" -#include -#include +#include "zstd_file_writer.h" namespace LAMMPS_NS { @@ -32,13 +35,7 @@ class DumpAtomZstd : public DumpAtom { virtual ~DumpAtomZstd(); protected: - int compression_level; - int checksum_flag; - - ZSTD_CCtx * cctx; - FILE * zstdFp; - char * out_buffer; - size_t out_buffer_size; + ZstdFileWriter writer; virtual void openfile(); virtual void write_header(bigint); @@ -46,10 +43,6 @@ class DumpAtomZstd : public DumpAtom { virtual void write(); virtual int modify_param(int, char **); - - void zstd_write(const void * buffer, size_t length); - void zstd_flush(); - void zstd_close(); }; } diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index 16794bcafd..420425b050 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #include "dump_custom_zstd.h" #include "domain.h" #include "error.h" @@ -25,32 +29,16 @@ using namespace LAMMPS_NS; DumpCustomZstd::DumpCustomZstd(LAMMPS *lmp, int narg, char **arg) : DumpCustom(lmp, narg, arg) { - cctx = nullptr; - zstdFp = nullptr; - fp = nullptr; - out_buffer_size = ZSTD_CStreamOutSize(); - out_buffer = new char[out_buffer_size]; - - checksum_flag = 1; - compression_level = 0; // = default - if (!compressed) error->all(FLERR,"Dump custom/zstd only writes compressed files"); } - /* ---------------------------------------------------------------------- */ DumpCustomZstd::~DumpCustomZstd() { - if(cctx && zstdFp) zstd_close(); - - delete [] out_buffer; - out_buffer = nullptr; - out_buffer_size = 0; } - /* ---------------------------------------------------------------------- generic opening of a dump file ASCII or binary or gzipped @@ -103,25 +91,23 @@ void DumpCustomZstd::openfile() if (filewriter) { if (append_flag) { - zstdFp = fopen(filecurrent,"ab"); - } else { - zstdFp = fopen(filecurrent,"wb"); + error->one(FLERR, "dump/zstd currently doesn't support append"); } - if (zstdFp == nullptr) error->one(FLERR,"Cannot open dump file"); - - cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); - - if (cctx == nullptr) error->one(FLERR,"Cannot create Zstd context"); - } else zstdFp = nullptr; + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } // delete string with timestep replaced if (multifile) delete [] filecurrent; } +/* ---------------------------------------------------------------------- */ + void DumpCustomZstd::write_header(bigint ndump) { std::string header; @@ -151,7 +137,7 @@ void DumpCustomZstd::write_header(bigint ndump) } header += fmt::format("ITEM: ATOMS {}\n", columns); - zstd_write(header.c_str(), header.length()); + writer.write(header.c_str(), header.length()); } } @@ -159,14 +145,7 @@ void DumpCustomZstd::write_header(bigint ndump) void DumpCustomZstd::write_data(int n, double *mybuf) { - ZSTD_inBuffer input = { mybuf, (size_t)n, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); + writer.write(mybuf, n); } /* ---------------------------------------------------------------------- */ @@ -176,11 +155,10 @@ void DumpCustomZstd::write() DumpCustom::write(); if (filewriter) { if (multifile) { - zstd_close(); + writer.close(); } else { - if (flush_flag && zstdFp) { - zstd_flush(); - fflush(zstdFp); + if (flush_flag && writer.isopen()) { + writer.flush(); } } } @@ -192,67 +170,22 @@ int DumpCustomZstd::modify_param(int narg, char **arg) { int consumed = DumpCustom::modify_param(narg, arg); if(consumed == 0) { - if (strcmp(arg[0],"checksum") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - if (strcmp(arg[1],"yes") == 0) checksum_flag = 1; - else if (strcmp(arg[1],"no") == 0) checksum_flag = 0; - else error->all(FLERR,"Illegal dump_modify command"); - return 2; - } else if (strcmp(arg[0],"compression_level") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - compression_level = force->inumeric(FLERR,arg[1]); - int min_level = ZSTD_minCLevel(); - int max_level = ZSTD_maxCLevel(); - if (compression_level < min_level || compression_level > max_level) - error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); - return 2; + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); } } return consumed; } - -/* ---------------------------------------------------------------------- */ - -void DumpCustomZstd::zstd_write(const void * buffer, size_t length) -{ - ZSTD_inBuffer input = { buffer, length, 0 }; - ZSTD_EndDirective mode = ZSTD_e_continue; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(input.pos < input.size); -} - -void DumpCustomZstd::zstd_flush() { - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_flush; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomZstd::zstd_close() -{ - size_t remaining; - ZSTD_inBuffer input = { nullptr, 0, 0 }; - ZSTD_EndDirective mode = ZSTD_e_end; - - do { - ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; - remaining = ZSTD_compressStream2(cctx, &output, &input, mode); - fwrite(out_buffer, sizeof(char), output.pos, zstdFp); - } while(remaining); - - ZSTD_freeCCtx(cctx); - cctx = nullptr; - if (zstdFp) fclose(zstdFp); - zstdFp = nullptr; -} diff --git a/src/COMPRESS/dump_custom_zstd.h b/src/COMPRESS/dump_custom_zstd.h index 4736567dec..fbda89d490 100644 --- a/src/COMPRESS/dump_custom_zstd.h +++ b/src/COMPRESS/dump_custom_zstd.h @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #ifdef DUMP_CLASS DumpStyle(custom/zstd,DumpCustomZstd) @@ -21,7 +25,7 @@ DumpStyle(custom/zstd,DumpCustomZstd) #define LMP_DUMP_CUSTOM_ZSTD_H #include "dump_custom.h" -#include +#include "zstd_file_writer.h" #include namespace LAMMPS_NS { @@ -32,13 +36,7 @@ class DumpCustomZstd : public DumpCustom { virtual ~DumpCustomZstd(); protected: - int compression_level; - int checksum_flag; - - ZSTD_CCtx * cctx; - FILE * zstdFp; - char * out_buffer; - size_t out_buffer_size; + ZstdFileWriter writer; virtual void openfile(); virtual void write_header(bigint); @@ -46,10 +44,6 @@ class DumpCustomZstd : public DumpCustom { virtual void write(); virtual int modify_param(int, char **); - - void zstd_write(const void * buffer, size_t length); - void zstd_flush(); - void zstd_close(); }; } diff --git a/src/COMPRESS/zstd_file_writer.cpp b/src/COMPRESS/zstd_file_writer.cpp new file mode 100644 index 0000000000..eb63d4e767 --- /dev/null +++ b/src/COMPRESS/zstd_file_writer.cpp @@ -0,0 +1,158 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#include "zstd_file_writer.h" +#include +#include + +using namespace LAMMPS_NS; + +ZstdFileWriter::ZstdFileWriter() : FileWriter(), + fp(nullptr), + cctx(nullptr), + compression_level(0), + checksum_flag(1) +{ + out_buffer_size = ZSTD_CStreamOutSize(); + out_buffer = new char[out_buffer_size]; +} + +/* ---------------------------------------------------------------------- */ + +ZstdFileWriter::~ZstdFileWriter() +{ + close(); + + delete [] out_buffer; + out_buffer = nullptr; + out_buffer_size = 0; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::open(const std::string & path) +{ + if(isopen()) return; + + fp = fopen(path.c_str(), "wb"); + + if (!fp) { + throw FileWriterException(fmt::format("Could not open file '{}'", path)); + } + + cctx = ZSTD_createCCtx(); + + if (!cctx) { + fclose(fp); + fp = nullptr; + throw FileWriterException("Could not create Zstd context"); + } + + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, checksum_flag); +} + +/* ---------------------------------------------------------------------- */ + +size_t ZstdFileWriter::write(const void * buffer, size_t length) +{ + if(!isopen()) return 0; + + ZSTD_inBuffer input = { buffer, length, 0 }; + ZSTD_EndDirective mode = ZSTD_e_continue; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + size_t const remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(input.pos < input.size); + + return length; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::flush() +{ + if(!isopen()) return; + + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_flush; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(remaining); + + fflush(fp); +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::close() +{ + if(!isopen()) return; + + size_t remaining; + ZSTD_inBuffer input = { nullptr, 0, 0 }; + ZSTD_EndDirective mode = ZSTD_e_end; + + do { + ZSTD_outBuffer output = { out_buffer, out_buffer_size, 0 }; + remaining = ZSTD_compressStream2(cctx, &output, &input, mode); + fwrite(out_buffer, sizeof(char), output.pos, fp); + } while(remaining); + + ZSTD_freeCCtx(cctx); + cctx = nullptr; + fclose(fp); + fp = nullptr; +} + +/* ---------------------------------------------------------------------- */ + +bool ZstdFileWriter::isopen() const +{ + return fp && cctx; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::setCompressionLevel(int level) +{ + if (isopen()) + throw FileWriterException("Compression level can not be changed while file is open"); + + const int min_level = ZSTD_minCLevel(); + const int max_level = ZSTD_maxCLevel(); + + if(level < min_level || level > max_level) + throw FileWriterException(fmt::format("Compression level must in the range of [{}, {}]", min_level, max_level)); + + compression_level = level; +} + +/* ---------------------------------------------------------------------- */ + +void ZstdFileWriter::setChecksum(bool enabled) +{ + if (isopen()) + throw FileWriterException("Checksum flag can not be changed while file is open"); + checksum_flag = enabled ? 1 : 0; +} diff --git a/src/COMPRESS/zstd_file_writer.h b/src/COMPRESS/zstd_file_writer.h new file mode 100644 index 0000000000..8a974a8599 --- /dev/null +++ b/src/COMPRESS/zstd_file_writer.h @@ -0,0 +1,52 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifndef LMP_ZSTD_FILE_WRITER_H +#define LMP_ZSTD_FILE_WRITER_H + +#include "file_writer.h" +#include +#include +#include + +namespace LAMMPS_NS { + +class ZstdFileWriter : public FileWriter { + int compression_level; + int checksum_flag; + + ZSTD_CCtx * cctx; + FILE * fp; + char * out_buffer; + size_t out_buffer_size; +public: + ZstdFileWriter(); + virtual ~ZstdFileWriter(); + virtual void open(const std::string & path) override; + virtual void close() override; + virtual void flush() override; + virtual size_t write(const void * buffer, size_t length) override; + virtual bool isopen() const override; + + void setCompressionLevel(int level); + void setChecksum(bool enabled); +}; + + +} + +#endif diff --git a/src/file_writer.h b/src/file_writer.h new file mode 100644 index 0000000000..766ccc85e7 --- /dev/null +++ b/src/file_writer.h @@ -0,0 +1,53 @@ + +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifndef LMP_FILE_WRITER_H +#define LMP_FILE_WRITER_H + +#include + +namespace LAMMPS_NS { + +class FileWriter { +public: + FileWriter() = default; + virtual ~FileWriter() = default; + virtual void open(const std::string & path) = 0; + virtual void close() = 0; + virtual void flush() = 0; + virtual size_t write(const void * buffer, size_t length) = 0; + virtual bool isopen() const = 0; +}; + +class FileWriterException : public std::exception { + std::string message; +public: + FileWriterException(const std::string & msg) : message(msg) { + } + + ~FileWriterException() throw() { + } + + virtual const char * what() const throw() { + return message.c_str(); + } +}; + +} + +#endif diff --git a/unittest/formats/test_dump_atom_gz.cpp b/unittest/formats/test_dump_atom_gz.cpp index 438098ce4c..ab1f57cc1b 100644 --- a/unittest/formats/test_dump_atom_gz.cpp +++ b/unittest/formats/test_dump_atom_gz.cpp @@ -79,17 +79,19 @@ TEST_F(DumpAtomGZTest, compressed_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_run0.melt"; - auto compressed_file = "dump_compressed_run0.melt.gz"; + auto text_file = "dump_gz_text_run0.melt"; + auto compressed_file = "dump_gz_compressed_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); auto converted_file = convert_compressed_to_text(compressed_file); - ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_THAT(converted_file, Eq("dump_gz_compressed_run0.melt")); ASSERT_FILE_EXISTS(converted_file); ASSERT_FILE_EQUAL(text_file, converted_file); delete_file(text_file); @@ -101,11 +103,13 @@ TEST_F(DumpAtomGZTest, compressed_with_units_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_units_run0.melt"; - auto compressed_file = "dump_compressed_with_units_run0.melt.gz"; + auto text_file = "dump_gz_text_with_units_run0.melt"; + auto compressed_file = "dump_gz_compressed_with_units_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -122,11 +126,13 @@ TEST_F(DumpAtomGZTest, compressed_with_time_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_time_run0.melt"; - auto compressed_file = "dump_compressed_with_time_run0.melt.gz"; + auto text_file = "dump_gz_text_with_time_run0.melt"; + auto compressed_file = "dump_gz_compressed_with_time_run0.melt.gz"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -143,12 +149,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_run0.melt"; - auto compressed_file = "dump_compressed_tri_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -165,12 +173,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_units_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_units_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_units_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_units_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no units yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -187,12 +197,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_time_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_time_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_time_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_time_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "scale no time yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -209,12 +221,14 @@ TEST_F(DumpAtomGZTest, compressed_triclinic_with_image_run0) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_image_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_image_run0.melt.gz"; + auto text_file = "dump_gz_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_gz_compressed_tri_with_image_run0.melt.gz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/gz", "image yes", 0); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); diff --git a/unittest/formats/test_dump_atom_zstd.cpp b/unittest/formats/test_dump_atom_zstd.cpp index 470634a185..d588a803a6 100644 --- a/unittest/formats/test_dump_atom_zstd.cpp +++ b/unittest/formats/test_dump_atom_zstd.cpp @@ -79,8 +79,8 @@ TEST_F(DumpAtomZSTDTest, compressed_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_run0.melt"; - auto compressed_file = "dump_compressed_run0.melt.zst"; + auto text_file = "dump_zstd_text_run0.melt"; + auto compressed_file = "dump_zstd_compressed_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); @@ -92,7 +92,7 @@ TEST_F(DumpAtomZSTDTest, compressed_run0) auto converted_file = convert_compressed_to_text(compressed_file); - ASSERT_THAT(converted_file, Eq("dump_compressed_run0.melt")); + ASSERT_THAT(converted_file, Eq("dump_zstd_compressed_run0.melt")); ASSERT_FILE_EXISTS(converted_file); ASSERT_FILE_EQUAL(text_file, converted_file); delete_file(text_file); @@ -104,8 +104,8 @@ TEST_F(DumpAtomZSTDTest, compressed_with_units_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_units_run0.melt"; - auto compressed_file = "dump_compressed_with_units_run0.melt.zst"; + auto text_file = "dump_zstd_text_with_units_run0.melt"; + auto compressed_file = "dump_zstd_compressed_with_units_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); @@ -128,8 +128,8 @@ TEST_F(DumpAtomZSTDTest, compressed_with_time_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_with_time_run0.melt"; - auto compressed_file = "dump_compressed_with_time_run0.melt.zst"; + auto text_file = "dump_zstd_text_with_time_run0.melt"; + auto compressed_file = "dump_zstd_compressed_with_time_run0.melt.zst"; generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); @@ -152,8 +152,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_run0.melt"; - auto compressed_file = "dump_compressed_tri_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "", 0); @@ -177,8 +177,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_units_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_units_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_units_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_units_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_units_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no units yes", 0); @@ -202,8 +202,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_time_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_time_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_time_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_time_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_time_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "scale no time yes", 0); @@ -227,8 +227,8 @@ TEST_F(DumpAtomZSTDTest, compressed_triclinic_with_image_run0) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_text_tri_with_image_run0.melt"; - auto compressed_file = "dump_compressed_tri_with_image_run0.melt.zst"; + auto text_file = "dump_zstd_text_tri_with_image_run0.melt"; + auto compressed_file = "dump_zstd_compressed_tri_with_image_run0.melt.zst"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "atom/zstd", "image yes", 0); diff --git a/unittest/formats/test_dump_custom_gz.cpp b/unittest/formats/test_dump_custom_gz.cpp index c0be5f76c8..12e1eca289 100644 --- a/unittest/formats/test_dump_custom_gz.cpp +++ b/unittest/formats/test_dump_custom_gz.cpp @@ -73,12 +73,14 @@ TEST_F(DumpCustomGZTest, compressed_run1) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_text_run1.melt"; - auto compressed_file = "dump_custom_compressed_run1.melt.gz"; + auto text_file = "dump_custom_gz_text_run1.melt"; + auto compressed_file = "dump_custom_gz_compressed_run1.melt.gz"; auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -95,14 +97,16 @@ TEST_F(DumpCustomGZTest, compressed_triclinic_run1) { if(!GZIP_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_tri_text_run1.melt"; - auto compressed_file = "dump_custom_tri_compressed_run1.melt.gz"; + auto text_file = "dump_custom_gz_tri_text_run1.melt"; + auto compressed_file = "dump_custom_gz_tri_compressed_run1.melt.gz"; auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "custom/gz", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); diff --git a/unittest/formats/test_dump_custom_zstd.cpp b/unittest/formats/test_dump_custom_zstd.cpp index 1b156b3f20..69576c9584 100644 --- a/unittest/formats/test_dump_custom_zstd.cpp +++ b/unittest/formats/test_dump_custom_zstd.cpp @@ -73,12 +73,14 @@ TEST_F(DumpCustomZstdTest, compressed_run1) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_text_run1.melt"; - auto compressed_file = "dump_custom_compressed_run1.melt.zst"; + auto text_file = "dump_custom_zstd_text_run1.melt"; + auto compressed_file = "dump_custom_zstd_compressed_run1.melt.zst"; auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); @@ -95,14 +97,16 @@ TEST_F(DumpCustomZstdTest, compressed_triclinic_run1) { if(!ZSTD_BINARY) GTEST_SKIP(); - auto text_file = "dump_custom_tri_text_run1.melt"; - auto compressed_file = "dump_custom_tri_compressed_run1.melt.zst"; + auto text_file = "dump_custom_zstd_tri_text_run1.melt"; + auto compressed_file = "dump_custom_zstd_tri_compressed_run1.melt.zst"; auto fields = "id type proc x y z xs ys zs xsu ysu zsu vx vy vz fx fy fz"; enable_triclinic(); generate_text_and_compressed_dump(text_file, compressed_file, "custom/zstd", fields, "units yes", 1); + TearDown(); + ASSERT_FILE_EXISTS(text_file); ASSERT_FILE_EXISTS(compressed_file); From 5faca3aef084a7255aed9224974ceaeda11d65f0 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 13:36:42 -0400 Subject: [PATCH 11/26] Add tests for dump cfg/gz --- unittest/formats/CMakeLists.txt | 6 ++ unittest/formats/test_dump_cfg.cpp | 34 +++---- unittest/formats/test_dump_cfg_gz.cpp | 129 ++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 17 deletions(-) create mode 100644 unittest/formats/test_dump_cfg_gz.cpp diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index 039913ce71..cacbfb83af 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -44,6 +44,12 @@ if (PKG_COMPRESS) set_tests_properties(DumpCustomGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpCustomGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + add_executable(test_dump_cfg_gz test_dump_cfg_gz.cpp) + target_link_libraries(test_dump_cfg_gz PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpCfgGZ COMMAND test_dump_cfg_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpCfgGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpCfgGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + find_program(ZSTD_BINARY NAMES zstd) if (ZSTD_BINARY) diff --git a/unittest/formats/test_dump_cfg.cpp b/unittest/formats/test_dump_cfg.cpp index 2bf6be259d..8096bf1913 100644 --- a/unittest/formats/test_dump_cfg.cpp +++ b/unittest/formats/test_dump_cfg.cpp @@ -46,7 +46,7 @@ TEST_F(DumpCfgTest, invalid_options) TEST_F(DumpCfgTest, require_multifile) { - auto dump_file = "dump_melt_run.cfg"; + auto dump_file = "dump.melt.cfg_run.cfg"; auto fields = "mass type xs ys zs id proc procp1 x y z ix iy iz xu yu zu xsu ysu zsu vx vy vz fx fy fz"; if (!verbose) ::testing::internal::CaptureStdout(); @@ -60,58 +60,58 @@ TEST_F(DumpCfgTest, require_multifile) TEST_F(DumpCfgTest, run0) { - auto dump_file = "dump_cfg_run*.melt"; + auto dump_file = "dump_cfg_run*.melt.cfg"; auto fields = "mass type xs ys zs id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; generate_dump(dump_file, fields, "", 0); - ASSERT_FILE_EXISTS("dump_cfg_run0.melt"); - auto lines = read_lines("dump_cfg_run0.melt"); + ASSERT_FILE_EXISTS("dump_cfg_run0.melt.cfg"); + auto lines = read_lines("dump_cfg_run0.melt.cfg"); ASSERT_EQ(lines.size(), 124); ASSERT_THAT(lines[0], Eq("Number of particles = 32")); - delete_file("dump_cfg_run0.melt"); + delete_file("dump_cfg_run0.melt.cfg"); } TEST_F(DumpCfgTest, unwrap_run0) { - auto dump_file = "dump_cfg_unwrap_run*.melt"; + auto dump_file = "dump_cfg_unwrap_run*.melt.cfg"; auto fields = "mass type xsu ysu zsu id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; generate_dump(dump_file, fields, "", 0); - ASSERT_FILE_EXISTS("dump_cfg_unwrap_run0.melt"); - auto lines = read_lines("dump_cfg_unwrap_run0.melt"); + ASSERT_FILE_EXISTS("dump_cfg_unwrap_run0.melt.cfg"); + auto lines = read_lines("dump_cfg_unwrap_run0.melt.cfg"); ASSERT_EQ(lines.size(), 124); ASSERT_THAT(lines[0], Eq("Number of particles = 32")); - delete_file("dump_cfg_unwrap_run0.melt"); + delete_file("dump_cfg_unwrap_run0.melt.cfg"); } TEST_F(DumpCfgTest, no_buffer_run0) { - auto dump_file = "dump_cfg_no_buffer_run*.melt"; + auto dump_file = "dump_cfg_no_buffer_run*.melt.cfg"; auto fields = "mass type xsu ysu zsu id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; generate_dump(dump_file, fields, "buffer no", 0); - ASSERT_FILE_EXISTS("dump_cfg_no_buffer_run0.melt"); - auto lines = read_lines("dump_cfg_no_buffer_run0.melt"); + ASSERT_FILE_EXISTS("dump_cfg_no_buffer_run0.melt.cfg"); + auto lines = read_lines("dump_cfg_no_buffer_run0.melt.cfg"); ASSERT_EQ(lines.size(), 124); ASSERT_THAT(lines[0], Eq("Number of particles = 32")); - delete_file("dump_cfg_no_buffer_run0.melt"); + delete_file("dump_cfg_no_buffer_run0.melt.cfg"); } TEST_F(DumpCfgTest, no_unwrap_no_buffer_run0) { - auto dump_file = "dump_cfg_no_unwrap_no_buffer_run*.melt"; + auto dump_file = "dump_cfg_no_unwrap_no_buffer_run*.melt.cfg"; auto fields = "mass type xs ys zs id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; generate_dump(dump_file, fields, "buffer no", 0); - ASSERT_FILE_EXISTS("dump_cfg_no_unwrap_no_buffer_run0.melt"); - auto lines = read_lines("dump_cfg_no_unwrap_no_buffer_run0.melt"); + ASSERT_FILE_EXISTS("dump_cfg_no_unwrap_no_buffer_run0.melt.cfg"); + auto lines = read_lines("dump_cfg_no_unwrap_no_buffer_run0.melt.cfg"); ASSERT_EQ(lines.size(), 124); ASSERT_THAT(lines[0], Eq("Number of particles = 32")); - delete_file("dump_cfg_no_unwrap_no_buffer_run0.melt"); + delete_file("dump_cfg_no_unwrap_no_buffer_run0.melt.cfg"); } diff --git a/unittest/formats/test_dump_cfg_gz.cpp b/unittest/formats/test_dump_cfg_gz.cpp new file mode 100644 index 0000000000..f04b1db6e2 --- /dev/null +++ b/unittest/formats/test_dump_cfg_gz.cpp @@ -0,0 +1,129 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * GZIP_BINARY = nullptr; + +using ::testing::Eq; + +class DumpCfgGZTest : public MeltTest { + std::string dump_style = "cfg"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", GZIP_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpCfgGZTest, compressed_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_files = "dump_cfg_gz_text_run*.melt.cfg"; + auto compressed_files = "dump_cfg_gz_compressed_run*.melt.cfg.gz"; + auto text_file = "dump_cfg_gz_text_run0.melt.cfg"; + auto compressed_file = "dump_cfg_gz_compressed_run0.melt.cfg.gz"; + auto fields = "mass type xs ys zs id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_files, compressed_files, "cfg/gz", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + + +TEST_F(DumpCfgGZTest, compressed_unwrap_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_files = "dump_cfg_unwrap_gz_text_run*.melt.cfg"; + auto compressed_files = "dump_cfg_unwrap_gz_compressed_run*.melt.cfg.gz"; + auto text_file = "dump_cfg_unwrap_gz_text_run0.melt.cfg"; + auto compressed_file = "dump_cfg_unwrap_gz_compressed_run0.melt.cfg.gz"; + auto fields = "mass type xsu ysu zsu id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_files, compressed_files, "cfg/gz", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + GZIP_BINARY = getenv("GZIP_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From a922355e1965cb4d5d8cbd44481d4960fb266336 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 14:20:36 -0400 Subject: [PATCH 12/26] Add compression_level parameter to dump cfg/gz --- src/COMPRESS/dump_cfg_gz.cpp | 28 ++++++++++++++++++++++++++-- src/COMPRESS/dump_cfg_gz.h | 3 +++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index aaeb878584..83c698a9cc 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -18,6 +18,7 @@ #include "update.h" #include +#include using namespace LAMMPS_NS; #define UNWRAPEXPAND 10.0 @@ -27,6 +28,8 @@ DumpCFGGZ::DumpCFGGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; + compression_level = Z_BEST_COMPRESSION; + if (!compressed) error->all(FLERR,"Dump cfg/gz only writes compressed files"); } @@ -93,12 +96,15 @@ void DumpCFGGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { + std::string mode; if (append_flag) { - gzFp = gzopen(filecurrent,"ab9"); + mode = fmt::format("ab{}", compression_level); } else { - gzFp = gzopen(filecurrent,"wb9"); + mode = fmt::format("wb{}", compression_level); } + gzFp = gzopen(filecurrent, mode.c_str()); + if (gzFp == NULL) error->one(FLERR,"Cannot open dump file"); } else gzFp = NULL; @@ -164,3 +170,21 @@ void DumpCFGGZ::write() } } +/* ---------------------------------------------------------------------- */ + +int DumpCFGGZ::modify_param(int narg, char **arg) +{ + int consumed = DumpCFG::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int min_level = Z_DEFAULT_COMPRESSION; + int max_level = Z_BEST_COMPRESSION; + if (compression_level < 0 || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); + return 2; + } + } + return consumed; +} + diff --git a/src/COMPRESS/dump_cfg_gz.h b/src/COMPRESS/dump_cfg_gz.h index aefde431cf..0c6ed24f06 100644 --- a/src/COMPRESS/dump_cfg_gz.h +++ b/src/COMPRESS/dump_cfg_gz.h @@ -31,12 +31,15 @@ class DumpCFGGZ : public DumpCFG { virtual ~DumpCFGGZ(); protected: + int compression_level; gzFile gzFp; // file pointer for the compressed output stream virtual void openfile(); virtual void write_header(bigint); virtual void write_data(int, double *); virtual void write(); + + virtual int modify_param(int, char **); }; } From 99b83333c98adf56fbf143b02dd53f661d893e9a Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 14:20:52 -0400 Subject: [PATCH 13/26] Add dump cfg/zstd --- src/COMPRESS/dump_cfg_zstd.cpp | 190 ++++++++++++++++++++++++ src/COMPRESS/dump_cfg_zstd.h | 59 ++++++++ unittest/formats/CMakeLists.txt | 6 + unittest/formats/test_dump_cfg_zstd.cpp | 129 ++++++++++++++++ 4 files changed, 384 insertions(+) create mode 100644 src/COMPRESS/dump_cfg_zstd.cpp create mode 100644 src/COMPRESS/dump_cfg_zstd.h create mode 100644 unittest/formats/test_dump_cfg_zstd.cpp diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp new file mode 100644 index 0000000000..ac746df431 --- /dev/null +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -0,0 +1,190 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "dump_cfg_zstd.h" +#include "atom.h" +#include "domain.h" +#include "error.h" +#include "update.h" +#include "force.h" + +#include +#include + +using namespace LAMMPS_NS; +#define UNWRAPEXPAND 10.0 + +DumpCFGZstd::DumpCFGZstd(LAMMPS *lmp, int narg, char **arg) : + DumpCFG(lmp, narg, arg) +{ + if (!compressed) + error->all(FLERR,"Dump cfg/zstd only writes compressed files"); +} + + +/* ---------------------------------------------------------------------- */ + +DumpCFGZstd::~DumpCFGZstd() +{ +} + + +/* ---------------------------------------------------------------------- + generic opening of a dump file + ASCII or binary or zstdipped + some derived classes override this function +------------------------------------------------------------------------- */ + +void DumpCFGZstd::openfile() +{ + // single file, already opened, so just return + + if (singlefile_opened) return; + if (multifile == 0) singlefile_opened = 1; + + // if one file per timestep, replace '*' with current timestep + + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + *ptr = '*'; + if (maxfiles > 0) { + if (numfiles < maxfiles) { + nameslist[numfiles] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[numfiles],filecurrent); + ++numfiles; + } else { + remove(nameslist[fileidx]); + delete[] nameslist[fileidx]; + nameslist[fileidx] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[fileidx],filecurrent); + fileidx = (fileidx + 1) % maxfiles; + } + } + } + + // each proc with filewriter = 1 opens a file + + if (filewriter) { + if (append_flag) { + error->one(FLERR, "dump cfg/zstd currently doesn't support append"); + } + + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + + // delete string with timestep replaced + + if (multifile) delete [] filecurrent; +} + +/* ---------------------------------------------------------------------- */ + +void DumpCFGZstd::write_header(bigint n) +{ + // set scale factor used by AtomEye for CFG viz + // default = 1.0 + // for peridynamics, set to pre-computed PD scale factor + // so PD particles mimic C atoms + // for unwrapped coords, set to UNWRAPEXPAND (10.0) + // so molecules are not split across periodic box boundaries + + double scale = 1.0; + if (atom->peri_flag) scale = atom->pdscale; + else if (unwrapflag == 1) scale = UNWRAPEXPAND; + + std::string header = fmt::format("Number of particles = {}\n", n); + header += fmt::format("A = {0:g} Angstrom (basic length-scale)\n", scale); + header += fmt::format("H0(1,1) = {0:g} A\n",domain->xprd); + header += fmt::format("H0(1,2) = 0 A \n"); + header += fmt::format("H0(1,3) = 0 A \n"); + header += fmt::format("H0(2,1) = {0:g} A \n",domain->xy); + header += fmt::format("H0(2,2) = {0:g} A\n",domain->yprd); + header += fmt::format("H0(2,3) = 0 A \n"); + header += fmt::format("H0(3,1) = {0:g} A \n",domain->xz); + header += fmt::format("H0(3,2) = {0:g} A \n",domain->yz); + header += fmt::format("H0(3,3) = {0:g} A\n",domain->zprd); + header += fmt::format(".NO_VELOCITY.\n"); + header += fmt::format("entry_count = {}\n",nfield-2); + for (int i = 0; i < nfield-5; i++) + header += fmt::format("auxiliary[{}] = {}\n",i,auxname[i]); + + writer.write(header.c_str(), header.length()); +} + +/* ---------------------------------------------------------------------- */ + +void DumpCFGZstd::write_data(int n, double *mybuf) +{ + writer.write(mybuf, n); +} + +/* ---------------------------------------------------------------------- */ + +void DumpCFGZstd::write() +{ + DumpCFG::write(); + if (filewriter) { + if (multifile) { + writer.close(); + } else { + if (flush_flag && writer.isopen()) { + writer.flush(); + } + } + } +} + +/* ---------------------------------------------------------------------- */ + +int DumpCFGZstd::modify_param(int narg, char **arg) +{ + int consumed = DumpCFG::modify_param(narg, arg); + if(consumed == 0) { + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_cfg_zstd.h b/src/COMPRESS/dump_cfg_zstd.h new file mode 100644 index 0000000000..71e06b0529 --- /dev/null +++ b/src/COMPRESS/dump_cfg_zstd.h @@ -0,0 +1,59 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(cfg/zstd,DumpCFGZstd) + +#else + +#ifndef LMP_DUMP_CFG_ZSTD_H +#define LMP_DUMP_CFG_ZSTD_H + +#include "dump_cfg.h" +#include "zstd_file_writer.h" + +namespace LAMMPS_NS { + +class DumpCFGZstd : public DumpCFG { + public: + DumpCFGZstd(class LAMMPS *, int, char **); + virtual ~DumpCFGZstd(); + + protected: + ZstdFileWriter writer; + + virtual void openfile(); + virtual void write_header(bigint); + virtual void write_data(int, double *); + virtual void write(); + + virtual int modify_param(int, char **); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Dump cfg/zstd only writes compressed files + +The dump cfg/zstd output file name must have a .zstd suffix. + +E: Cannot open dump file + +Self-explanatory. + +*/ diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index cacbfb83af..8a07bee0e8 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -64,6 +64,12 @@ if (PKG_COMPRESS) add_test(NAME DumpCustomZstd COMMAND test_dump_custom_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpCustomZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpCustomZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") + + add_executable(test_dump_cfg_zstd test_dump_cfg_zstd.cpp) + target_link_libraries(test_dump_cfg_zstd PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpCfgZstd COMMAND test_dump_cfg_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpCfgZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpCfgZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") endif() endif() diff --git a/unittest/formats/test_dump_cfg_zstd.cpp b/unittest/formats/test_dump_cfg_zstd.cpp new file mode 100644 index 0000000000..9ac84577cb --- /dev/null +++ b/unittest/formats/test_dump_cfg_zstd.cpp @@ -0,0 +1,129 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * ZSTD_BINARY = nullptr; + +using ::testing::Eq; + +class DumpCfgZstdTest : public MeltTest { + std::string dump_style = "cfg"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", ZSTD_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpCfgZstdTest, compressed_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_files = "dump_cfg_zstd_text_run*.melt.cfg"; + auto compressed_files = "dump_cfg_zstd_compressed_run*.melt.cfg.zst"; + auto text_file = "dump_cfg_zstd_text_run0.melt.cfg"; + auto compressed_file = "dump_cfg_zstd_compressed_run0.melt.cfg.zst"; + auto fields = "mass type xs ys zs id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_files, compressed_files, "cfg/gz", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + + +TEST_F(DumpCfgZstdTest, compressed_unwrap_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_files = "dump_cfg_unwrap_zstd_text_run*.melt.cfg"; + auto compressed_files = "dump_cfg_unwrap_zstd_compressed_run*.melt.cfg.zst"; + auto text_file = "dump_cfg_unwrap_zstd_text_run0.melt.cfg"; + auto compressed_file = "dump_cfg_unwrap_zstd_compressed_run0.melt.cfg.zst"; + auto fields = "mass type xsu ysu zsu id proc procp1 x y z ix iy iz vx vy vz fx fy fz"; + + generate_text_and_compressed_dump(text_files, compressed_files, "cfg/zstd", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + ZSTD_BINARY = getenv("ZSTD_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From 443a81715218683507f0dfe7a685b58f58cf144e Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 14:54:10 -0400 Subject: [PATCH 14/26] Add tests for dump local/gz and xyz/gz --- src/COMPRESS/dump_local_gz.cpp | 14 ++-- src/dump_local.cpp | 12 +-- unittest/formats/CMakeLists.txt | 12 +++ unittest/formats/test_dump_local_gz.cpp | 106 ++++++++++++++++++++++++ unittest/formats/test_dump_xyz_gz.cpp | 101 ++++++++++++++++++++++ 5 files changed, 232 insertions(+), 13 deletions(-) create mode 100644 unittest/formats/test_dump_local_gz.cpp create mode 100644 unittest/formats/test_dump_xyz_gz.cpp diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index c27c795f75..0642e52b2b 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -116,18 +116,18 @@ void DumpLocalGZ::write_header(bigint ndump) gzprintf(gzFp,"ITEM: TIMESTEP\n"); gzprintf(gzFp,BIGINT_FORMAT "\n",update->ntimestep); - gzprintf(gzFp,"ITEM: NUMBER OF ATOMS\n"); + gzprintf(gzFp,"ITEM: NUMBER OF ENTRIES\n"); gzprintf(gzFp,BIGINT_FORMAT "\n",ndump); - if (domain->triclinic == 0) { - gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxylo,boxyhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); - } else { + if (domain->triclinic) { gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxxlo,boxxhi,boxxy); gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxylo,boxyhi,boxxz); gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxzlo,boxzhi,boxyz); + } else { + gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); + gzprintf(gzFp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); + gzprintf(gzFp,"%-1.16g %-1.16g\n",boxylo,boxyhi); + gzprintf(gzFp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); } gzprintf(gzFp,"ITEM: %s %s\n",label,columns); } diff --git a/src/dump_local.cpp b/src/dump_local.cpp index 21a96d1e8a..103836b8a2 100644 --- a/src/dump_local.cpp +++ b/src/dump_local.cpp @@ -268,14 +268,14 @@ void DumpLocal::write_header(bigint ndump) fprintf(fp,BIGINT_FORMAT "\n",ndump); if (domain->triclinic) { fprintf(fp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - fprintf(fp,"%g %g %g\n",boxxlo,boxxhi,boxxy); - fprintf(fp,"%g %g %g\n",boxylo,boxyhi,boxxz); - fprintf(fp,"%g %g %g\n",boxzlo,boxzhi,boxyz); + fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxxlo,boxxhi,boxxy); + fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxylo,boxyhi,boxxz); + fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxzlo,boxzhi,boxyz); } else { fprintf(fp,"ITEM: BOX BOUNDS %s\n",boundstr); - fprintf(fp,"%g %g\n",boxxlo,boxxhi); - fprintf(fp,"%g %g\n",boxylo,boxyhi); - fprintf(fp,"%g %g\n",boxzlo,boxzhi); + fprintf(fp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); + fprintf(fp,"%-1.16g %-1.16g\n",boxylo,boxyhi); + fprintf(fp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); } fprintf(fp,"ITEM: %s %s\n",label,columns); } diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index 8a07bee0e8..80c09d691a 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -50,6 +50,18 @@ if (PKG_COMPRESS) set_tests_properties(DumpCfgGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpCfgGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + add_executable(test_dump_xyz_gz test_dump_xyz_gz.cpp) + target_link_libraries(test_dump_xyz_gz PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpXYZGZ COMMAND test_dump_xyz_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpXYZGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpXYZGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + + add_executable(test_dump_local_gz test_dump_local_gz.cpp) + target_link_libraries(test_dump_local_gz PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpLocalGZ COMMAND test_dump_local_gz WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpLocalGZ PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpLocalGZ PROPERTIES ENVIRONMENT "GZIP_BINARY=${GZIP_BINARY}") + find_program(ZSTD_BINARY NAMES zstd) if (ZSTD_BINARY) diff --git a/unittest/formats/test_dump_local_gz.cpp b/unittest/formats/test_dump_local_gz.cpp new file mode 100644 index 0000000000..e3d176be19 --- /dev/null +++ b/unittest/formats/test_dump_local_gz.cpp @@ -0,0 +1,106 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * GZIP_BINARY = nullptr; + +using ::testing::Eq; + +class DumpLocalGZTest : public MeltTest { + std::string dump_style = "local"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", GZIP_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpLocalGZTest, compressed_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + if (!verbose) ::testing::internal::CaptureStdout(); + command("compute comp all pair/local dist eng"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + + auto text_files = "dump_local_gz_text_run*.melt.local"; + auto compressed_files = "dump_local_gz_compressed_run*.melt.local.gz"; + auto text_file = "dump_local_gz_text_run0.melt.local"; + auto compressed_file = "dump_local_gz_compressed_run0.melt.local.gz"; + auto fields = "index c_comp[1]"; + + generate_text_and_compressed_dump(text_files, compressed_files, "local/gz", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + GZIP_BINARY = getenv("GZIP_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} diff --git a/unittest/formats/test_dump_xyz_gz.cpp b/unittest/formats/test_dump_xyz_gz.cpp new file mode 100644 index 0000000000..f57067f8e6 --- /dev/null +++ b/unittest/formats/test_dump_xyz_gz.cpp @@ -0,0 +1,101 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * GZIP_BINARY = nullptr; + +using ::testing::Eq; + +class DumpXYZGZTest : public MeltTest { + std::string dump_style = "xyz"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {}", dump_style, text_file)); + command(fmt::format("dump id1 all {} 1 {}", compression_style, compressed_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", GZIP_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpXYZGZTest, compressed_run0) +{ + if(!GZIP_BINARY) GTEST_SKIP(); + + auto text_files = "dump_xyz_gz_text_run*.melt.xyz"; + auto compressed_files = "dump_xyz_gz_compressed_run*.melt.xyz.gz"; + auto text_file = "dump_xyz_gz_text_run0.melt.xyz"; + auto compressed_file = "dump_xyz_gz_compressed_run0.melt.xyz.gz"; + + generate_text_and_compressed_dump(text_files, compressed_files, "xyz/gz", "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + GZIP_BINARY = getenv("GZIP_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From a0f82a4b18f86a690ed2428631d287631a8c838a Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 15:00:57 -0400 Subject: [PATCH 15/26] Add compression_level parameter to dump xyz/gz and local/gz --- src/COMPRESS/dump_local_gz.cpp | 27 +++++++++++++++++++++++++-- src/COMPRESS/dump_local_gz.h | 3 +++ src/COMPRESS/dump_xyz_gz.cpp | 28 ++++++++++++++++++++++++++-- src/COMPRESS/dump_xyz_gz.h | 3 +++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index 0642e52b2b..280b66bb2a 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -17,6 +17,7 @@ #include "update.h" #include +#include using namespace LAMMPS_NS; @@ -25,6 +26,8 @@ DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; + compression_level = Z_BEST_COMPRESSION; + if (!compressed) error->all(FLERR,"Dump local/gz only writes compressed files"); } @@ -91,12 +94,15 @@ void DumpLocalGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { + std::string mode; if (append_flag) { - gzFp = gzopen(filecurrent,"ab9"); + mode = fmt::format("ab{}", compression_level); } else { - gzFp = gzopen(filecurrent,"wb9"); + mode = fmt::format("wb{}", compression_level); } + gzFp = gzopen(filecurrent, mode.c_str()); + if (gzFp == NULL) error->one(FLERR,"Cannot open dump file"); } else gzFp = NULL; @@ -171,3 +177,20 @@ void DumpLocalGZ::write() } } +/* ---------------------------------------------------------------------- */ + +int DumpLocalGZ::modify_param(int narg, char **arg) +{ + int consumed = DumpLocal::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int min_level = Z_DEFAULT_COMPRESSION; + int max_level = Z_BEST_COMPRESSION; + if (compression_level < 0 || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); + return 2; + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_local_gz.h b/src/COMPRESS/dump_local_gz.h index cc788863de..b3f7c7dcf8 100644 --- a/src/COMPRESS/dump_local_gz.h +++ b/src/COMPRESS/dump_local_gz.h @@ -31,12 +31,15 @@ class DumpLocalGZ : public DumpLocal { virtual ~DumpLocalGZ(); protected: + int compression_level; gzFile gzFp; // file pointer for the compressed output stream virtual void openfile(); virtual void write_header(bigint); virtual void write_data(int, double *); virtual void write(); + + virtual int modify_param(int, char **); }; } diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index c76b7afa17..bd6000591b 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -16,6 +16,7 @@ #include "update.h" #include +#include using namespace LAMMPS_NS; @@ -24,6 +25,8 @@ DumpXYZGZ::DumpXYZGZ(LAMMPS *lmp, int narg, char **arg) : { gzFp = NULL; + compression_level = Z_BEST_COMPRESSION; + if (!compressed) error->all(FLERR,"Dump xyz/gz only writes compressed files"); } @@ -90,12 +93,15 @@ void DumpXYZGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { + std::string mode; if (append_flag) { - gzFp = gzopen(filecurrent,"ab9"); + mode = fmt::format("ab{}", compression_level); } else { - gzFp = gzopen(filecurrent,"wb9"); + mode = fmt::format("wb{}", compression_level); } + gzFp = gzopen(filecurrent, mode.c_str()); + if (gzFp == NULL) error->one(FLERR,"Cannot open dump file"); } else gzFp = NULL; @@ -134,3 +140,21 @@ void DumpXYZGZ::write() } } } + +/* ---------------------------------------------------------------------- */ + +int DumpXYZGZ::modify_param(int narg, char **arg) +{ + int consumed = DumpXYZ::modify_param(narg, arg); + if(consumed == 0) { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int min_level = Z_DEFAULT_COMPRESSION; + int max_level = Z_BEST_COMPRESSION; + if (compression_level < 0 || compression_level > max_level) + error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); + return 2; + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_xyz_gz.h b/src/COMPRESS/dump_xyz_gz.h index 45d5456ddb..834db488a5 100644 --- a/src/COMPRESS/dump_xyz_gz.h +++ b/src/COMPRESS/dump_xyz_gz.h @@ -31,12 +31,15 @@ class DumpXYZGZ : public DumpXYZ { virtual ~DumpXYZGZ(); protected: + int compression_level; gzFile gzFp; // file pointer for the compressed output stream virtual void openfile(); virtual void write_header(bigint); virtual void write_data(int, double *); virtual void write(); + + virtual int modify_param(int, char **); }; } From c5a2e50bf5f1a92bc2c9386e03da20ed124dd220 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 15:38:20 -0400 Subject: [PATCH 16/26] Add Zstd variants of dump local and xyz --- src/COMPRESS/dump_cfg_zstd.cpp | 4 + src/COMPRESS/dump_cfg_zstd.h | 4 + src/COMPRESS/dump_local_gz.cpp | 14 +- src/COMPRESS/dump_local_zstd.cpp | 191 ++++++++++++++++++++++ src/COMPRESS/dump_local_zstd.h | 63 +++++++ src/COMPRESS/dump_xyz_zstd.cpp | 166 +++++++++++++++++++ src/COMPRESS/dump_xyz_zstd.h | 63 +++++++ src/dump_local.cpp | 12 +- unittest/formats/CMakeLists.txt | 12 ++ unittest/formats/test_dump_local_zstd.cpp | 106 ++++++++++++ unittest/formats/test_dump_xyz_zstd.cpp | 101 ++++++++++++ 11 files changed, 723 insertions(+), 13 deletions(-) create mode 100644 src/COMPRESS/dump_local_zstd.cpp create mode 100644 src/COMPRESS/dump_local_zstd.h create mode 100644 src/COMPRESS/dump_xyz_zstd.cpp create mode 100644 src/COMPRESS/dump_xyz_zstd.h create mode 100644 unittest/formats/test_dump_local_zstd.cpp create mode 100644 unittest/formats/test_dump_xyz_zstd.cpp diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp index ac746df431..2b683b4861 100644 --- a/src/COMPRESS/dump_cfg_zstd.cpp +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #include "dump_cfg_zstd.h" #include "atom.h" #include "domain.h" diff --git a/src/COMPRESS/dump_cfg_zstd.h b/src/COMPRESS/dump_cfg_zstd.h index 71e06b0529..653ff6a776 100644 --- a/src/COMPRESS/dump_cfg_zstd.h +++ b/src/COMPRESS/dump_cfg_zstd.h @@ -11,6 +11,10 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + #ifdef DUMP_CLASS DumpStyle(cfg/zstd,DumpCFGZstd) diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index 280b66bb2a..eb7e31ad36 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -122,18 +122,18 @@ void DumpLocalGZ::write_header(bigint ndump) gzprintf(gzFp,"ITEM: TIMESTEP\n"); gzprintf(gzFp,BIGINT_FORMAT "\n",update->ntimestep); - gzprintf(gzFp,"ITEM: NUMBER OF ENTRIES\n"); + gzprintf(gzFp,"ITEM: NUMBER OF %s\n",label); gzprintf(gzFp,BIGINT_FORMAT "\n",ndump); if (domain->triclinic) { gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxxlo,boxxhi,boxxy); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxylo,boxyhi,boxxz); - gzprintf(gzFp,"%-1.16g %-1.16g %-1.16g\n",boxzlo,boxzhi,boxyz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz); + gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz); } else { gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxylo,boxyhi); - gzprintf(gzFp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi); + gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi); } gzprintf(gzFp,"ITEM: %s %s\n",label,columns); } diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp new file mode 100644 index 0000000000..643e29f2cc --- /dev/null +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -0,0 +1,191 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#include "dump_local_zstd.h" +#include "domain.h" +#include "error.h" +#include "update.h" +#include "force.h" + +#include +#include + +using namespace LAMMPS_NS; + +DumpLocalZstd::DumpLocalZstd(LAMMPS *lmp, int narg, char **arg) : + DumpLocal(lmp, narg, arg) +{ + if (!compressed) + error->all(FLERR,"Dump local/zstd only writes compressed files"); +} + + +/* ---------------------------------------------------------------------- */ + +DumpLocalZstd::~DumpLocalZstd() +{ +} + + +/* ---------------------------------------------------------------------- + generic opening of a dump file + ASCII or binary or gzipped + some derived classes override this function +------------------------------------------------------------------------- */ + +void DumpLocalZstd::openfile() +{ + // single file, already opened, so just return + + if (singlefile_opened) return; + if (multifile == 0) singlefile_opened = 1; + + // if one file per timestep, replace '*' with current timestep + + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + *ptr = '*'; + if (maxfiles > 0) { + if (numfiles < maxfiles) { + nameslist[numfiles] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[numfiles],filecurrent); + ++numfiles; + } else { + remove(nameslist[fileidx]); + delete[] nameslist[fileidx]; + nameslist[fileidx] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[fileidx],filecurrent); + fileidx = (fileidx + 1) % maxfiles; + } + } + } + + // each proc with filewriter = 1 opens a file + + if (filewriter) { + if (append_flag) { + error->one(FLERR, "dump cfg/zstd currently doesn't support append"); + } + + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + + // delete string with timestep replaced + + if (multifile) delete [] filecurrent; +} + +void DumpLocalZstd::write_header(bigint ndump) +{ + std::string header; + + if ((multiproc) || (!multiproc && me == 0)) { + if (unit_flag && !unit_count) { + ++unit_count; + header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style); + } + + if (time_flag) { + header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time()); + } + + header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); + header += fmt::format("ITEM: NUMBER OF {}\n{}\n", label, ndump); + if (domain->triclinic == 0) { + header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi); + } else { + header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz); + } + header += fmt::format("ITEM: {} {}\n", label, columns); + + writer.write(header.c_str(), header.length()); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpLocalZstd::write_data(int n, double *mybuf) +{ + writer.write(mybuf, sizeof(char)*n); +} + +/* ---------------------------------------------------------------------- */ + +void DumpLocalZstd::write() +{ + DumpLocal::write(); + if (filewriter) { + if (multifile) { + writer.close(); + } else { + if (flush_flag && writer.isopen()) { + writer.flush(); + } + } + } +} + +/* ---------------------------------------------------------------------- */ + +int DumpLocalZstd::modify_param(int narg, char **arg) +{ + int consumed = DumpLocal::modify_param(narg, arg); + if(consumed == 0) { + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_local_zstd.h b/src/COMPRESS/dump_local_zstd.h new file mode 100644 index 0000000000..5f3ddb84f2 --- /dev/null +++ b/src/COMPRESS/dump_local_zstd.h @@ -0,0 +1,63 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(local/zstd,DumpLocalZstd) + +#else + +#ifndef LMP_DUMP_LOCAL_ZSTD_H +#define LMP_DUMP_LOCAL_ZSTD_H + +#include "dump_local.h" +#include "zstd_file_writer.h" + +namespace LAMMPS_NS { + +class DumpLocalZstd : public DumpLocal { + public: + DumpLocalZstd(class LAMMPS *, int, char **); + virtual ~DumpLocalZstd(); + + protected: + ZstdFileWriter writer; + + virtual void openfile(); + virtual void write_header(bigint); + virtual void write_data(int, double *); + virtual void write(); + + virtual int modify_param(int, char **); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Dump local/zstd only writes compressed files + +The dump local/zstd output file name must have a .zst suffix. + +E: Cannot open dump file + +Self-explanatory. + +*/ diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp new file mode 100644 index 0000000000..e825051474 --- /dev/null +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -0,0 +1,166 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#include "dump_xyz_zstd.h" +#include "error.h" +#include "update.h" +#include "force.h" + +#include +#include + +using namespace LAMMPS_NS; + +DumpXYZZstd::DumpXYZZstd(LAMMPS *lmp, int narg, char **arg) : + DumpXYZ(lmp, narg, arg) +{ + if (!compressed) + error->all(FLERR,"Dump xyz/zstd only writes compressed files"); +} + + +/* ---------------------------------------------------------------------- */ + +DumpXYZZstd::~DumpXYZZstd() +{ +} + + +/* ---------------------------------------------------------------------- + generic opening of a dump file + ASCII or binary or gzipped + some derived classes override this function +------------------------------------------------------------------------- */ + +void DumpXYZZstd::openfile() +{ + // single file, already opened, so just return + + if (singlefile_opened) return; + if (multifile == 0) singlefile_opened = 1; + + // if one file per timestep, replace '*' with current timestep + + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); + *ptr = '\0'; + if (padflag == 0) + sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", + filestar,update->ntimestep,ptr+1); + else { + char bif[8],pad[16]; + strcpy(bif,BIGINT_FORMAT); + sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); + } + *ptr = '*'; + if (maxfiles > 0) { + if (numfiles < maxfiles) { + nameslist[numfiles] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[numfiles],filecurrent); + ++numfiles; + } else { + remove(nameslist[fileidx]); + delete[] nameslist[fileidx]; + nameslist[fileidx] = new char[strlen(filecurrent)+1]; + strcpy(nameslist[fileidx],filecurrent); + fileidx = (fileidx + 1) % maxfiles; + } + } + } + + // each proc with filewriter = 1 opens a file + + if (filewriter) { + if (append_flag) { + error->one(FLERR, "dump cfg/zstd currently doesn't support append"); + } + + try { + writer.open(filecurrent); + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + + // delete string with timestep replaced + + if (multifile) delete [] filecurrent; +} + +void DumpXYZZstd::write_header(bigint ndump) +{ + if (me == 0) { + std::string header = fmt::format("{}\n", ndump); + header += fmt::format("Atoms. Timestep: {}\n", update->ntimestep); + writer.write(header.c_str(), header.length()); + } +} + +/* ---------------------------------------------------------------------- */ + +void DumpXYZZstd::write_data(int n, double *mybuf) +{ + writer.write(mybuf, n); +} + +/* ---------------------------------------------------------------------- */ + +void DumpXYZZstd::write() +{ + DumpXYZ::write(); + if (filewriter) { + if (multifile) { + writer.close(); + } else { + if (flush_flag && writer.isopen()) { + writer.flush(); + } + } + } +} + +/* ---------------------------------------------------------------------- */ + +int DumpXYZZstd::modify_param(int narg, char **arg) +{ + int consumed = DumpXYZ::modify_param(narg, arg); + if(consumed == 0) { + try { + if (strcmp(arg[0],"checksum") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + if (strcmp(arg[1],"yes") == 0) writer.setChecksum(true); + else if (strcmp(arg[1],"no") == 0) writer.setChecksum(false); + else error->all(FLERR,"Illegal dump_modify command"); + return 2; + } else if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = force->inumeric(FLERR,arg[1]); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException & e) { + error->one(FLERR, e.what()); + } + } + return consumed; +} diff --git a/src/COMPRESS/dump_xyz_zstd.h b/src/COMPRESS/dump_xyz_zstd.h new file mode 100644 index 0000000000..e1991202d6 --- /dev/null +++ b/src/COMPRESS/dump_xyz_zstd.h @@ -0,0 +1,63 @@ +/* -*- c++ -*- ---------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +/* ---------------------------------------------------------------------- + Contributing author: Richard Berger (Temple U) +------------------------------------------------------------------------- */ + +#ifdef DUMP_CLASS + +DumpStyle(xyz/zstd,DumpXYZZstd) + +#else + +#ifndef LMP_DUMP_XYZ_ZSTD_H +#define LMP_DUMP_XYZ_ZSTD_H + +#include "dump_xyz.h" +#include "zstd_file_writer.h" + +namespace LAMMPS_NS { + +class DumpXYZZstd : public DumpXYZ { + public: + DumpXYZZstd(class LAMMPS *, int, char **); + virtual ~DumpXYZZstd(); + + protected: + ZstdFileWriter writer; + + virtual void openfile(); + virtual void write_header(bigint); + virtual void write_data(int, double *); + virtual void write(); + + virtual int modify_param(int, char **); +}; + +} + +#endif +#endif + +/* ERROR/WARNING messages: + +E: Dump xyz/zstd only writes compressed files + +The dump xyz/zstd output file name must have a .zst suffix. + +E: Cannot open dump file + +Self-explanatory. + +*/ diff --git a/src/dump_local.cpp b/src/dump_local.cpp index 103836b8a2..0709eed952 100644 --- a/src/dump_local.cpp +++ b/src/dump_local.cpp @@ -268,14 +268,14 @@ void DumpLocal::write_header(bigint ndump) fprintf(fp,BIGINT_FORMAT "\n",ndump); if (domain->triclinic) { fprintf(fp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxxlo,boxxhi,boxxy); - fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxylo,boxyhi,boxxz); - fprintf(fp,"%-1.16g %-1.16g %-1.16g\n",boxzlo,boxzhi,boxyz); + fprintf(fp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy); + fprintf(fp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz); + fprintf(fp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz); } else { fprintf(fp,"ITEM: BOX BOUNDS %s\n",boundstr); - fprintf(fp,"%-1.16g %-1.16g\n",boxxlo,boxxhi); - fprintf(fp,"%-1.16g %-1.16g\n",boxylo,boxyhi); - fprintf(fp,"%-1.16g %-1.16g\n",boxzlo,boxzhi); + fprintf(fp,"%-1.16e %-1.16e\n",boxxlo,boxxhi); + fprintf(fp,"%-1.16e %-1.16e\n",boxylo,boxyhi); + fprintf(fp,"%-1.16e %-1.16e\n",boxzlo,boxzhi); } fprintf(fp,"ITEM: %s %s\n",label,columns); } diff --git a/unittest/formats/CMakeLists.txt b/unittest/formats/CMakeLists.txt index 80c09d691a..0a3e7bce56 100644 --- a/unittest/formats/CMakeLists.txt +++ b/unittest/formats/CMakeLists.txt @@ -82,6 +82,18 @@ if (PKG_COMPRESS) add_test(NAME DumpCfgZstd COMMAND test_dump_cfg_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_tests_properties(DumpCfgZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") set_tests_properties(DumpCfgZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") + + add_executable(test_dump_xyz_zstd test_dump_xyz_zstd.cpp) + target_link_libraries(test_dump_xyz_zstd PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpXYZZstd COMMAND test_dump_xyz_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpXYZZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpXYZZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") + + add_executable(test_dump_local_zstd test_dump_local_zstd.cpp) + target_link_libraries(test_dump_local_zstd PRIVATE lammps GTest::GMock GTest::GTest) + add_test(NAME DumpLocalZstd COMMAND test_dump_local_zstd WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties(DumpLocalZstd PROPERTIES ENVIRONMENT "LAMMPS_POTENTIALS=${LAMMPS_POTENTIALS_DIR}") + set_tests_properties(DumpLocalZstd PROPERTIES ENVIRONMENT "ZSTD_BINARY=${ZSTD_BINARY}") endif() endif() diff --git a/unittest/formats/test_dump_local_zstd.cpp b/unittest/formats/test_dump_local_zstd.cpp new file mode 100644 index 0000000000..cf29f4ee04 --- /dev/null +++ b/unittest/formats/test_dump_local_zstd.cpp @@ -0,0 +1,106 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * ZSTD_BINARY = nullptr; + +using ::testing::Eq; + +class DumpLocalGZTest : public MeltTest { + std::string dump_style = "local"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string fields, std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {} {}", dump_style, text_file, fields)); + command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_file, fields)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", ZSTD_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpLocalGZTest, compressed_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + if (!verbose) ::testing::internal::CaptureStdout(); + command("compute comp all pair/local dist eng"); + if (!verbose) ::testing::internal::GetCapturedStdout(); + + auto text_files = "dump_local_zstd_text_run*.melt.local"; + auto compressed_files = "dump_local_zstd_compressed_run*.melt.local.zst"; + auto text_file = "dump_local_zstd_text_run0.melt.local"; + auto compressed_file = "dump_local_zstd_compressed_run0.melt.local.zst"; + auto fields = "index c_comp[1]"; + + generate_text_and_compressed_dump(text_files, compressed_files, "local/zstd", fields, "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + ZSTD_BINARY = getenv("ZSTD_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} diff --git a/unittest/formats/test_dump_xyz_zstd.cpp b/unittest/formats/test_dump_xyz_zstd.cpp new file mode 100644 index 0000000000..70684b594f --- /dev/null +++ b/unittest/formats/test_dump_xyz_zstd.cpp @@ -0,0 +1,101 @@ +/* ---------------------------------------------------------------------- + LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator + http://lammps.sandia.gov, Sandia National Laboratories + Steve Plimpton, sjplimp@sandia.gov + + Copyright (2003) Sandia Corporation. Under the terms of Contract + DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains + certain rights in this software. This software is distributed under + the GNU General Public License. + + See the README file in the top-level LAMMPS directory. +------------------------------------------------------------------------- */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "fmt/format.h" +#include "utils.h" +#include "../testing/core.h" +#include "../testing/systems/melt.h" +#include "../testing/utils.h" + +char * ZSTD_BINARY = nullptr; + +using ::testing::Eq; + +class DumpXYZGZTest : public MeltTest { + std::string dump_style = "xyz"; +public: + void generate_text_and_compressed_dump(std::string text_file, std::string compressed_file, std::string compression_style, + std::string dump_modify_options, int ntimesteps) { + if (!verbose) ::testing::internal::CaptureStdout(); + command(fmt::format("dump id0 all {} 1 {}", dump_style, text_file)); + command(fmt::format("dump id1 all {} 1 {}", compression_style, compressed_file)); + + if (!dump_modify_options.empty()) { + command(fmt::format("dump_modify id0 {}", dump_modify_options)); + command(fmt::format("dump_modify id1 {}", dump_modify_options)); + } + + command(fmt::format("run {}", ntimesteps)); + if (!verbose) ::testing::internal::GetCapturedStdout(); + } + + std::string convert_compressed_to_text(std::string compressed_file) { + if (!verbose) ::testing::internal::CaptureStdout(); + std::string converted_file = compressed_file.substr(0, compressed_file.find_last_of('.')); + std::string cmdline = fmt::format("{} -d -c {} > {}", ZSTD_BINARY, compressed_file, converted_file); + system(cmdline.c_str()); + if (!verbose) ::testing::internal::GetCapturedStdout(); + return converted_file; + } +}; + +TEST_F(DumpXYZGZTest, compressed_run0) +{ + if(!ZSTD_BINARY) GTEST_SKIP(); + + auto text_files = "dump_xyz_zstd_text_run*.melt.xyz"; + auto compressed_files = "dump_xyz_zstd_compressed_run*.melt.xyz.zst"; + auto text_file = "dump_xyz_zstd_text_run0.melt.xyz"; + auto compressed_file = "dump_xyz_zstd_compressed_run0.melt.xyz.zst"; + + generate_text_and_compressed_dump(text_files, compressed_files, "xyz/zstd", "", 0); + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + ::testing::InitGoogleMock(&argc, argv); + + // handle arguments passed via environment variable + if (const char *var = getenv("TEST_ARGS")) { + std::vector env = utils::split_words(var); + for (auto arg : env) { + if (arg == "-v") { + verbose = true; + } + } + } + + ZSTD_BINARY = getenv("ZSTD_BINARY"); + + if ((argc > 1) && (strcmp(argv[1], "-v") == 0)) verbose = true; + + int rv = RUN_ALL_TESTS(); + MPI_Finalize(); + return rv; +} From 3865de870592bc556c023604ef16fe7e65632177 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 16:43:06 -0400 Subject: [PATCH 17/26] Add lost line --- src/COMPRESS/dump_atom_gz.cpp | 3 ++- src/COMPRESS/dump_cfg_gz.cpp | 4 +++- src/COMPRESS/dump_custom_gz.cpp | 3 ++- src/COMPRESS/dump_local_gz.cpp | 4 +++- src/COMPRESS/dump_xyz_gz.cpp | 4 +++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 575b6a9f44..f094168873 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -173,7 +173,8 @@ int DumpAtomGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - if (compression_level < 0 || compression_level > max_level) + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index 83c698a9cc..a56a222b2b 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -16,6 +16,7 @@ #include "domain.h" #include "error.h" #include "update.h" +#include "force.h" #include #include @@ -180,7 +181,8 @@ int DumpCFGGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - if (compression_level < 0 || compression_level > max_level) + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 13fa564e52..805190e0ef 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -173,7 +173,8 @@ int DumpCustomGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - if (compression_level < 0 || compression_level > max_level) + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index eb7e31ad36..67b7f8e057 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -15,6 +15,7 @@ #include "domain.h" #include "error.h" #include "update.h" +#include "force.h" #include #include @@ -187,7 +188,8 @@ int DumpLocalGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - if (compression_level < 0 || compression_level > max_level) + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index bd6000591b..9ca49b8321 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -14,6 +14,7 @@ #include "dump_xyz_gz.h" #include "error.h" #include "update.h" +#include "force.h" #include #include @@ -151,7 +152,8 @@ int DumpXYZGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - if (compression_level < 0 || compression_level > max_level) + compression_level = force->inumeric(FLERR,arg[1]); + if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; } From 0925fc822d9efb217a5a51fb19707ad00fa43df6 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 18:21:58 -0400 Subject: [PATCH 18/26] Make Zstd support optional --- cmake/Modules/Packages/COMPRESS.cmake | 8 ++++++-- src/COMPRESS/README | 4 ++++ src/COMPRESS/dump_atom_zstd.cpp | 4 ++++ src/COMPRESS/dump_atom_zstd.h | 3 +++ src/COMPRESS/dump_cfg_zstd.cpp | 2 ++ src/COMPRESS/dump_cfg_zstd.h | 2 ++ src/COMPRESS/dump_custom_zstd.cpp | 4 ++++ src/COMPRESS/dump_custom_zstd.h | 3 +++ src/COMPRESS/dump_local_zstd.cpp | 4 ++++ src/COMPRESS/dump_local_zstd.h | 3 +++ src/COMPRESS/dump_xyz_zstd.cpp | 4 ++++ src/COMPRESS/dump_xyz_zstd.h | 3 +++ 12 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index c92767692a..4d392ab516 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -1,5 +1,9 @@ find_package(ZLIB REQUIRED) target_link_libraries(lammps PRIVATE ZLIB::ZLIB) -find_package(Zstd REQUIRED) -target_link_libraries(lammps PRIVATE Zstd::Zstd) +find_package(Zstd) + +if(Zstd_FOUND) + target_compile_definitions(lammps PRIVATE -DLAMMPS_ZSTD) + target_link_libraries(lammps PRIVATE Zstd::Zstd) +endif() diff --git a/src/COMPRESS/README b/src/COMPRESS/README index 1a31445d0b..617757c476 100644 --- a/src/COMPRESS/README +++ b/src/COMPRESS/README @@ -10,5 +10,9 @@ alternative for compressed file I/O on systems where using a pipe can cause problems, e.g. when using RDMA communication with pinned memory like clusters with Infiniband or Myrinet. +Update 08/2020: Added variants that use the Zstd compression library instead +of zlib. To enable, set -DLAMMPS_ZSTD. These provide a wider range of +compression levels. See http://facebook.github.io/zstd/ for more details. + Currently a few selected dump styles are supported for writing via this packaging. diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index 30862ab854..c3085a3b56 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #include "dump_atom_zstd.h" #include "domain.h" #include "error.h" @@ -189,3 +191,5 @@ int DumpAtomZstd::modify_param(int narg, char **arg) } return consumed; } + +#endif diff --git a/src/COMPRESS/dump_atom_zstd.h b/src/COMPRESS/dump_atom_zstd.h index d1bfd9d00f..30108eeee9 100644 --- a/src/COMPRESS/dump_atom_zstd.h +++ b/src/COMPRESS/dump_atom_zstd.h @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #ifdef DUMP_CLASS DumpStyle(atom/zstd,DumpAtomZstd) @@ -47,6 +49,7 @@ class DumpAtomZstd : public DumpAtom { } +#endif #endif #endif diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp index 2b683b4861..f153484164 100644 --- a/src/COMPRESS/dump_cfg_zstd.cpp +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -15,6 +15,7 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD #include "dump_cfg_zstd.h" #include "atom.h" #include "domain.h" @@ -192,3 +193,4 @@ int DumpCFGZstd::modify_param(int narg, char **arg) } return consumed; } +#endif diff --git a/src/COMPRESS/dump_cfg_zstd.h b/src/COMPRESS/dump_cfg_zstd.h index 653ff6a776..6f4fe8face 100644 --- a/src/COMPRESS/dump_cfg_zstd.h +++ b/src/COMPRESS/dump_cfg_zstd.h @@ -15,6 +15,7 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD #ifdef DUMP_CLASS DumpStyle(cfg/zstd,DumpCFGZstd) @@ -47,6 +48,7 @@ class DumpCFGZstd : public DumpCFG { } +#endif #endif #endif diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index 420425b050..d3bf3467eb 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #include "dump_custom_zstd.h" #include "domain.h" #include "error.h" @@ -189,3 +191,5 @@ int DumpCustomZstd::modify_param(int narg, char **arg) } return consumed; } + +#endif diff --git a/src/COMPRESS/dump_custom_zstd.h b/src/COMPRESS/dump_custom_zstd.h index fbda89d490..ce3f1325c5 100644 --- a/src/COMPRESS/dump_custom_zstd.h +++ b/src/COMPRESS/dump_custom_zstd.h @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #ifdef DUMP_CLASS DumpStyle(custom/zstd,DumpCustomZstd) @@ -48,6 +50,7 @@ class DumpCustomZstd : public DumpCustom { } +#endif #endif #endif diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp index 643e29f2cc..73402bee47 100644 --- a/src/COMPRESS/dump_local_zstd.cpp +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #include "dump_local_zstd.h" #include "domain.h" #include "error.h" @@ -189,3 +191,5 @@ int DumpLocalZstd::modify_param(int narg, char **arg) } return consumed; } + +#endif diff --git a/src/COMPRESS/dump_local_zstd.h b/src/COMPRESS/dump_local_zstd.h index 5f3ddb84f2..398b1c3337 100644 --- a/src/COMPRESS/dump_local_zstd.h +++ b/src/COMPRESS/dump_local_zstd.h @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #ifdef DUMP_CLASS DumpStyle(local/zstd,DumpLocalZstd) @@ -47,6 +49,7 @@ class DumpLocalZstd : public DumpLocal { } +#endif #endif #endif diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp index e825051474..fb65af63e5 100644 --- a/src/COMPRESS/dump_xyz_zstd.cpp +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #include "dump_xyz_zstd.h" #include "error.h" #include "update.h" @@ -164,3 +166,5 @@ int DumpXYZZstd::modify_param(int narg, char **arg) } return consumed; } + +#endif diff --git a/src/COMPRESS/dump_xyz_zstd.h b/src/COMPRESS/dump_xyz_zstd.h index e1991202d6..2422341557 100644 --- a/src/COMPRESS/dump_xyz_zstd.h +++ b/src/COMPRESS/dump_xyz_zstd.h @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #ifdef DUMP_CLASS DumpStyle(xyz/zstd,DumpXYZZstd) @@ -47,6 +49,7 @@ class DumpXYZZstd : public DumpXYZ { } +#endif #endif #endif From fe833e6c87e6e084d43a462b47be1ec34def18c1 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Mon, 31 Aug 2020 19:49:17 -0400 Subject: [PATCH 19/26] Add missing #ifdef --- src/COMPRESS/zstd_file_writer.cpp | 4 ++++ src/COMPRESS/zstd_file_writer.h | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/COMPRESS/zstd_file_writer.cpp b/src/COMPRESS/zstd_file_writer.cpp index eb63d4e767..3f71a41402 100644 --- a/src/COMPRESS/zstd_file_writer.cpp +++ b/src/COMPRESS/zstd_file_writer.cpp @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #include "zstd_file_writer.h" #include #include @@ -156,3 +158,5 @@ void ZstdFileWriter::setChecksum(bool enabled) throw FileWriterException("Checksum flag can not be changed while file is open"); checksum_flag = enabled ? 1 : 0; } + +#endif diff --git a/src/COMPRESS/zstd_file_writer.h b/src/COMPRESS/zstd_file_writer.h index 8a974a8599..eec993bd4e 100644 --- a/src/COMPRESS/zstd_file_writer.h +++ b/src/COMPRESS/zstd_file_writer.h @@ -15,6 +15,8 @@ Contributing author: Richard Berger (Temple U) ------------------------------------------------------------------------- */ +#ifdef LAMMPS_ZSTD + #ifndef LMP_ZSTD_FILE_WRITER_H #define LMP_ZSTD_FILE_WRITER_H @@ -50,3 +52,4 @@ public: } #endif +#endif From 8e0c4d764ea2b34c6467dd9f4e89d496c3cf0221 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 10:58:13 -0400 Subject: [PATCH 20/26] Change include for fmt --- src/COMPRESS/dump_atom_gz.cpp | 2 +- src/COMPRESS/dump_atom_zstd.cpp | 2 +- src/COMPRESS/dump_cfg_gz.cpp | 2 +- src/COMPRESS/dump_cfg_zstd.cpp | 2 +- src/COMPRESS/dump_custom_gz.cpp | 2 +- src/COMPRESS/dump_custom_zstd.cpp | 2 +- src/COMPRESS/dump_local_gz.cpp | 2 +- src/COMPRESS/dump_local_zstd.cpp | 2 +- src/COMPRESS/dump_xyz_gz.cpp | 2 +- src/COMPRESS/dump_xyz_zstd.cpp | 2 +- src/COMPRESS/zstd_file_writer.cpp | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index f094168873..38ec0fb59e 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -18,7 +18,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index c3085a3b56..9c0f428905 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -24,7 +24,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index a56a222b2b..25f29eda2f 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -19,7 +19,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; #define UNWRAPEXPAND 10.0 diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp index f153484164..125c4f3406 100644 --- a/src/COMPRESS/dump_cfg_zstd.cpp +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -24,7 +24,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; #define UNWRAPEXPAND 10.0 diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 805190e0ef..6770e16bbc 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -18,7 +18,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index d3bf3467eb..afb6d81435 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -24,7 +24,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index 67b7f8e057..0d1f666322 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -18,7 +18,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp index 73402bee47..3184dd06ad 100644 --- a/src/COMPRESS/dump_local_zstd.cpp +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -24,7 +24,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index 9ca49b8321..d1f7b59dfe 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -17,7 +17,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp index fb65af63e5..83cb90627b 100644 --- a/src/COMPRESS/dump_xyz_zstd.cpp +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -23,7 +23,7 @@ #include "force.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; diff --git a/src/COMPRESS/zstd_file_writer.cpp b/src/COMPRESS/zstd_file_writer.cpp index 3f71a41402..3a2cc15856 100644 --- a/src/COMPRESS/zstd_file_writer.cpp +++ b/src/COMPRESS/zstd_file_writer.cpp @@ -19,7 +19,7 @@ #include "zstd_file_writer.h" #include -#include +#include "fmt/format.h" using namespace LAMMPS_NS; From 3db1a6d690a6a72b81bba6c76eb51308bd853435 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 11:14:36 -0400 Subject: [PATCH 21/26] Use PkgConfig to find Zstd --- cmake/Modules/FindZstd.cmake | 28 --------------------------- cmake/Modules/Packages/COMPRESS.cmake | 5 +++-- 2 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 cmake/Modules/FindZstd.cmake diff --git a/cmake/Modules/FindZstd.cmake b/cmake/Modules/FindZstd.cmake deleted file mode 100644 index 252767cb24..0000000000 --- a/cmake/Modules/FindZstd.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Find Zstd library -# -# Zstd_INCLUDE_DIRS - where to find zstd.h, etc. -# Zstd_LIBRARIES - List of libraries when using libzstd -# Zstd_FOUND - True if libzstd is found. - -find_path(Zstd_INCLUDE_DIR NAMES zstd.h) -find_library(Zstd_LIBRARY NAMES zstd) - -# handle the QUIET and REQUIRED arguments and -# set Zstd_FOUND to TRUE if all variables are non-zero -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Zstd DEFAULT_MSG Zstd_LIBRARY Zstd_INCLUDE_DIR) - -# Copy the results to the output variables and target. -if(Zstd_FOUND) - set(Zstd_LIBRARIES ${Zstd_LIBRARY}) - set(Zstd_INCLUDE_DIRS ${Zstd_INCLUDE_DIR}) - - if(NOT TARGET Zstd::Zstd) - add_library(Zstd::Zstd UNKNOWN IMPORTED) - set_target_properties(Zstd::Zstd PROPERTIES - IMPORTED_LOCATION "${Zstd_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${Zstd_INCLUDE_DIR}") - endif() -endif() - -mark_as_advanced(Zstd_INCLUDE_DIR Zstd_LIBRARY) diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index 4d392ab516..f48bbe4d03 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -1,9 +1,10 @@ find_package(ZLIB REQUIRED) target_link_libraries(lammps PRIVATE ZLIB::ZLIB) -find_package(Zstd) +find_package(PkgConfig REQUIRED) +pkg_check_modules(Zstd IMPORTED_TARGET libzstd) if(Zstd_FOUND) target_compile_definitions(lammps PRIVATE -DLAMMPS_ZSTD) - target_link_libraries(lammps PRIVATE Zstd::Zstd) + target_link_libraries(lammps PRIVATE PkgConfig::Zstd) endif() From a2b5c379f237032bbf84bc72b725907cfb53967c Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 12:17:08 -0400 Subject: [PATCH 22/26] Update docs for COMPRESS styles --- doc/src/dump.rst | 19 ++++++++++++++----- doc/src/dump_modify.rst | 41 +++++++++++++++++++++++++++++++++++++++++ doc/src/write_dump.rst | 5 +++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/doc/src/dump.rst b/doc/src/dump.rst index 36a20defb6..d225b88662 100644 --- a/doc/src/dump.rst +++ b/doc/src/dump.rst @@ -46,17 +46,19 @@ Syntax *atom* args = none *atom/gz* args = none + *atom/zstd* args = none *atom/mpiio* args = none *atom/adios* args = none, discussed on :doc:`dump atom/adios ` doc page *cfg* args = same as *custom* args, see below *cfg/gz* args = same as *custom* args, see below + *cfg/zstd* args = same as *custom* args, see below *cfg/mpiio* args = same as *custom* args, see below - *custom*\ , *custom/gz*\ , *custom/mpiio* args = see below + *custom*\ , *custom/gz*\ , *custom/zstd* , *custom/mpiio* args = see below *custom/adios* args = same as *custom* args, discussed on :doc:`dump custom/adios ` doc page *dcd* args = none *h5md* args = discussed on :doc:`dump h5md ` doc page *image* args = discussed on :doc:`dump image ` doc page - *local* args = see below + *local*, *local/gz*, *local/zstd* args = see below *molfile* args = discussed on :doc:`dump molfile ` doc page *movie* args = discussed on :doc:`dump image ` doc page *netcdf* args = discussed on :doc:`dump netcdf ` doc page @@ -65,9 +67,10 @@ Syntax *xtc* args = none *xyz* args = none *xyz/gz* args = none + *xyz/zstd* args = none *xyz/mpiio* args = none -* *custom* or *custom/gz* or *custom/mpiio* or *netcdf* or *netcdf/mpiio* args = list of atom attributes +* *custom* or *custom/gz* or *custom/zstd* or *custom/mpiio* or *netcdf* or *netcdf/mpiio* args = list of atom attributes .. parsed-literal:: @@ -111,7 +114,7 @@ Syntax d_name = per-atom floating point vector with name, managed by fix property/atom i_name = per-atom integer vector with name, managed by fix property/atom -* *local* args = list of local attributes +* *local* or *local/gz* or *local/zstd* args = list of local attributes .. parsed-literal:: @@ -130,6 +133,7 @@ Examples dump myDump all atom 100 dump.atom dump myDump all atom/mpiio 100 dump.atom.mpiio dump myDump all atom/gz 100 dump.atom.gz + dump myDump all atom/zstd 100 dump.atom.zst dump 2 subgroup atom 50 dump.run.bin dump 2 subgroup atom 50 dump.run.mpiio.bin dump 4a all custom 100 dump.myforce.* id type x y vx fx @@ -187,7 +191,7 @@ default. For the *dcd*\ , *xtc*\ , *xyz*\ , and *molfile* styles, sorting by atom ID is on by default. See the :doc:`dump_modify ` doc page for details. -The *atom/gz*\ , *cfg/gz*\ , *custom/gz*\ , and *xyz/gz* styles are identical +The *atom/gz*\ , *cfg/gz*\ , *custom/gz*\ , *local/gz*, and *xyz/gz* styles are identical in command syntax to the corresponding styles without "gz", however, they generate compressed files using the zlib library. Thus the filename suffix ".gz" is mandatory. This is an alternative approach to writing @@ -198,6 +202,11 @@ For the remainder of this doc page, you should thus consider the *atom* and *atom/gz* styles (etc) to be inter-changeable, with the exception of the required filename suffix. +Similarily, the *atom/zstd*\ , *cfg/zstd*\ , *custom/zstd*\ , *local/zstd*, +and *xyz/zstd* styles are identical to the gz styles, but use the Zstd +compression library instead. See the :doc:`dump_modify ` doc +page for details on how to control the compression level in both variants. + As explained below, the *atom/mpiio*\ , *cfg/mpiio*\ , *custom/mpiio*\ , and *xyz/mpiio* styles are identical in command syntax and in the format of the dump files they create, to the corresponding styles without diff --git a/doc/src/dump_modify.rst b/doc/src/dump_modify.rst index 6c7d4eb0f5..753cce703c 100644 --- a/doc/src/dump_modify.rst +++ b/doc/src/dump_modify.rst @@ -114,6 +114,21 @@ Syntax *framerate* arg = fps fps = frames per second for movie +* these keywords apply only to the */gz* and */zstd* dump styles +* keyword = *compression_level* + + .. parsed-literal:: + + *compression_level* args = level + level = integer specifying the compression level that should be used (see below for supported levels) + +* these keywords apply only to the */zstd* dump styles +* keyword = *compression_level* + + .. parsed-literal:: + + *checksum* args = *yes* or *no* (add checksum at end of zst file) + Examples """""""" @@ -962,6 +977,28 @@ images less frequently. ---------- +The COMPRESS package offers both GZ and Zstd compression variants of styles +atom, custom, local, cfg, and xyz. When using these styles the compression +level can be controlled by the :code:`compression_level` parameter. File names +with these styles have to end in either :code:`.gz` or :code:`.zst`. + +GZ supports compression levels from -1 (default), 0 (no compression), and 1 to +9. 9 being the best compression. The COMPRESS :code:`/gz` styles use 9 as +default compression level. + +Zstd offers a wider range of compression levels, including negative +levels that sacrifice compression for performance. 0 is the +default, positive levels are 1 to 22, with 22 being the most expensive +compression. Zstd promises higher compression/decompression speeds for +similar compression ratios. For more details see +`http://facebook.github.io/zstd/`. + +In addition, Zstd compressed files can have a checksum of the entire +contents. The Zstd enabled dump styles enable this feature by default and it +can be disabled with the :code:`checksum` parameter. + +---------- + Restrictions """""""""""" none @@ -1010,6 +1047,10 @@ The option defaults are * color = 140 color names are pre-defined as listed below * framerate = 24 +* compression_level = 9 (gz variants) +* compression_level = 0 (zstd variants) +* checksum = yes (zstd variants) + ---------- These are the standard 109 element names that LAMMPS pre-defines for diff --git a/doc/src/write_dump.rst b/doc/src/write_dump.rst index f0f57bd674..2e3831b40f 100644 --- a/doc/src/write_dump.rst +++ b/doc/src/write_dump.rst @@ -31,6 +31,11 @@ Examples write_dump all image snap*.jpg element element & bond atom 0.3 shiny 0.1 ssao yes 6345 0.2 size 1600 1600 & modify backcolor white element C C O H N C C C O H H S O H + write_dump all atom/gz dump.atom.gz modify compression_level 9 + write_dump flow custom/zstd dump.%.myforce.zst & + id type c_myF[3] v_ke & + modify sort id & + compression_level 15 Description """"""""""" From f899a0f484fe5635f4363c1246ba29f25bb17c17 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 12:25:38 -0400 Subject: [PATCH 23/26] Reorder init to supress warnings --- src/COMPRESS/zstd_file_writer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/COMPRESS/zstd_file_writer.cpp b/src/COMPRESS/zstd_file_writer.cpp index 3a2cc15856..e764409c73 100644 --- a/src/COMPRESS/zstd_file_writer.cpp +++ b/src/COMPRESS/zstd_file_writer.cpp @@ -24,10 +24,10 @@ using namespace LAMMPS_NS; ZstdFileWriter::ZstdFileWriter() : FileWriter(), - fp(nullptr), - cctx(nullptr), compression_level(0), - checksum_flag(1) + checksum_flag(1), + cctx(nullptr), + fp(nullptr) { out_buffer_size = ZSTD_CStreamOutSize(); out_buffer = new char[out_buffer_size]; From 9d3e3526a0fc337928527461a1a8a31e1e42cc43 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 12:30:30 -0400 Subject: [PATCH 24/26] Add false positives and fix spelling in docs --- doc/src/dump.rst | 2 +- doc/utils/sphinx-config/false_positives.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/src/dump.rst b/doc/src/dump.rst index d225b88662..3f6a8812c3 100644 --- a/doc/src/dump.rst +++ b/doc/src/dump.rst @@ -202,7 +202,7 @@ For the remainder of this doc page, you should thus consider the *atom* and *atom/gz* styles (etc) to be inter-changeable, with the exception of the required filename suffix. -Similarily, the *atom/zstd*\ , *cfg/zstd*\ , *custom/zstd*\ , *local/zstd*, +Similarly, the *atom/zstd*\ , *cfg/zstd*\ , *custom/zstd*\ , *local/zstd*, and *xyz/zstd* styles are identical to the gz styles, but use the Zstd compression library instead. See the :doc:`dump_modify ` doc page for details on how to control the compression level in both variants. diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 2bf3e917cf..ddeb036acf 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -3383,3 +3383,6 @@ zz Zm PowerShell filesystems +Zstd +zstd +checksum From 40ea03234f6d6aabcf5ddeb2ce384f5a5f573198 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 12:57:47 -0400 Subject: [PATCH 25/26] Update force->inumeric to utils::inumeric --- src/COMPRESS/dump_atom_gz.cpp | 4 ++-- src/COMPRESS/dump_atom_zstd.cpp | 4 ++-- src/COMPRESS/dump_cfg_gz.cpp | 4 ++-- src/COMPRESS/dump_cfg_zstd.cpp | 4 ++-- src/COMPRESS/dump_custom_gz.cpp | 4 ++-- src/COMPRESS/dump_custom_zstd.cpp | 4 ++-- src/COMPRESS/dump_local_gz.cpp | 4 ++-- src/COMPRESS/dump_local_zstd.cpp | 4 ++-- src/COMPRESS/dump_xyz_gz.cpp | 4 ++-- src/COMPRESS/dump_xyz_zstd.cpp | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 38ec0fb59e..6c78ba7910 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -15,7 +15,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -173,7 +173,7 @@ int DumpAtomGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - compression_level = force->inumeric(FLERR,arg[1]); + compression_level = utils::inumeric(FLERR, arg[1], false, lmp); if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index 9c0f428905..a8da9a1a2c 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -21,7 +21,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -181,7 +181,7 @@ int DumpAtomZstd::modify_param(int narg, char **arg) return 2; } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int compression_level = force->inumeric(FLERR,arg[1]); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level); return 2; } diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index 25f29eda2f..2d48e02f3c 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -16,7 +16,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -181,7 +181,7 @@ int DumpCFGGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - compression_level = force->inumeric(FLERR,arg[1]); + compression_level = utils::inumeric(FLERR, arg[1], false, lmp); if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp index 125c4f3406..0f74e2550b 100644 --- a/src/COMPRESS/dump_cfg_zstd.cpp +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -21,7 +21,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -183,7 +183,7 @@ int DumpCFGZstd::modify_param(int narg, char **arg) return 2; } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int compression_level = force->inumeric(FLERR,arg[1]); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level); return 2; } diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 6770e16bbc..35a2d25a2a 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -15,7 +15,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -173,7 +173,7 @@ int DumpCustomGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - compression_level = force->inumeric(FLERR,arg[1]); + compression_level = utils::inumeric(FLERR, arg[1], false, lmp); if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index afb6d81435..395ba51fdc 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -21,7 +21,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -181,7 +181,7 @@ int DumpCustomZstd::modify_param(int narg, char **arg) return 2; } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int compression_level = force->inumeric(FLERR,arg[1]); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level); return 2; } diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index 0d1f666322..a346de3127 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -15,7 +15,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -188,7 +188,7 @@ int DumpLocalGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - compression_level = force->inumeric(FLERR,arg[1]); + compression_level = utils::inumeric(FLERR, arg[1], false, lmp); if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp index 3184dd06ad..3e658fd070 100644 --- a/src/COMPRESS/dump_local_zstd.cpp +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -21,7 +21,7 @@ #include "domain.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -181,7 +181,7 @@ int DumpLocalZstd::modify_param(int narg, char **arg) return 2; } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int compression_level = force->inumeric(FLERR,arg[1]); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level); return 2; } diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index d1f7b59dfe..79d2d20997 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -14,7 +14,7 @@ #include "dump_xyz_gz.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -152,7 +152,7 @@ int DumpXYZGZ::modify_param(int narg, char **arg) if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); int min_level = Z_DEFAULT_COMPRESSION; int max_level = Z_BEST_COMPRESSION; - compression_level = force->inumeric(FLERR,arg[1]); + compression_level = utils::inumeric(FLERR, arg[1], false, lmp); if (compression_level < min_level || compression_level > max_level) error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); return 2; diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp index 83cb90627b..066609eebb 100644 --- a/src/COMPRESS/dump_xyz_zstd.cpp +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -20,7 +20,7 @@ #include "dump_xyz_zstd.h" #include "error.h" #include "update.h" -#include "force.h" +#include "utils.h" #include #include "fmt/format.h" @@ -156,7 +156,7 @@ int DumpXYZZstd::modify_param(int narg, char **arg) return 2; } else if (strcmp(arg[0],"compression_level") == 0) { if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int compression_level = force->inumeric(FLERR,arg[1]); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level); return 2; } From 60b98f6b91ef31a1a31690e9700d2a0039f59d17 Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Tue, 1 Sep 2020 13:52:56 -0400 Subject: [PATCH 26/26] Require libzstd>=1.4 for Zstd dump styles --- cmake/Modules/Packages/COMPRESS.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/Packages/COMPRESS.cmake b/cmake/Modules/Packages/COMPRESS.cmake index f48bbe4d03..887dfb88a6 100644 --- a/cmake/Modules/Packages/COMPRESS.cmake +++ b/cmake/Modules/Packages/COMPRESS.cmake @@ -2,7 +2,7 @@ find_package(ZLIB REQUIRED) target_link_libraries(lammps PRIVATE ZLIB::ZLIB) find_package(PkgConfig REQUIRED) -pkg_check_modules(Zstd IMPORTED_TARGET libzstd) +pkg_check_modules(Zstd IMPORTED_TARGET libzstd>=1.4) if(Zstd_FOUND) target_compile_definitions(lammps PRIVATE -DLAMMPS_ZSTD)