From 511f64fde4ab569d2ac9bde67c9e9c6d6aa7e3da Mon Sep 17 00:00:00 2001 From: Richard Berger Date: Fri, 9 Apr 2021 16:36:30 -0400 Subject: [PATCH] Update dump custom/gz --- src/COMPRESS/dump_atom_gz.cpp | 4 +- src/COMPRESS/dump_atom_zstd.cpp | 4 +- src/COMPRESS/dump_cfg_gz.cpp | 4 +- src/COMPRESS/dump_cfg_zstd.cpp | 4 +- src/COMPRESS/dump_custom_gz.cpp | 98 +++++++++---------- src/COMPRESS/dump_custom_gz.h | 5 +- src/COMPRESS/dump_custom_zstd.cpp | 7 +- src/COMPRESS/dump_local_gz.cpp | 5 +- src/COMPRESS/dump_local_zstd.cpp | 4 +- src/COMPRESS/dump_xyz_gz.cpp | 4 +- src/COMPRESS/dump_xyz_zstd.cpp | 4 +- .../formats/test_dump_custom_compressed.cpp | 33 ++++++- 12 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 254e146800..b3b202e373 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -71,7 +71,9 @@ void DumpAtomGZ::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_atom_zstd.cpp b/src/COMPRESS/dump_atom_zstd.cpp index dd744b5d49..f739a53322 100644 --- a/src/COMPRESS/dump_atom_zstd.cpp +++ b/src/COMPRESS/dump_atom_zstd.cpp @@ -79,7 +79,9 @@ void DumpAtomZstd::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index 6bbf118789..23c0d82429 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -73,7 +73,9 @@ void DumpCFGGZ::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_cfg_zstd.cpp b/src/COMPRESS/dump_cfg_zstd.cpp index 978e695d1a..5bc6ac86dc 100644 --- a/src/COMPRESS/dump_cfg_zstd.cpp +++ b/src/COMPRESS/dump_cfg_zstd.cpp @@ -81,7 +81,9 @@ void DumpCFGZstd::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 4f03a4a232..5cecf22b5d 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -11,39 +11,28 @@ See the README file in the top-level LAMMPS directory. ------------------------------------------------------------------------- */ -#include "dump_custom_gz.h" #include "domain.h" +#include "dump_custom_gz.h" #include "error.h" #include "update.h" - #include - using namespace LAMMPS_NS; DumpCustomGZ::DumpCustomGZ(LAMMPS *lmp, int narg, char **arg) : DumpCustom(lmp, narg, arg) { - gzFp = nullptr; - - compression_level = Z_BEST_COMPRESSION; - if (!compressed) error->all(FLERR,"Dump custom/gz only writes compressed files"); } - /* ---------------------------------------------------------------------- */ DumpCustomGZ::~DumpCustomGZ() { - if (gzFp) gzclose(gzFp); - gzFp = nullptr; - fp = nullptr; } - /* ---------------------------------------------------------------------- generic opening of a dump file ASCII or binary or gzipped @@ -82,7 +71,9 @@ void DumpCustomGZ::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; @@ -93,17 +84,12 @@ void DumpCustomGZ::openfile() // each proc with filewriter = 1 opens a file if (filewriter) { - std::string mode; - if (append_flag) { - mode = fmt::format("ab{}", compression_level); - } else { - mode = fmt::format("wb{}", compression_level); + try { + writer.open(filecurrent, append_flag); + } catch (FileWriterException &e) { + error->one(FLERR, e.what()); } - - gzFp = gzopen(filecurrent, mode.c_str()); - - if (gzFp == nullptr) error->one(FLERR,"Cannot open dump file"); - } else gzFp = nullptr; + } // delete string with timestep replaced @@ -112,29 +98,34 @@ void DumpCustomGZ::openfile() void DumpCustomGZ::write_header(bigint ndump) { + std::string header; + if ((multiproc) || (!multiproc && me == 0)) { if (unit_flag && !unit_count) { ++unit_count; - gzprintf(gzFp,"ITEM: UNITS\n%s\n",update->unit_style); + header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style); } - if (time_flag) gzprintf(gzFp,"ITEM: TIME\n%.16g\n",compute_time()); - gzprintf(gzFp,"ITEM: TIMESTEP\n"); - gzprintf(gzFp,BIGINT_FORMAT "\n",update->ntimestep); - gzprintf(gzFp,"ITEM: NUMBER OF ATOMS\n"); - gzprintf(gzFp,BIGINT_FORMAT "\n",ndump); - if (domain->triclinic == 0) { - gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr); - gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi); - gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi); - gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi); - } else { - gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr); - gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy); - gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz); - gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz); + if (time_flag) { + header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time()); } - gzprintf(gzFp,"ITEM: ATOMS %s\n",columns); + + header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); + header += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); + if (domain->triclinic == 0) { + header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi); + header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi); + } else { + header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz); + header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz); + } + header += fmt::format("ITEM: ATOMS {}\n", columns); + + writer.write(header.c_str(), header.length()); } } @@ -142,7 +133,7 @@ void DumpCustomGZ::write_header(bigint ndump) void DumpCustomGZ::write_data(int n, double *mybuf) { - gzwrite(gzFp,mybuf,sizeof(char)*n); + writer.write(mybuf, n); } /* ---------------------------------------------------------------------- */ @@ -152,11 +143,11 @@ void DumpCustomGZ::write() DumpCustom::write(); if (filewriter) { if (multifile) { - gzclose(gzFp); - gzFp = nullptr; + writer.close(); } else { - if (flush_flag) - gzflush(gzFp,Z_SYNC_FLUSH); + if (flush_flag && writer.isopen()) { + writer.flush(); + } } } } @@ -167,14 +158,15 @@ int DumpCustomGZ::modify_param(int narg, char **arg) { int consumed = DumpCustom::modify_param(narg, arg); if (consumed == 0) { - if (strcmp(arg[0],"compression_level") == 0) { - if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); - int min_level = Z_DEFAULT_COMPRESSION; - int max_level = Z_BEST_COMPRESSION; - compression_level = utils::inumeric(FLERR, arg[1], false, lmp); - if (compression_level < min_level || compression_level > max_level) - error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); - return 2; + try { + if (strcmp(arg[0],"compression_level") == 0) { + if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); + int compression_level = utils::inumeric(FLERR, arg[1], false, lmp); + writer.setCompressionLevel(compression_level); + return 2; + } + } catch (FileWriterException &e) { + error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what())); } } return consumed; diff --git a/src/COMPRESS/dump_custom_gz.h b/src/COMPRESS/dump_custom_gz.h index 184f3563f1..db30b944ec 100644 --- a/src/COMPRESS/dump_custom_gz.h +++ b/src/COMPRESS/dump_custom_gz.h @@ -21,7 +21,7 @@ DumpStyle(custom/gz,DumpCustomGZ) #define LMP_DUMP_CUSTOM_GZ_H #include "dump_custom.h" -#include +#include "gz_file_writer.h" namespace LAMMPS_NS { @@ -31,8 +31,7 @@ class DumpCustomGZ : public DumpCustom { virtual ~DumpCustomGZ(); protected: - int compression_level; - gzFile gzFp; // file pointer for the compressed output stream + GzFileWriter writer; virtual void openfile(); virtual void write_header(bigint); diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index 3aa3f874ea..bd248bd0fc 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -20,7 +20,6 @@ #include "domain.h" #include "dump_custom_zstd.h" #include "error.h" -#include "file_writer.h" #include "update.h" #include @@ -79,7 +78,9 @@ void DumpCustomZstd::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; @@ -184,7 +185,7 @@ int DumpCustomZstd::modify_param(int narg, char **arg) return 2; } } catch (FileWriterException &e) { - error->one(FLERR, e.what()); + error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what())); } } return consumed; diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index a0a39b51e0..9b0ac0c344 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -27,7 +27,6 @@ DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) : error->all(FLERR,"Dump local/gz only writes compressed files"); } - /* ---------------------------------------------------------------------- */ DumpLocalGZ::~DumpLocalGZ() @@ -72,7 +71,9 @@ void DumpLocalGZ::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp index a4303f3b25..05cd0bb1ae 100644 --- a/src/COMPRESS/dump_local_zstd.cpp +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -78,7 +78,9 @@ void DumpLocalZstd::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index 5c90c48d0f..0697f19ce3 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -71,7 +71,9 @@ void DumpXYZGZ::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp index 03edf561b1..cb75542337 100644 --- a/src/COMPRESS/dump_xyz_zstd.cpp +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -78,7 +78,9 @@ void DumpXYZZstd::openfile() nameslist[numfiles] = utils::strdup(filecurrent); ++numfiles; } else { - remove(nameslist[fileidx]); + if (remove(nameslist[fileidx]) != 0) { + error->warning(FLERR, fmt::format("Could not delete {}", nameslist[fileidx])); + } delete[] nameslist[fileidx]; nameslist[fileidx] = utils::strdup(filecurrent); fileidx = (fileidx + 1) % maxfiles; diff --git a/unittest/formats/test_dump_custom_compressed.cpp b/unittest/formats/test_dump_custom_compressed.cpp index fb70206590..5ad231d440 100644 --- a/unittest/formats/test_dump_custom_compressed.cpp +++ b/unittest/formats/test_dump_custom_compressed.cpp @@ -58,6 +58,35 @@ TEST_F(DumpCustomCompressTest, compressed_run1) delete_file(converted_file); } +TEST_F(DumpCustomCompressTest, compressed_with_time_run1) +{ + if (!COMPRESS_BINARY) GTEST_SKIP(); + + auto base_name = "with_time_custom_run1.melt"; + auto text_file = text_dump_filename(base_name); + auto compressed_file = compressed_dump_filename(base_name); + auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; + + if(compression_style == "custom/zstd") { + generate_text_and_compressed_dump(text_file, compressed_file, fields, fields, "time yes", "time yes checksum yes", 1); + } else { + generate_text_and_compressed_dump(text_file, compressed_file, fields, "time yes", 1); + } + + TearDown(); + + ASSERT_FILE_EXISTS(text_file); + ASSERT_FILE_EXISTS(compressed_file); + + auto converted_file = convert_compressed_to_text(compressed_file); + + ASSERT_FILE_EXISTS(converted_file); + ASSERT_FILE_EQUAL(text_file, converted_file); + delete_file(text_file); + delete_file(compressed_file); + delete_file(converted_file); +} + TEST_F(DumpCustomCompressTest, compressed_triclinic_run1) { if (!COMPRESS_BINARY) GTEST_SKIP(); @@ -222,7 +251,7 @@ TEST_F(DumpCustomCompressTest, compressed_modify_bad_param) auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_bad_param_run0_*.melt.custom"), fields)); - TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*", + TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*", command("dump_modify id1 compression_level 12"); ); } @@ -234,7 +263,7 @@ TEST_F(DumpCustomCompressTest, compressed_modify_multi_bad_param) auto fields = "id type proc x y z ix iy iz xs ys zs xu yu zu xsu ysu zsu vx vy vz fx fy fz"; command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_multi_bad_param_run0_*.melt.custom"), fields)); - TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*", + TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*", command("dump_modify id1 pad 3 compression_level 12"); ); }