ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update dump local and local/gz

Browse Source
This commit is contained in:
Richard Berger
2021-04-09 15:42:28 -04:00
parent d19cd8fb11
commit cf41ea6faf
5 changed files with 285 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
src/COMPRESS/dump_local_gz.cpp
View File

@ -11,24 +11,18 @@
See the README file in the top-level LAMMPS directory. See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
#include "dump_local_gz.h"
#include "domain.h" #include "domain.h"
#include "dump_local_gz.h"
#include "error.h" #include "error.h"
#include "update.h" #include "update.h"
#include <cstring> #include <cstring>
using namespace LAMMPS_NS; using namespace LAMMPS_NS;
DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) : DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) :
DumpLocal(lmp, narg, arg) DumpLocal(lmp, narg, arg)
{ {
gzFp = nullptr;
compression_level = Z_BEST_COMPRESSION;
if (!compressed) if (!compressed)
error->all(FLERR,"Dump local/gz only writes compressed files"); error->all(FLERR,"Dump local/gz only writes compressed files");
} }
@ -38,12 +32,8 @@ DumpLocalGZ::DumpLocalGZ(LAMMPS *lmp, int narg, char **arg) :
DumpLocalGZ::~DumpLocalGZ() DumpLocalGZ::~DumpLocalGZ()
{ {
if (gzFp) gzclose(gzFp);
gzFp = nullptr;
fp = nullptr;
} }
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or gzipped
@ -93,17 +83,12 @@ void DumpLocalGZ::openfile()
// each proc with filewriter = 1 opens a file // each proc with filewriter = 1 opens a file
if (filewriter) { if (filewriter) {
std::string mode; try {
if (append_flag) { writer.open(filecurrent, append_flag);
mode = fmt::format("ab{}", compression_level); } catch (FileWriterException &e) {
} else { error->one(FLERR, e.what());
mode = fmt::format("wb{}", compression_level);
} }
}
gzFp = gzopen(filecurrent, mode.c_str());
if (gzFp == nullptr) error->one(FLERR,"Cannot open dump file");
} else gzFp = nullptr;
// delete string with timestep replaced // delete string with timestep replaced
@ -112,29 +97,34 @@ void DumpLocalGZ::openfile()
void DumpLocalGZ::write_header(bigint ndump) void DumpLocalGZ::write_header(bigint ndump)
{ {
std::string header;
if ((multiproc) || (!multiproc && me == 0)) { if ((multiproc) || (!multiproc && me == 0)) {
if (unit_flag && !unit_count) { if (unit_flag && !unit_count) {
++unit_count; ++unit_count;
gzprintf(gzFp,"ITEM: UNITS\n%s\n",update->unit_style); header = fmt::format("ITEM: UNITS\n{}\n",update->unit_style);
} }
if (time_flag) gzprintf(gzFp,"ITEM: TIME\n%.16g\n",compute_time());
gzprintf(gzFp,"ITEM: TIMESTEP\n"); if (time_flag) {
gzprintf(gzFp,BIGINT_FORMAT "\n",update->ntimestep); header += fmt::format("ITEM: TIME\n{0:.16g}\n", compute_time());
gzprintf(gzFp,"ITEM: NUMBER OF %s\n",label);
gzprintf(gzFp,BIGINT_FORMAT "\n",ndump);
if (domain->triclinic) {
gzprintf(gzFp,"ITEM: BOX BOUNDS xy xz yz %s\n",boundstr);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxxlo,boxxhi,boxxy);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxylo,boxyhi,boxxz);
gzprintf(gzFp,"%-1.16e %-1.16e %-1.16e\n",boxzlo,boxzhi,boxyz);
} else {
gzprintf(gzFp,"ITEM: BOX BOUNDS %s\n",boundstr);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxxlo,boxxhi);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxylo,boxyhi);
gzprintf(gzFp,"%-1.16e %-1.16e\n",boxzlo,boxzhi);
} }
gzprintf(gzFp,"ITEM: %s %s\n",label,columns);
header += fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep);
header += fmt::format("ITEM: NUMBER OF {}\n{}\n", label, ndump);
if (domain->triclinic == 0) {
header += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxxlo, boxxhi);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxylo, boxyhi);
header += fmt::format("{0:-1.16e} {1:-1.16e}\n", boxzlo, boxzhi);
} else {
header += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxxlo, boxxhi, boxxy);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxylo, boxyhi, boxxz);
header += fmt::format("{0:-1.16e} {1:-1.16e} {2:-1.16e}\n", boxzlo, boxzhi, boxyz);
}
header += fmt::format("ITEM: {} {}\n", label, columns);
writer.write(header.c_str(), header.length());
} }
} }
@ -143,19 +133,28 @@ void DumpLocalGZ::write_header(bigint ndump)
void DumpLocalGZ::write_data(int n, double *mybuf) void DumpLocalGZ::write_data(int n, double *mybuf)
{ {
if (buffer_flag == 1) { if (buffer_flag == 1) {
gzwrite(gzFp,mybuf,sizeof(char)*n); writer.write(mybuf, sizeof(char)*n);
} else { } else {
int i,j; constexpr size_t VBUFFER_SIZE = 256;
char vbuffer[VBUFFER_SIZE];
int m = 0; int m = 0;
for (i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
for (j = 0; j < size_one; j++) { for (int j = 0; j < size_one; j++) {
if (vtype[j] == INT) int written = 0;
gzprintf(gzFp,vformat[j],static_cast<int> (mybuf[m])); if (vtype[j] == Dump::INT) {
else gzprintf(gzFp,vformat[j],mybuf[m]); written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], static_cast<int> (mybuf[m]));
} else {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], mybuf[m]);
}
if (written > 0) {
writer.write(vbuffer, written);
} else if (written < 0) {
error->one(FLERR, "Error while writing dump local/gz output");
}
m++; m++;
} }
gzprintf(gzFp,"\n"); writer.write("\n", 1);
} }
} }
} }
@ -167,11 +166,11 @@ void DumpLocalGZ::write()
DumpLocal::write(); DumpLocal::write();
if (filewriter) { if (filewriter) {
if (multifile) { if (multifile) {
gzclose(gzFp); writer.close();
gzFp = nullptr;
} else { } else {
if (flush_flag) if (flush_flag && writer.isopen()) {
gzflush(gzFp,Z_SYNC_FLUSH); writer.flush();
}
} }
} }
} }
@ -182,14 +181,15 @@ int DumpLocalGZ::modify_param(int narg, char **arg)
{ {
int consumed = DumpLocal::modify_param(narg, arg); int consumed = DumpLocal::modify_param(narg, arg);
if (consumed == 0) { if (consumed == 0) {
if (strcmp(arg[0],"compression_level") == 0) { try {
if (narg < 2) error->all(FLERR,"Illegal dump_modify command"); if (strcmp(arg[0],"compression_level") == 0) {
int min_level = Z_DEFAULT_COMPRESSION; if (narg < 2) error->all(FLERR,"Illegal dump_modify command");
int max_level = Z_BEST_COMPRESSION; int compression_level = utils::inumeric(FLERR, arg[1], false, lmp);
compression_level = utils::inumeric(FLERR, arg[1], false, lmp); writer.setCompressionLevel(compression_level);
if (compression_level < min_level || compression_level > max_level) return 2;
error->all(FLERR, fmt::format("Illegal dump_modify command: compression level must in the range of [{}, {}]", min_level, max_level)); }
return 2; } catch (FileWriterException &e) {
error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what()));
} }
} }
return consumed; return consumed;

5
src/COMPRESS/dump_local_gz.h
View File

@ -21,7 +21,7 @@ DumpStyle(local/gz,DumpLocalGZ)
#define LMP_DUMP_LOCAL_GZ_H #define LMP_DUMP_LOCAL_GZ_H
#include "dump_local.h" #include "dump_local.h"
#include <zlib.h> #include "gz_file_writer.h"
namespace LAMMPS_NS { namespace LAMMPS_NS {
@ -31,8 +31,7 @@ class DumpLocalGZ : public DumpLocal {
virtual ~DumpLocalGZ(); virtual ~DumpLocalGZ();
protected: protected:
int compression_level; GzFileWriter writer;
gzFile gzFp; // file pointer for the compressed output stream
virtual void openfile(); virtual void openfile();
virtual void write_header(bigint); virtual void write_header(bigint);

33
src/COMPRESS/dump_local_zstd.cpp
View File

@ -17,15 +17,13 @@
#ifdef LAMMPS_ZSTD #ifdef LAMMPS_ZSTD
#include "dump_local_zstd.h"
#include "domain.h" #include "domain.h"
#include "dump_local_zstd.h"
#include "error.h" #include "error.h"
#include "update.h" #include "update.h"
#include <cstring> #include <cstring>
using namespace LAMMPS_NS; using namespace LAMMPS_NS;
DumpLocalZstd::DumpLocalZstd(LAMMPS *lmp, int narg, char **arg) : DumpLocalZstd::DumpLocalZstd(LAMMPS *lmp, int narg, char **arg) :
@ -42,7 +40,6 @@ DumpLocalZstd::~DumpLocalZstd()
{ {
} }
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or gzipped
@ -145,7 +142,31 @@ void DumpLocalZstd::write_header(bigint ndump)
void DumpLocalZstd::write_data(int n, double *mybuf) void DumpLocalZstd::write_data(int n, double *mybuf)
{ {
writer.write(mybuf, sizeof(char)*n); if (buffer_flag == 1) {
writer.write(mybuf, sizeof(char)*n);
} else {
constexpr size_t VBUFFER_SIZE = 256;
char vbuffer[VBUFFER_SIZE];
int m = 0;
for (int i = 0; i < n; i++) {
for (int j = 0; j < size_one; j++) {
int written = 0;
if (vtype[j] == Dump::INT) {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], static_cast<int> (mybuf[m]));
} else {
written = snprintf(vbuffer, VBUFFER_SIZE, vformat[j], mybuf[m]);
}
if (written > 0) {
writer.write(vbuffer, written);
} else if (written < 0) {
error->one(FLERR, "Error while writing dump local/gz output");
}
m++;
}
writer.write("\n", 1);
}
}
} }
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
@ -184,7 +205,7 @@ int DumpLocalZstd::modify_param(int narg, char **arg)
return 2; return 2;
} }
} catch (FileWriterException &e) { } catch (FileWriterException &e) {
error->one(FLERR, e.what()); error->one(FLERR, fmt::format("Illegal dump_modify command: {}", e.what()));
} }
} }
return consumed; return consumed;

66
unittest/formats/test_dump_local.cpp
View File

@ -93,6 +93,18 @@ TEST_F(DumpLocalTest, run0)
delete_file(dump_file); delete_file(dump_file);
} }
TEST_F(DumpLocalTest, label_run0)
{
auto dump_file = "dump_local_label_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "label ELEMENTS", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_THAT(lines[2], Eq("ITEM: NUMBER OF ELEMENTS"));
ASSERT_THAT(lines[8], Eq("ITEM: ELEMENTS index c_comp[1] "));
delete_file(dump_file);
}
TEST_F(DumpLocalTest, format_line_run0) TEST_F(DumpLocalTest, format_line_run0)
{ {
auto dump_file = "dump_local_format_line_run0.melt"; auto dump_file = "dump_local_format_line_run0.melt";
@ -170,6 +182,60 @@ TEST_F(DumpLocalTest, no_buffer_run0)
delete_file(dump_file); delete_file(dump_file);
} }
TEST_F(DumpLocalTest, with_units_run0)
{
auto dump_file = "dump_with_units_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "units yes", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_EQ(lines.size(), 875);
ASSERT_THAT(lines[0], Eq("ITEM: UNITS"));
ASSERT_THAT(lines[1], Eq("lj"));
ASSERT_THAT(lines[2], Eq("ITEM: TIMESTEP"));
ASSERT_EQ(std::stoi(lines[3]), 0);
ASSERT_THAT(lines[4], Eq("ITEM: NUMBER OF ENTRIES"));
ASSERT_EQ(std::stoi(lines[5]), 864);
}
TEST_F(DumpLocalTest, with_time_run0)
{
auto dump_file = "dump_with_time_run0.melt";
generate_dump(dump_file, "index c_comp[1]", "time yes", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_EQ(lines.size(), 875);
ASSERT_THAT(lines[0], Eq("ITEM: TIME"));
ASSERT_THAT(std::stof(lines[1]), 0.0);
ASSERT_THAT(lines[2], Eq("ITEM: TIMESTEP"));
ASSERT_EQ(std::stoi(lines[3]), 0);
ASSERT_THAT(lines[4], Eq("ITEM: NUMBER OF ENTRIES"));
ASSERT_EQ(std::stoi(lines[5]), 864);
}
TEST_F(DumpLocalTest, triclinic_run0)
{
auto dump_file = "dump_local_triclinic_run0.melt";
enable_triclinic();
generate_dump(dump_file, "index c_comp[1]", "", 0);
ASSERT_FILE_EXISTS(dump_file);
auto lines = read_lines(dump_file);
ASSERT_THAT(lines[4], Eq("ITEM: BOX BOUNDS xy xz yz pp pp pp"));
ASSERT_EQ(utils::split_words(lines[5]).size(), 3);
ASSERT_EQ(utils::split_words(lines[6]).size(), 3);
ASSERT_EQ(utils::split_words(lines[7]).size(), 3);
delete_file(dump_file);
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
MPI_Init(&argc, &argv); MPI_Init(&argc, &argv);

133
unittest/formats/test_dump_local_compressed.cpp
View File

@ -69,6 +69,135 @@ TEST_F(DumpLocalCompressTest, compressed_run0)
delete_file(converted_file_0); delete_file(converted_file_0);
} }
TEST_F(DumpLocalCompressTest, compressed_no_buffer_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "no_buffer_run*.melt.local";
auto base_name_0 = "no_buffer_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "buffer no", "buffer no checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "buffer no", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_with_time_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "with_time_run*.melt.local";
auto base_name_0 = "with_time_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "time yes", "time yes checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "time yes", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_with_units_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
auto base_name = "with_units_run*.melt.local";
auto base_name_0 = "with_units_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "units yes", "units yes checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "units yes", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_triclinic_run0)
{
if (!COMPRESS_BINARY) GTEST_SKIP();
enable_triclinic();
auto base_name = "triclinic_run*.melt.local";
auto base_name_0 = "triclinic_run0.melt.local";
auto text_files = text_dump_filename(base_name);
auto compressed_files = compressed_dump_filename(base_name);
auto text_file_0 = text_dump_filename(base_name_0);
auto compressed_file_0 = compressed_dump_filename(base_name_0);
auto fields = "index c_comp[1]";
if(compression_style == "local/zstd") {
generate_text_and_compressed_dump(text_files, compressed_files, fields, fields, "", "checksum yes", 0);
} else {
generate_text_and_compressed_dump(text_files, compressed_files, fields, "", 0);
}
TearDown();
ASSERT_FILE_EXISTS(text_file_0);
ASSERT_FILE_EXISTS(compressed_file_0);
auto converted_file_0 = convert_compressed_to_text(compressed_file_0);
ASSERT_FILE_EXISTS(converted_file_0);
ASSERT_FILE_EQUAL(text_file_0, converted_file_0);
delete_file(text_file_0);
delete_file(compressed_file_0);
delete_file(converted_file_0);
}
TEST_F(DumpLocalCompressTest, compressed_multi_file_run1) TEST_F(DumpLocalCompressTest, compressed_multi_file_run1)
{ {
if (!COMPRESS_BINARY) GTEST_SKIP(); if (!COMPRESS_BINARY) GTEST_SKIP();
@ -209,7 +338,7 @@ TEST_F(DumpLocalCompressTest, compressed_modify_bad_param)
command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_bad_param_run0_*.melt.local"), fields)); command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_bad_param_run0_*.melt.local"), fields));
END_HIDE_OUTPUT(); END_HIDE_OUTPUT();
TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*", TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*",
command("dump_modify id1 compression_level 12"); command("dump_modify id1 compression_level 12");
); );
} }
@ -224,7 +353,7 @@ TEST_F(DumpLocalCompressTest, compressed_modify_multi_bad_param)
command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_multi_bad_param_run0_*.melt.local"), fields)); command(fmt::format("dump id1 all {} 1 {} {}", compression_style, compressed_dump_filename("modify_multi_bad_param_run0_*.melt.local"), fields));
END_HIDE_OUTPUT(); END_HIDE_OUTPUT();
TEST_FAILURE(".*ERROR: Illegal dump_modify command: compression level must in the range of.*", TEST_FAILURE(".*ERROR on proc 0: Illegal dump_modify command: Compression level must in the range of.*",
command("dump_modify id1 pad 3 compression_level 12"); command("dump_modify id1 pad 3 compression_level 12");
); );
} }