diff --git a/doc/src/Build_settings.rst b/doc/src/Build_settings.rst index c7397935d8..01bd8e2c80 100644 --- a/doc/src/Build_settings.rst +++ b/doc/src/Build_settings.rst @@ -354,8 +354,10 @@ Read or write compressed files ----------------------------------------- If this option is enabled, large files can be read or written with -gzip compression by several LAMMPS commands, including -:doc:`read_data `, :doc:`rerun `, and :doc:`dump `. +compression by ``gzip`` or similar tools by several LAMMPS commands, +including :doc:`read_data `, :doc:`rerun `, and +:doc:`dump `. Currently supported compression tools are: +``gzip``, ``bzip2``, ``zstd``, and ``lzma``. .. tabs:: @@ -364,8 +366,7 @@ gzip compression by several LAMMPS commands, including .. code-block:: bash -D WITH_GZIP=value # yes or no - # default is yes if CMake can find gzip, else no - -D GZIP_EXECUTABLE=path # path to gzip executable if CMake cannot find it + # default is yes if CMake can find the gzip program, else no .. tab:: Traditional make @@ -373,14 +374,15 @@ gzip compression by several LAMMPS commands, including LMP_INC = -DLAMMPS_GZIP -This option requires that your operating system fully supports the "popen()" -function in the standard runtime library and that a ``gzip`` executable can be -found by LAMMPS during a run. +This option requires that your operating system fully supports the +"popen()" function in the standard runtime library and that a ``gzip`` +or other executable can be found by LAMMPS in the standard search path +during a run. .. note:: - On some clusters with high-speed networks, using the "fork()" library - call (required by "popen()") can interfere with the fast communication + On clusters with high-speed networks, using the "fork()" library call + (required by "popen()") can interfere with the fast communication library and lead to simulations using compressed output or input to hang or crash. For selected operations, compressed file I/O is also available using a compression library instead, which is what the diff --git a/doc/src/Developer_platform.rst b/doc/src/Developer_platform.rst index 19836b1ff2..4cea36f94c 100644 --- a/doc/src/Developer_platform.rst +++ b/doc/src/Developer_platform.rst @@ -139,11 +139,11 @@ Dynamically loaded object or library functions Compressed file I/O functions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. doxygenfunction:: has_zip_extension +.. doxygenfunction:: has_compress_extension :project: progguide -.. doxygenfunction:: zip_read +.. doxygenfunction:: compressed_read :project: progguide -.. doxygenfunction:: zip_write +.. doxygenfunction:: compressed_write :project: progguide diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 107e252074..ccc321c7e0 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -1223,6 +1223,7 @@ Guo gw gyromagnetic gz +gzip gzipped Haak Hafskjold diff --git a/src/COMPRESS/dump_atom_gz.cpp b/src/COMPRESS/dump_atom_gz.cpp index 38229a71bb..5f799054bb 100644 --- a/src/COMPRESS/dump_atom_gz.cpp +++ b/src/COMPRESS/dump_atom_gz.cpp @@ -33,7 +33,7 @@ DumpAtomGZ::~DumpAtomGZ() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_cfg_gz.cpp b/src/COMPRESS/dump_cfg_gz.cpp index ea51ce87c3..8d6347d978 100644 --- a/src/COMPRESS/dump_cfg_gz.cpp +++ b/src/COMPRESS/dump_cfg_gz.cpp @@ -35,7 +35,7 @@ DumpCFGGZ::~DumpCFGGZ() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_custom_gz.cpp b/src/COMPRESS/dump_custom_gz.cpp index 7cbab10cb4..f40cf8d3d1 100644 --- a/src/COMPRESS/dump_custom_gz.cpp +++ b/src/COMPRESS/dump_custom_gz.cpp @@ -33,7 +33,7 @@ DumpCustomGZ::~DumpCustomGZ() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_custom_zstd.cpp b/src/COMPRESS/dump_custom_zstd.cpp index 8b7d153fc4..347c9c08e2 100644 --- a/src/COMPRESS/dump_custom_zstd.cpp +++ b/src/COMPRESS/dump_custom_zstd.cpp @@ -45,7 +45,7 @@ DumpCustomZstd::~DumpCustomZstd() /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_local_gz.cpp b/src/COMPRESS/dump_local_gz.cpp index 206e2aeb09..a23fa1a1ac 100644 --- a/src/COMPRESS/dump_local_gz.cpp +++ b/src/COMPRESS/dump_local_gz.cpp @@ -33,7 +33,7 @@ DumpLocalGZ::~DumpLocalGZ() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_local_zstd.cpp b/src/COMPRESS/dump_local_zstd.cpp index 91fb36bb6e..1d3fc4756e 100644 --- a/src/COMPRESS/dump_local_zstd.cpp +++ b/src/COMPRESS/dump_local_zstd.cpp @@ -39,7 +39,7 @@ DumpLocalZstd::~DumpLocalZstd() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_xyz_gz.cpp b/src/COMPRESS/dump_xyz_gz.cpp index a1acad7fc2..2d9548bf14 100644 --- a/src/COMPRESS/dump_xyz_gz.cpp +++ b/src/COMPRESS/dump_xyz_gz.cpp @@ -32,7 +32,7 @@ DumpXYZGZ::~DumpXYZGZ() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/COMPRESS/dump_xyz_zstd.cpp b/src/COMPRESS/dump_xyz_zstd.cpp index 534bc0ee56..b56488ec09 100644 --- a/src/COMPRESS/dump_xyz_zstd.cpp +++ b/src/COMPRESS/dump_xyz_zstd.cpp @@ -38,7 +38,7 @@ DumpXYZZstd::~DumpXYZZstd() {} /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ diff --git a/src/EXTRA-FIX/fix_tmd.cpp b/src/EXTRA-FIX/fix_tmd.cpp index 123814b22a..4d85687e4d 100644 --- a/src/EXTRA-FIX/fix_tmd.cpp +++ b/src/EXTRA-FIX/fix_tmd.cpp @@ -517,14 +517,14 @@ void FixTMD::readfile(char *file) /* ---------------------------------------------------------------------- proc 0 opens TMD data file - test if gzipped + test if compressed ------------------------------------------------------------------------- */ void FixTMD::open(const std::string &file) { - if (platform::has_zip_extension(file)) { + if (platform::has_compress_extension(file)) { compressed = 1; - fp = platform::zip_read(file); + fp = platform::compressed_read(file); if (!fp) error->one(FLERR,"Cannot open compressed file for reading"); } else { compressed = 0; diff --git a/src/PHONON/dynamical_matrix.cpp b/src/PHONON/dynamical_matrix.cpp index 8861a87491..e236e24a15 100644 --- a/src/PHONON/dynamical_matrix.cpp +++ b/src/PHONON/dynamical_matrix.cpp @@ -211,7 +211,7 @@ void DynamicalMatrix::options(int narg, char **arg) /* ---------------------------------------------------------------------- generic opening of a file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ @@ -223,8 +223,8 @@ void DynamicalMatrix::openfile(const char *filename) if (me == 0) { if (compressed) { - fp = platform::zip_write(std::string(filename)+".gz"); - if (!fp) error->one(FLERR,"Cannot open gzipped file"); + fp = platform::compressed_write(std::string(filename)+".gz"); + if (!fp) error->one(FLERR,"Cannot open compressed file"); } else if (binaryflag) { fp = fopen(filename,"wb"); } else { diff --git a/src/PHONON/third_order.cpp b/src/PHONON/third_order.cpp index 04a2417f13..3206882a4a 100644 --- a/src/PHONON/third_order.cpp +++ b/src/PHONON/third_order.cpp @@ -210,7 +210,7 @@ void ThirdOrder::options(int narg, char **arg) /* ---------------------------------------------------------------------- generic opening of a file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ @@ -222,8 +222,8 @@ void ThirdOrder::openfile(const char* filename) if (me == 0) { if (compressed) { - fp = platform::zip_write(std::string(filename)+".gz"); - if (!fp) error->one(FLERR,"Cannot open gzipped file"); + fp = platform::compressed_write(std::string(filename)+".gz"); + if (!fp) error->one(FLERR,"Cannot open compressed file"); } else if (binaryflag) { fp = fopen(filename,"wb"); } else { diff --git a/src/REAXFF/fix_reaxff_bonds.cpp b/src/REAXFF/fix_reaxff_bonds.cpp index 07d0c73f3d..740a32a0a6 100644 --- a/src/REAXFF/fix_reaxff_bonds.cpp +++ b/src/REAXFF/fix_reaxff_bonds.cpp @@ -52,9 +52,9 @@ FixReaxFFBonds::FixReaxFFBonds(LAMMPS *lmp, int narg, char **arg) : if (nevery <= 0) error->all(FLERR,"Illegal fix reaxff/bonds command"); if (me == 0) { - if (platform::has_zip_extension(arg[4])) { + if (platform::has_compress_extension(arg[4])) { compressed = 1; - fp = platform::zip_write(arg[4]); + fp = platform::compressed_write(arg[4]); if (!fp) error->one(FLERR,"Cannot open compressed file"); } else fp = fopen(arg[4],"w"); diff --git a/src/REAXFF/fix_reaxff_species.cpp b/src/REAXFF/fix_reaxff_species.cpp index 897f36d988..7bf6f34aa7 100644 --- a/src/REAXFF/fix_reaxff_species.cpp +++ b/src/REAXFF/fix_reaxff_species.cpp @@ -107,8 +107,8 @@ FixReaxFFSpecies::FixReaxFFSpecies(LAMMPS *lmp, int narg, char **arg) : strcpy(tmparg[2],arg[5]); if (me == 0) { - if (platform::has_zip_extension(arg[6])) { - fp = platform::zip_write(arg[6]); + if (platform::has_compress_extension(arg[6])) { + fp = platform::compressed_write(arg[6]); compressed = 1; if (!fp) error->one(FLERR,"Cannot open compressed file"); } else fp = fopen(arg[6],"w"); diff --git a/src/REPLICA/neb.cpp b/src/REPLICA/neb.cpp index 318ee0a3c7..8cfc4b5c24 100644 --- a/src/REPLICA/neb.cpp +++ b/src/REPLICA/neb.cpp @@ -548,9 +548,9 @@ void NEB::readfile(char *file, int flag) void NEB::open(char *file) { compressed = 0; - if (platform::has_zip_extension(file)) { + if (platform::has_compress_extension(file)) { compressed = 1; - fp = platform::zip_read(file); + fp = platform::compressed_read(file); if (!fp) error->one(FLERR,"Cannot open compressed file"); } else fp = fopen(file,"r"); diff --git a/src/SPIN/neb_spin.cpp b/src/SPIN/neb_spin.cpp index 80f5f56324..1c987f8e7e 100644 --- a/src/SPIN/neb_spin.cpp +++ b/src/SPIN/neb_spin.cpp @@ -687,14 +687,14 @@ int NEBSpin::initial_rotation(double *spi, double *sploc, double fraction) /* ---------------------------------------------------------------------- universe proc 0 opens NEBSpin data file - test if gzipped + test if compressed ------------------------------------------------------------------------- */ void NEBSpin::open(char *file) { compressed = 0; - if (platform::has_zip_extension(file)) { - fp = platform::zip_read(file); + if (platform::has_compress_extension(file)) { + fp = platform::compressed_read(file); if (!fp) error->one(FLERR,"Cannot open compressed file"); } else fp = fopen(file,"r"); diff --git a/src/dump.cpp b/src/dump.cpp index 7dbe95c1d3..d00c42086d 100644 --- a/src/dump.cpp +++ b/src/dump.cpp @@ -143,7 +143,7 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : Pointers(lmp) if (strchr(filename,'*')) multifile = 1; if (utils::strmatch(filename, "\\.bin$")) binary = 1; - if (platform::has_zip_extension(filename)) compressed = 1; + if (platform::has_compress_extension(filename)) compressed = 1; } /* ---------------------------------------------------------------------- */ @@ -528,7 +528,7 @@ void Dump::write() /* ---------------------------------------------------------------------- generic opening of a dump file - ASCII or binary or gzipped + ASCII or binary or compressed some derived classes override this function ------------------------------------------------------------------------- */ @@ -578,7 +578,7 @@ void Dump::openfile() if (filewriter) { if (compressed) { - fp = platform::zip_write(filecurrent); + fp = platform::compressed_write(filecurrent); } else if (binary) { fp = fopen(filecurrent,"wb"); } else if (append_flag) { diff --git a/src/library.cpp b/src/library.cpp index bf5ea50f58..ed208413e6 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -4420,15 +4420,16 @@ int lammps_config_has_mpi_support() /* ---------------------------------------------------------------------- */ -/** Check if the LAMMPS library supports compressed files via a pipe to gzip +/** Check if the LAMMPS library supports reading or writing compressed + * files via a pipe to gzip or similar compression programs \verbatim embed:rst Several LAMMPS commands (e.g. :doc:`read_data`, :doc:`write_data`, :doc:`dump styles atom, custom, and xyz `) support reading and writing compressed files via creating a pipe to the ``gzip`` program. This function checks whether this feature was :ref:`enabled at compile -time `. It does **not** check whether the ``gzip`` itself is -installed and usable. +time `. It does **not** check whether``gzip`` or any other +supported compression programs themselves are installed and usable. \endverbatim * * \return 1 if yes, otherwise 0 diff --git a/src/platform.cpp b/src/platform.cpp index 2d56adcc11..00636ec703 100644 --- a/src/platform.cpp +++ b/src/platform.cpp @@ -69,40 +69,40 @@ /* ------------------------------------------------------------------ */ /// Struct for listing on-the-fly compression/decompression commands -struct zip_info { +struct compress_info { /// identifier for the different compression algorithms enum styles { NONE, GZIP, BZIP2, ZSTD, XZ, LZMA, LZ4 }; const std::string extension; ///< filename extension for the current algorithm const std::string command; ///< command to perform compression or decompression - const std::string zipflags; ///< flags to append to compress from stdin to stdout - const std::string unzipflags; ///< flags to decompress file to stdout + const std::string compressflags; ///< flags to append to compress from stdin to stdout + const std::string uncompressflags; ///< flags to decompress file to stdout const int style; ///< compression style flag }; // clang-format off -static const std::vector zip_styles = { - {"", "", "", "", zip_info::NONE}, - {"gz", "gzip", " > ", " -cdf ", zip_info::GZIP}, - {"bz2", "bzip2", " > ", " -cdf ", zip_info::BZIP2}, - {"zst", "zstd", " -q > ", " -cdf ", zip_info::ZSTD}, - {"xz", "xz", " > ", " -cdf ", zip_info::XZ}, - {"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", zip_info::LZMA}, - {"lz4", "lz4", " > ", " -cdf ", zip_info::LZ4}, +static const std::vector compress_styles = { + {"", "", "", "", compress_info::NONE}, + {"gz", "gzip", " > ", " -cdf ", compress_info::GZIP}, + {"bz2", "bzip2", " > ", " -cdf ", compress_info::BZIP2}, + {"zst", "zstd", " -q > ", " -cdf ", compress_info::ZSTD}, + {"xz", "xz", " > ", " -cdf ", compress_info::XZ}, + {"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", compress_info::LZMA}, + {"lz4", "lz4", " > ", " -cdf ", compress_info::LZ4}, }; // clang-format on /* ------------------------------------------------------------------ */ -static const zip_info &find_zip_type(const std::string &file) +static const compress_info &find_compress_type(const std::string &file) { std::size_t dot = file.find_last_of('.'); if (dot != std::string::npos) { const std::string ext = file.substr(dot + 1); - for (const auto &i : zip_styles) { + for (const auto &i : compress_styles) { if (i.extension == ext) return i; } } - return zip_styles[0]; + return compress_styles[0]; } /* ------------------------------------------------------------------ */ @@ -925,26 +925,26 @@ bool platform::file_is_readable(const std::string &path) check if filename has a known compression extension ------------------------------------------------------------------------- */ -bool platform::has_zip_extension(const std::string &file) +bool platform::has_compress_extension(const std::string &file) { - return find_zip_type(file).style != zip_info::NONE; + return find_compress_type(file).style != compress_info::NONE; } /* ---------------------------------------------------------------------- open pipe to read a compressed file ------------------------------------------------------------------------- */ -FILE *platform::zip_read(const std::string &file) +FILE *platform::compressed_read(const std::string &file) { FILE *fp = nullptr; #if defined(LAMMPS_GZIP) - auto zip = find_zip_type(file); - if (zip.style == zip_info::NONE) return nullptr; + auto compress = find_compress_type(file); + if (compress.style == compress_info::NONE) return nullptr; - if (find_exe_path(zip.command).size()) + if (find_exe_path(compress.command).size()) // put quotes around file name so that they may contain blanks - fp = popen((zip.command + zip.unzipflags + "\"" + file + "\""), "r"); + fp = popen((compress.command + compress.uncompressflags + "\"" + file + "\""), "r"); #endif return fp; } @@ -953,17 +953,17 @@ FILE *platform::zip_read(const std::string &file) open pipe to write a compressed file ------------------------------------------------------------------------- */ -FILE *platform::zip_write(const std::string &file) +FILE *platform::compressed_write(const std::string &file) { FILE *fp = nullptr; #if defined(LAMMPS_GZIP) - auto zip = find_zip_type(file); - if (zip.style == zip_info::NONE) return nullptr; + auto compress = find_compress_type(file); + if (compress.style == compress_info::NONE) return nullptr; - if (find_exe_path(zip.command).size()) + if (find_exe_path(compress.command).size()) // put quotes around file name so that they may contain blanks - fp = popen((zip.command + zip.zipflags + "\"" + file + "\""), "w"); + fp = popen((compress.command + compress.compressflags + "\"" + file + "\""), "w"); #endif return fp; } diff --git a/src/platform.h b/src/platform.h index 794fd692b0..de8ecce016 100644 --- a/src/platform.h +++ b/src/platform.h @@ -357,26 +357,26 @@ namespace platform { /*! Check if a file name ends in a known extension for a compressed file format * - * Currently supported file extensions are: .gz, .bz2, .zstd, .xz, .lzma, lz4 + * Currently supported file extensions are: .gz, .bz2, .zst, .xz, .lzma, lz4 * * \param file name of the file to check * \return true if the file has a known extension, otherwise false */ - bool has_zip_extension(const std::string &file); + bool has_compress_extension(const std::string &file); /*! Open pipe to compressed text file for reading * * \param file name of the file to open * \return FILE pointer to pipe using for reading the compressed file. */ - FILE *zip_read(const std::string &file); + FILE *compressed_read(const std::string &file); /*! Open pipe to compressed text file for writing * * \param file name of the file to open * \return FILE pointer to pipe using for reading the compressed file. */ - FILE *zip_write(const std::string &file); + FILE *compressed_write(const std::string &file); } // namespace platform } // namespace LAMMPS_NS diff --git a/src/read_data.cpp b/src/read_data.cpp index bbc3e20d44..c17ab97229 100644 --- a/src/read_data.cpp +++ b/src/read_data.cpp @@ -1951,14 +1951,14 @@ int ReadData::reallocate(int **pcount, int cmax, int amax) /* ---------------------------------------------------------------------- proc 0 opens data file - test if gzipped + test if compressed ------------------------------------------------------------------------- */ void ReadData::open(const std::string &file) { - if (platform::has_zip_extension(file)) { + if (platform::has_compress_extension(file)) { compressed = 1; - fp = platform::zip_read(file); + fp = platform::compressed_read(file); if (!fp) error->one(FLERR,"Cannot open compressed file {}", file); } else { compressed = 0; diff --git a/src/reader.cpp b/src/reader.cpp index 3a07417912..f2652d868d 100644 --- a/src/reader.cpp +++ b/src/reader.cpp @@ -36,9 +36,9 @@ void Reader::open_file(const std::string &file) { if (fp != nullptr) close_file(); - if (platform::has_zip_extension(file)) { + if (platform::has_compress_extension(file)) { compressed = 1; - fp = platform::zip_read(file); + fp = platform::compressed_read(file); if (!fp) error->one(FLERR,"Cannot open compressed file for reading"); } else { compressed = 0;