rename "zip" functions to "compress" functions. update related docs

This commit is contained in:
Axel Kohlmeyer
2021-10-07 12:38:11 -04:00
parent 4f17082d74
commit 6a9bb577cf
24 changed files with 83 additions and 79 deletions

View File

@ -354,8 +354,10 @@ Read or write compressed files
----------------------------------------- -----------------------------------------
If this option is enabled, large files can be read or written with If this option is enabled, large files can be read or written with
gzip compression by several LAMMPS commands, including compression by ``gzip`` or similar tools by several LAMMPS commands,
:doc:`read_data <read_data>`, :doc:`rerun <rerun>`, and :doc:`dump <dump>`. including :doc:`read_data <read_data>`, :doc:`rerun <rerun>`, and
:doc:`dump <dump>`. Currently supported compression tools are:
``gzip``, ``bzip2``, ``zstd``, and ``lzma``.
.. tabs:: .. tabs::
@ -364,8 +366,7 @@ gzip compression by several LAMMPS commands, including
.. code-block:: bash .. code-block:: bash
-D WITH_GZIP=value # yes or no -D WITH_GZIP=value # yes or no
# default is yes if CMake can find gzip, else no # default is yes if CMake can find the gzip program, else no
-D GZIP_EXECUTABLE=path # path to gzip executable if CMake cannot find it
.. tab:: Traditional make .. tab:: Traditional make
@ -373,14 +374,15 @@ gzip compression by several LAMMPS commands, including
LMP_INC = -DLAMMPS_GZIP LMP_INC = -DLAMMPS_GZIP
This option requires that your operating system fully supports the "popen()" This option requires that your operating system fully supports the
function in the standard runtime library and that a ``gzip`` executable can be "popen()" function in the standard runtime library and that a ``gzip``
found by LAMMPS during a run. or other executable can be found by LAMMPS in the standard search path
during a run.
.. note:: .. note::
On some clusters with high-speed networks, using the "fork()" library On clusters with high-speed networks, using the "fork()" library call
call (required by "popen()") can interfere with the fast communication (required by "popen()") can interfere with the fast communication
library and lead to simulations using compressed output or input to library and lead to simulations using compressed output or input to
hang or crash. For selected operations, compressed file I/O is also hang or crash. For selected operations, compressed file I/O is also
available using a compression library instead, which is what the available using a compression library instead, which is what the

View File

@ -139,11 +139,11 @@ Dynamically loaded object or library functions
Compressed file I/O functions Compressed file I/O functions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. doxygenfunction:: has_zip_extension .. doxygenfunction:: has_compress_extension
:project: progguide :project: progguide
.. doxygenfunction:: zip_read .. doxygenfunction:: compressed_read
:project: progguide :project: progguide
.. doxygenfunction:: zip_write .. doxygenfunction:: compressed_write
:project: progguide :project: progguide

View File

@ -1223,6 +1223,7 @@ Guo
gw gw
gyromagnetic gyromagnetic
gz gz
gzip
gzipped gzipped
Haak Haak
Hafskjold Hafskjold

View File

@ -33,7 +33,7 @@ DumpAtomGZ::~DumpAtomGZ() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -35,7 +35,7 @@ DumpCFGGZ::~DumpCFGGZ() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -33,7 +33,7 @@ DumpCustomGZ::~DumpCustomGZ() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -45,7 +45,7 @@ DumpCustomZstd::~DumpCustomZstd()
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -33,7 +33,7 @@ DumpLocalGZ::~DumpLocalGZ() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -39,7 +39,7 @@ DumpLocalZstd::~DumpLocalZstd() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -32,7 +32,7 @@ DumpXYZGZ::~DumpXYZGZ() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -38,7 +38,7 @@ DumpXYZZstd::~DumpXYZZstd() {}
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */

View File

@ -517,14 +517,14 @@ void FixTMD::readfile(char *file)
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
proc 0 opens TMD data file proc 0 opens TMD data file
test if gzipped test if compressed
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
void FixTMD::open(const std::string &file) void FixTMD::open(const std::string &file)
{ {
if (platform::has_zip_extension(file)) { if (platform::has_compress_extension(file)) {
compressed = 1; compressed = 1;
fp = platform::zip_read(file); fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file for reading"); if (!fp) error->one(FLERR,"Cannot open compressed file for reading");
} else { } else {
compressed = 0; compressed = 0;

View File

@ -211,7 +211,7 @@ void DynamicalMatrix::options(int narg, char **arg)
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a file generic opening of a file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
@ -223,8 +223,8 @@ void DynamicalMatrix::openfile(const char *filename)
if (me == 0) { if (me == 0) {
if (compressed) { if (compressed) {
fp = platform::zip_write(std::string(filename)+".gz"); fp = platform::compressed_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open gzipped file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else if (binaryflag) { } else if (binaryflag) {
fp = fopen(filename,"wb"); fp = fopen(filename,"wb");
} else { } else {

View File

@ -210,7 +210,7 @@ void ThirdOrder::options(int narg, char **arg)
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a file generic opening of a file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
@ -222,8 +222,8 @@ void ThirdOrder::openfile(const char* filename)
if (me == 0) { if (me == 0) {
if (compressed) { if (compressed) {
fp = platform::zip_write(std::string(filename)+".gz"); fp = platform::compressed_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open gzipped file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else if (binaryflag) { } else if (binaryflag) {
fp = fopen(filename,"wb"); fp = fopen(filename,"wb");
} else { } else {

View File

@ -52,9 +52,9 @@ FixReaxFFBonds::FixReaxFFBonds(LAMMPS *lmp, int narg, char **arg) :
if (nevery <= 0) error->all(FLERR,"Illegal fix reaxff/bonds command"); if (nevery <= 0) error->all(FLERR,"Illegal fix reaxff/bonds command");
if (me == 0) { if (me == 0) {
if (platform::has_zip_extension(arg[4])) { if (platform::has_compress_extension(arg[4])) {
compressed = 1; compressed = 1;
fp = platform::zip_write(arg[4]); fp = platform::compressed_write(arg[4]);
if (!fp) error->one(FLERR,"Cannot open compressed file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(arg[4],"w"); } else fp = fopen(arg[4],"w");

View File

@ -107,8 +107,8 @@ FixReaxFFSpecies::FixReaxFFSpecies(LAMMPS *lmp, int narg, char **arg) :
strcpy(tmparg[2],arg[5]); strcpy(tmparg[2],arg[5]);
if (me == 0) { if (me == 0) {
if (platform::has_zip_extension(arg[6])) { if (platform::has_compress_extension(arg[6])) {
fp = platform::zip_write(arg[6]); fp = platform::compressed_write(arg[6]);
compressed = 1; compressed = 1;
if (!fp) error->one(FLERR,"Cannot open compressed file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(arg[6],"w"); } else fp = fopen(arg[6],"w");

View File

@ -548,9 +548,9 @@ void NEB::readfile(char *file, int flag)
void NEB::open(char *file) void NEB::open(char *file)
{ {
compressed = 0; compressed = 0;
if (platform::has_zip_extension(file)) { if (platform::has_compress_extension(file)) {
compressed = 1; compressed = 1;
fp = platform::zip_read(file); fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(file,"r"); } else fp = fopen(file,"r");

View File

@ -687,14 +687,14 @@ int NEBSpin::initial_rotation(double *spi, double *sploc, double fraction)
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
universe proc 0 opens NEBSpin data file universe proc 0 opens NEBSpin data file
test if gzipped test if compressed
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
void NEBSpin::open(char *file) void NEBSpin::open(char *file)
{ {
compressed = 0; compressed = 0;
if (platform::has_zip_extension(file)) { if (platform::has_compress_extension(file)) {
fp = platform::zip_read(file); fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file"); if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(file,"r"); } else fp = fopen(file,"r");

View File

@ -143,7 +143,7 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : Pointers(lmp)
if (strchr(filename,'*')) multifile = 1; if (strchr(filename,'*')) multifile = 1;
if (utils::strmatch(filename, "\\.bin$")) binary = 1; if (utils::strmatch(filename, "\\.bin$")) binary = 1;
if (platform::has_zip_extension(filename)) compressed = 1; if (platform::has_compress_extension(filename)) compressed = 1;
} }
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
@ -528,7 +528,7 @@ void Dump::write()
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
generic opening of a dump file generic opening of a dump file
ASCII or binary or gzipped ASCII or binary or compressed
some derived classes override this function some derived classes override this function
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
@ -578,7 +578,7 @@ void Dump::openfile()
if (filewriter) { if (filewriter) {
if (compressed) { if (compressed) {
fp = platform::zip_write(filecurrent); fp = platform::compressed_write(filecurrent);
} else if (binary) { } else if (binary) {
fp = fopen(filecurrent,"wb"); fp = fopen(filecurrent,"wb");
} else if (append_flag) { } else if (append_flag) {

View File

@ -4420,15 +4420,16 @@ int lammps_config_has_mpi_support()
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
/** Check if the LAMMPS library supports compressed files via a pipe to gzip /** Check if the LAMMPS library supports reading or writing compressed
* files via a pipe to gzip or similar compression programs
\verbatim embed:rst \verbatim embed:rst
Several LAMMPS commands (e.g. :doc:`read_data`, :doc:`write_data`, Several LAMMPS commands (e.g. :doc:`read_data`, :doc:`write_data`,
:doc:`dump styles atom, custom, and xyz <dump>`) support reading and :doc:`dump styles atom, custom, and xyz <dump>`) support reading and
writing compressed files via creating a pipe to the ``gzip`` program. writing compressed files via creating a pipe to the ``gzip`` program.
This function checks whether this feature was :ref:`enabled at compile This function checks whether this feature was :ref:`enabled at compile
time <gzip>`. It does **not** check whether the ``gzip`` itself is time <gzip>`. It does **not** check whether``gzip`` or any other
installed and usable. supported compression programs themselves are installed and usable.
\endverbatim \endverbatim
* *
* \return 1 if yes, otherwise 0 * \return 1 if yes, otherwise 0

View File

@ -69,40 +69,40 @@
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
/// Struct for listing on-the-fly compression/decompression commands /// Struct for listing on-the-fly compression/decompression commands
struct zip_info { struct compress_info {
/// identifier for the different compression algorithms /// identifier for the different compression algorithms
enum styles { NONE, GZIP, BZIP2, ZSTD, XZ, LZMA, LZ4 }; enum styles { NONE, GZIP, BZIP2, ZSTD, XZ, LZMA, LZ4 };
const std::string extension; ///< filename extension for the current algorithm const std::string extension; ///< filename extension for the current algorithm
const std::string command; ///< command to perform compression or decompression const std::string command; ///< command to perform compression or decompression
const std::string zipflags; ///< flags to append to compress from stdin to stdout const std::string compressflags; ///< flags to append to compress from stdin to stdout
const std::string unzipflags; ///< flags to decompress file to stdout const std::string uncompressflags; ///< flags to decompress file to stdout
const int style; ///< compression style flag const int style; ///< compression style flag
}; };
// clang-format off // clang-format off
static const std::vector<zip_info> zip_styles = { static const std::vector<compress_info> compress_styles = {
{"", "", "", "", zip_info::NONE}, {"", "", "", "", compress_info::NONE},
{"gz", "gzip", " > ", " -cdf ", zip_info::GZIP}, {"gz", "gzip", " > ", " -cdf ", compress_info::GZIP},
{"bz2", "bzip2", " > ", " -cdf ", zip_info::BZIP2}, {"bz2", "bzip2", " > ", " -cdf ", compress_info::BZIP2},
{"zst", "zstd", " -q > ", " -cdf ", zip_info::ZSTD}, {"zst", "zstd", " -q > ", " -cdf ", compress_info::ZSTD},
{"xz", "xz", " > ", " -cdf ", zip_info::XZ}, {"xz", "xz", " > ", " -cdf ", compress_info::XZ},
{"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", zip_info::LZMA}, {"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", compress_info::LZMA},
{"lz4", "lz4", " > ", " -cdf ", zip_info::LZ4}, {"lz4", "lz4", " > ", " -cdf ", compress_info::LZ4},
}; };
// clang-format on // clang-format on
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
static const zip_info &find_zip_type(const std::string &file) static const compress_info &find_compress_type(const std::string &file)
{ {
std::size_t dot = file.find_last_of('.'); std::size_t dot = file.find_last_of('.');
if (dot != std::string::npos) { if (dot != std::string::npos) {
const std::string ext = file.substr(dot + 1); const std::string ext = file.substr(dot + 1);
for (const auto &i : zip_styles) { for (const auto &i : compress_styles) {
if (i.extension == ext) return i; if (i.extension == ext) return i;
} }
} }
return zip_styles[0]; return compress_styles[0];
} }
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
@ -925,26 +925,26 @@ bool platform::file_is_readable(const std::string &path)
check if filename has a known compression extension check if filename has a known compression extension
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
bool platform::has_zip_extension(const std::string &file) bool platform::has_compress_extension(const std::string &file)
{ {
return find_zip_type(file).style != zip_info::NONE; return find_compress_type(file).style != compress_info::NONE;
} }
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
open pipe to read a compressed file open pipe to read a compressed file
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
FILE *platform::zip_read(const std::string &file) FILE *platform::compressed_read(const std::string &file)
{ {
FILE *fp = nullptr; FILE *fp = nullptr;
#if defined(LAMMPS_GZIP) #if defined(LAMMPS_GZIP)
auto zip = find_zip_type(file); auto compress = find_compress_type(file);
if (zip.style == zip_info::NONE) return nullptr; if (compress.style == compress_info::NONE) return nullptr;
if (find_exe_path(zip.command).size()) if (find_exe_path(compress.command).size())
// put quotes around file name so that they may contain blanks // put quotes around file name so that they may contain blanks
fp = popen((zip.command + zip.unzipflags + "\"" + file + "\""), "r"); fp = popen((compress.command + compress.uncompressflags + "\"" + file + "\""), "r");
#endif #endif
return fp; return fp;
} }
@ -953,17 +953,17 @@ FILE *platform::zip_read(const std::string &file)
open pipe to write a compressed file open pipe to write a compressed file
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
FILE *platform::zip_write(const std::string &file) FILE *platform::compressed_write(const std::string &file)
{ {
FILE *fp = nullptr; FILE *fp = nullptr;
#if defined(LAMMPS_GZIP) #if defined(LAMMPS_GZIP)
auto zip = find_zip_type(file); auto compress = find_compress_type(file);
if (zip.style == zip_info::NONE) return nullptr; if (compress.style == compress_info::NONE) return nullptr;
if (find_exe_path(zip.command).size()) if (find_exe_path(compress.command).size())
// put quotes around file name so that they may contain blanks // put quotes around file name so that they may contain blanks
fp = popen((zip.command + zip.zipflags + "\"" + file + "\""), "w"); fp = popen((compress.command + compress.compressflags + "\"" + file + "\""), "w");
#endif #endif
return fp; return fp;
} }

View File

@ -357,26 +357,26 @@ namespace platform {
/*! Check if a file name ends in a known extension for a compressed file format /*! Check if a file name ends in a known extension for a compressed file format
* *
* Currently supported file extensions are: .gz, .bz2, .zstd, .xz, .lzma, lz4 * Currently supported file extensions are: .gz, .bz2, .zst, .xz, .lzma, lz4
* *
* \param file name of the file to check * \param file name of the file to check
* \return true if the file has a known extension, otherwise false */ * \return true if the file has a known extension, otherwise false */
bool has_zip_extension(const std::string &file); bool has_compress_extension(const std::string &file);
/*! Open pipe to compressed text file for reading /*! Open pipe to compressed text file for reading
* *
* \param file name of the file to open * \param file name of the file to open
* \return FILE pointer to pipe using for reading the compressed file. */ * \return FILE pointer to pipe using for reading the compressed file. */
FILE *zip_read(const std::string &file); FILE *compressed_read(const std::string &file);
/*! Open pipe to compressed text file for writing /*! Open pipe to compressed text file for writing
* *
* \param file name of the file to open * \param file name of the file to open
* \return FILE pointer to pipe using for reading the compressed file. */ * \return FILE pointer to pipe using for reading the compressed file. */
FILE *zip_write(const std::string &file); FILE *compressed_write(const std::string &file);
} // namespace platform } // namespace platform
} // namespace LAMMPS_NS } // namespace LAMMPS_NS

View File

@ -1951,14 +1951,14 @@ int ReadData::reallocate(int **pcount, int cmax, int amax)
/* ---------------------------------------------------------------------- /* ----------------------------------------------------------------------
proc 0 opens data file proc 0 opens data file
test if gzipped test if compressed
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
void ReadData::open(const std::string &file) void ReadData::open(const std::string &file)
{ {
if (platform::has_zip_extension(file)) { if (platform::has_compress_extension(file)) {
compressed = 1; compressed = 1;
fp = platform::zip_read(file); fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file {}", file); if (!fp) error->one(FLERR,"Cannot open compressed file {}", file);
} else { } else {
compressed = 0; compressed = 0;

View File

@ -36,9 +36,9 @@ void Reader::open_file(const std::string &file)
{ {
if (fp != nullptr) close_file(); if (fp != nullptr) close_file();
if (platform::has_zip_extension(file)) { if (platform::has_compress_extension(file)) {
compressed = 1; compressed = 1;
fp = platform::zip_read(file); fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file for reading"); if (!fp) error->one(FLERR,"Cannot open compressed file for reading");
} else { } else {
compressed = 0; compressed = 0;