ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename "zip" functions to "compress" functions. update related docs

Browse Source
This commit is contained in:
Axel Kohlmeyer
2021-10-07 12:38:11 -04:00
parent 4f17082d74
commit 6a9bb577cf
24 changed files with 83 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
doc/src/Build_settings.rst
View File

@ -354,8 +354,10 @@ Read or write compressed files
-----------------------------------------
If this option is enabled, large files can be read or written with
gzip compression by several LAMMPS commands, including
:doc:`read_data <read_data>`, :doc:`rerun <rerun>`, and :doc:`dump <dump>`.
compression by ``gzip`` or similar tools by several LAMMPS commands,
including :doc:`read_data <read_data>`, :doc:`rerun <rerun>`, and
:doc:`dump <dump>`. Currently supported compression tools are:
``gzip``, ``bzip2``, ``zstd``, and ``lzma``.
.. tabs::
@ -364,8 +366,7 @@ gzip compression by several LAMMPS commands, including
.. code-block:: bash
-D WITH_GZIP=value # yes or no
# default is yes if CMake can find gzip, else no
-D GZIP_EXECUTABLE=path # path to gzip executable if CMake cannot find it
# default is yes if CMake can find the gzip program, else no
.. tab:: Traditional make
@ -373,14 +374,15 @@ gzip compression by several LAMMPS commands, including
LMP_INC = -DLAMMPS_GZIP
This option requires that your operating system fully supports the "popen()"
function in the standard runtime library and that a ``gzip`` executable can be
found by LAMMPS during a run.
This option requires that your operating system fully supports the
"popen()" function in the standard runtime library and that a ``gzip``
or other executable can be found by LAMMPS in the standard search path
during a run.
.. note::
On some clusters with high-speed networks, using the "fork()" library
call (required by "popen()") can interfere with the fast communication
On clusters with high-speed networks, using the "fork()" library call
(required by "popen()") can interfere with the fast communication
library and lead to simulations using compressed output or input to
hang or crash. For selected operations, compressed file I/O is also
available using a compression library instead, which is what the

6
doc/src/Developer_platform.rst
View File

@ -139,11 +139,11 @@ Dynamically loaded object or library functions
Compressed file I/O functions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. doxygenfunction:: has_zip_extension
.. doxygenfunction:: has_compress_extension
:project: progguide
.. doxygenfunction:: zip_read
.. doxygenfunction:: compressed_read
:project: progguide
.. doxygenfunction:: zip_write
.. doxygenfunction:: compressed_write
:project: progguide

1
doc/utils/sphinx-config/false_positives.txt
View File

@ -1223,6 +1223,7 @@ Guo
gw
gyromagnetic
gz
gzip
gzipped
Haak
Hafskjold

2
src/COMPRESS/dump_atom_gz.cpp
View File

@ -33,7 +33,7 @@ DumpAtomGZ::~DumpAtomGZ() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_cfg_gz.cpp
View File

@ -35,7 +35,7 @@ DumpCFGGZ::~DumpCFGGZ() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_custom_gz.cpp
View File

@ -33,7 +33,7 @@ DumpCustomGZ::~DumpCustomGZ() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_custom_zstd.cpp
View File

@ -45,7 +45,7 @@ DumpCustomZstd::~DumpCustomZstd()
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_local_gz.cpp
View File

@ -33,7 +33,7 @@ DumpLocalGZ::~DumpLocalGZ() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_local_zstd.cpp
View File

@ -39,7 +39,7 @@ DumpLocalZstd::~DumpLocalZstd() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_xyz_gz.cpp
View File

@ -32,7 +32,7 @@ DumpXYZGZ::~DumpXYZGZ() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

2
src/COMPRESS/dump_xyz_zstd.cpp
View File

@ -38,7 +38,7 @@ DumpXYZZstd::~DumpXYZZstd() {}
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */

6
src/EXTRA-FIX/fix_tmd.cpp
View File

@ -517,14 +517,14 @@ void FixTMD::readfile(char *file)
/* ----------------------------------------------------------------------
proc 0 opens TMD data file
test if gzipped
test if compressed
------------------------------------------------------------------------- */
void FixTMD::open(const std::string &file)
{
if (platform::has_zip_extension(file)) {
if (platform::has_compress_extension(file)) {
compressed = 1;
fp = platform::zip_read(file);
fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file for reading");
} else {
compressed = 0;

6
src/PHONON/dynamical_matrix.cpp
View File

@ -211,7 +211,7 @@ void DynamicalMatrix::options(int narg, char **arg)
/* ----------------------------------------------------------------------
generic opening of a file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */
@ -223,8 +223,8 @@ void DynamicalMatrix::openfile(const char *filename)
if (me == 0) {
if (compressed) {
fp = platform::zip_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open gzipped file");
fp = platform::compressed_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else if (binaryflag) {
fp = fopen(filename,"wb");
} else {

6
src/PHONON/third_order.cpp
View File

@ -210,7 +210,7 @@ void ThirdOrder::options(int narg, char **arg)
/* ----------------------------------------------------------------------
generic opening of a file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */
@ -222,8 +222,8 @@ void ThirdOrder::openfile(const char* filename)
if (me == 0) {
if (compressed) {
fp = platform::zip_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open gzipped file");
fp = platform::compressed_write(std::string(filename)+".gz");
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else if (binaryflag) {
fp = fopen(filename,"wb");
} else {

4
src/REAXFF/fix_reaxff_bonds.cpp
View File

@ -52,9 +52,9 @@ FixReaxFFBonds::FixReaxFFBonds(LAMMPS *lmp, int narg, char **arg) :
if (nevery <= 0) error->all(FLERR,"Illegal fix reaxff/bonds command");
if (me == 0) {
if (platform::has_zip_extension(arg[4])) {
if (platform::has_compress_extension(arg[4])) {
compressed = 1;
fp = platform::zip_write(arg[4]);
fp = platform::compressed_write(arg[4]);
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(arg[4],"w");

4
src/REAXFF/fix_reaxff_species.cpp
View File

@ -107,8 +107,8 @@ FixReaxFFSpecies::FixReaxFFSpecies(LAMMPS *lmp, int narg, char **arg) :
strcpy(tmparg[2],arg[5]);
if (me == 0) {
if (platform::has_zip_extension(arg[6])) {
fp = platform::zip_write(arg[6]);
if (platform::has_compress_extension(arg[6])) {
fp = platform::compressed_write(arg[6]);
compressed = 1;
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(arg[6],"w");

4
src/REPLICA/neb.cpp
View File

@ -548,9 +548,9 @@ void NEB::readfile(char *file, int flag)
void NEB::open(char *file)
{
compressed = 0;
if (platform::has_zip_extension(file)) {
if (platform::has_compress_extension(file)) {
compressed = 1;
fp = platform::zip_read(file);
fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(file,"r");

6
src/SPIN/neb_spin.cpp
View File

@ -687,14 +687,14 @@ int NEBSpin::initial_rotation(double *spi, double *sploc, double fraction)
/* ----------------------------------------------------------------------
universe proc 0 opens NEBSpin data file
test if gzipped
test if compressed
------------------------------------------------------------------------- */
void NEBSpin::open(char *file)
{
compressed = 0;
if (platform::has_zip_extension(file)) {
fp = platform::zip_read(file);
if (platform::has_compress_extension(file)) {
fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file");
} else fp = fopen(file,"r");

6
src/dump.cpp
View File

@ -143,7 +143,7 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : Pointers(lmp)
if (strchr(filename,'*')) multifile = 1;
if (utils::strmatch(filename, "\\.bin$")) binary = 1;
if (platform::has_zip_extension(filename)) compressed = 1;
if (platform::has_compress_extension(filename)) compressed = 1;
}
/* ---------------------------------------------------------------------- */
@ -528,7 +528,7 @@ void Dump::write()
/* ----------------------------------------------------------------------
generic opening of a dump file
ASCII or binary or gzipped
ASCII or binary or compressed
some derived classes override this function
------------------------------------------------------------------------- */
@ -578,7 +578,7 @@ void Dump::openfile()
if (filewriter) {
if (compressed) {
fp = platform::zip_write(filecurrent);
fp = platform::compressed_write(filecurrent);
} else if (binary) {
fp = fopen(filecurrent,"wb");
} else if (append_flag) {

7
src/library.cpp
View File

@ -4420,15 +4420,16 @@ int lammps_config_has_mpi_support()
/* ---------------------------------------------------------------------- */
/** Check if the LAMMPS library supports compressed files via a pipe to gzip
/** Check if the LAMMPS library supports reading or writing compressed
* files via a pipe to gzip or similar compression programs
\verbatim embed:rst
Several LAMMPS commands (e.g. :doc:`read_data`, :doc:`write_data`,
:doc:`dump styles atom, custom, and xyz <dump>`) support reading and
writing compressed files via creating a pipe to the ``gzip`` program.
This function checks whether this feature was :ref:`enabled at compile
time <gzip>`. It does **not** check whether the ``gzip`` itself is
installed and usable.
time <gzip>`. It does **not** check whether``gzip`` or any other
supported compression programs themselves are installed and usable.
\endverbatim
*
* \return 1 if yes, otherwise 0

52
src/platform.cpp
View File

@ -69,40 +69,40 @@
/* ------------------------------------------------------------------ */
/// Struct for listing on-the-fly compression/decompression commands
struct zip_info {
struct compress_info {
/// identifier for the different compression algorithms
enum styles { NONE, GZIP, BZIP2, ZSTD, XZ, LZMA, LZ4 };
const std::string extension; ///< filename extension for the current algorithm
const std::string command; ///< command to perform compression or decompression
const std::string zipflags; ///< flags to append to compress from stdin to stdout
const std::string unzipflags; ///< flags to decompress file to stdout
const std::string compressflags; ///< flags to append to compress from stdin to stdout
const std::string uncompressflags; ///< flags to decompress file to stdout
const int style; ///< compression style flag
};
// clang-format off
static const std::vector<zip_info> zip_styles = {
{"", "", "", "", zip_info::NONE},
{"gz", "gzip", " > ", " -cdf ", zip_info::GZIP},
{"bz2", "bzip2", " > ", " -cdf ", zip_info::BZIP2},
{"zst", "zstd", " -q > ", " -cdf ", zip_info::ZSTD},
{"xz", "xz", " > ", " -cdf ", zip_info::XZ},
{"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", zip_info::LZMA},
{"lz4", "lz4", " > ", " -cdf ", zip_info::LZ4},
static const std::vector<compress_info> compress_styles = {
{"", "", "", "", compress_info::NONE},
{"gz", "gzip", " > ", " -cdf ", compress_info::GZIP},
{"bz2", "bzip2", " > ", " -cdf ", compress_info::BZIP2},
{"zst", "zstd", " -q > ", " -cdf ", compress_info::ZSTD},
{"xz", "xz", " > ", " -cdf ", compress_info::XZ},
{"lzma", "xz", " --format=lzma > ", " --format=lzma -cdf ", compress_info::LZMA},
{"lz4", "lz4", " > ", " -cdf ", compress_info::LZ4},
};
// clang-format on
/* ------------------------------------------------------------------ */
static const zip_info &find_zip_type(const std::string &file)
static const compress_info &find_compress_type(const std::string &file)
{
std::size_t dot = file.find_last_of('.');
if (dot != std::string::npos) {
const std::string ext = file.substr(dot + 1);
for (const auto &i : zip_styles) {
for (const auto &i : compress_styles) {
if (i.extension == ext) return i;
}
}
return zip_styles[0];
return compress_styles[0];
}
/* ------------------------------------------------------------------ */
@ -925,26 +925,26 @@ bool platform::file_is_readable(const std::string &path)
check if filename has a known compression extension
------------------------------------------------------------------------- */
bool platform::has_zip_extension(const std::string &file)
bool platform::has_compress_extension(const std::string &file)
{
return find_zip_type(file).style != zip_info::NONE;
return find_compress_type(file).style != compress_info::NONE;
}
/* ----------------------------------------------------------------------
open pipe to read a compressed file
------------------------------------------------------------------------- */
FILE *platform::zip_read(const std::string &file)
FILE *platform::compressed_read(const std::string &file)
{
FILE *fp = nullptr;
#if defined(LAMMPS_GZIP)
auto zip = find_zip_type(file);
if (zip.style == zip_info::NONE) return nullptr;
auto compress = find_compress_type(file);
if (compress.style == compress_info::NONE) return nullptr;
if (find_exe_path(zip.command).size())
if (find_exe_path(compress.command).size())
// put quotes around file name so that they may contain blanks
fp = popen((zip.command + zip.unzipflags + "\"" + file + "\""), "r");
fp = popen((compress.command + compress.uncompressflags + "\"" + file + "\""), "r");
#endif
return fp;
}
@ -953,17 +953,17 @@ FILE *platform::zip_read(const std::string &file)
open pipe to write a compressed file
------------------------------------------------------------------------- */
FILE *platform::zip_write(const std::string &file)
FILE *platform::compressed_write(const std::string &file)
{
FILE *fp = nullptr;
#if defined(LAMMPS_GZIP)
auto zip = find_zip_type(file);
if (zip.style == zip_info::NONE) return nullptr;
auto compress = find_compress_type(file);
if (compress.style == compress_info::NONE) return nullptr;
if (find_exe_path(zip.command).size())
if (find_exe_path(compress.command).size())
// put quotes around file name so that they may contain blanks
fp = popen((zip.command + zip.zipflags + "\"" + file + "\""), "w");
fp = popen((compress.command + compress.compressflags + "\"" + file + "\""), "w");
#endif
return fp;
}

8
src/platform.h
View File

@ -357,26 +357,26 @@ namespace platform {
/*! Check if a file name ends in a known extension for a compressed file format
*
* Currently supported file extensions are: .gz, .bz2, .zstd, .xz, .lzma, lz4
* Currently supported file extensions are: .gz, .bz2, .zst, .xz, .lzma, lz4
*
* \param file name of the file to check
* \return true if the file has a known extension, otherwise false */
bool has_zip_extension(const std::string &file);
bool has_compress_extension(const std::string &file);
/*! Open pipe to compressed text file for reading
*
* \param file name of the file to open
* \return FILE pointer to pipe using for reading the compressed file. */
FILE *zip_read(const std::string &file);
FILE *compressed_read(const std::string &file);
/*! Open pipe to compressed text file for writing
*
* \param file name of the file to open
* \return FILE pointer to pipe using for reading the compressed file. */
FILE *zip_write(const std::string &file);
FILE *compressed_write(const std::string &file);
} // namespace platform
} // namespace LAMMPS_NS

6
src/read_data.cpp
View File

@ -1951,14 +1951,14 @@ int ReadData::reallocate(int **pcount, int cmax, int amax)
/* ----------------------------------------------------------------------
proc 0 opens data file
test if gzipped
test if compressed
------------------------------------------------------------------------- */
void ReadData::open(const std::string &file)
{
if (platform::has_zip_extension(file)) {
if (platform::has_compress_extension(file)) {
compressed = 1;
fp = platform::zip_read(file);
fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file {}", file);
} else {
compressed = 0;

4
src/reader.cpp
View File

@ -36,9 +36,9 @@ void Reader::open_file(const std::string &file)
{
if (fp != nullptr) close_file();
if (platform::has_zip_extension(file)) {
if (platform::has_compress_extension(file)) {
compressed = 1;
fp = platform::zip_read(file);
fp = platform::compressed_read(file);
if (!fp) error->one(FLERR,"Cannot open compressed file for reading");
} else {
compressed = 0;