diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index ec28fdb06b..fac736c222 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -268,7 +268,6 @@ set(STANDARD_PACKAGES MOFFF MOLECULE MOLFILE - MPIIO NETCDF ORIENT PERI @@ -377,7 +376,6 @@ endif() # "hard" dependencies between packages resulting # in an error instead of skipping over files pkg_depends(ML-IAP ML-SNAP) -pkg_depends(MPIIO MPI) pkg_depends(ATC MANYBODY) pkg_depends(LATBOLTZ MPI) pkg_depends(SCAFACOS MPI) @@ -579,13 +577,6 @@ foreach(PKG ${STANDARD_PACKAGES}) RegisterPackages(${${PKG}_SOURCES_DIR}) endforeach() -# packages that need defines set -foreach(PKG MPIIO) - if(PKG_${PKG}) - target_compile_definitions(lammps PRIVATE -DLMP_${PKG}) - endif() -endforeach() - # dedicated check for entire contents of accelerator packages foreach(PKG ${SUFFIX_PACKAGES}) set(${PKG}_SOURCES_DIR ${LAMMPS_SOURCE_DIR}/${PKG}) diff --git a/cmake/presets/all_off.cmake b/cmake/presets/all_off.cmake index a920460b7b..e078879f70 100644 --- a/cmake/presets/all_off.cmake +++ b/cmake/presets/all_off.cmake @@ -63,7 +63,6 @@ set(ALL_PACKAGES MOFFF MOLECULE MOLFILE - MPIIO NETCDF OPENMP OPT diff --git a/cmake/presets/all_on.cmake b/cmake/presets/all_on.cmake index 99563bc82e..3f44a863f7 100644 --- a/cmake/presets/all_on.cmake +++ b/cmake/presets/all_on.cmake @@ -65,7 +65,6 @@ set(ALL_PACKAGES MOFFF MOLECULE MOLFILE - MPIIO NETCDF OPENMP OPT diff --git a/cmake/presets/mingw-cross.cmake b/cmake/presets/mingw-cross.cmake index 85f61be95b..f3565668b2 100644 --- a/cmake/presets/mingw-cross.cmake +++ b/cmake/presets/mingw-cross.cmake @@ -83,7 +83,6 @@ endforeach() # these two packages require a full MPI implementation if(BUILD_MPI) - set(PKG_MPIIO ON CACHE BOOL "" FORCE) set(PKG_LATBOLTZ ON CACHE BOOL "" FORCE) endif() diff --git a/cmake/presets/nolib.cmake b/cmake/presets/nolib.cmake index 5899f4bcb2..4a4a557505 100644 --- a/cmake/presets/nolib.cmake +++ b/cmake/presets/nolib.cmake @@ -19,7 +19,6 @@ set(PACKAGES_WITH_LIB ML-PACE ML-QUIP MOLFILE - MPIIO NETCDF PLUMED PYTHON diff --git a/src/.gitignore b/src/.gitignore index 115291c39a..8c1385199a 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -671,20 +671,14 @@ /dump_atom_gz.h /dump_atom_zstd.cpp /dump_atom_zstd.h -/dump_atom_mpiio.cpp -/dump_atom_mpiio.h /dump_cfg_gz.cpp /dump_cfg_gz.h -/dump_cfg_mpiio.cpp -/dump_cfg_mpiio.h /dump_cfg_zstd.cpp /dump_cfg_zstd.h /dump_custom_adios.cpp /dump_custom_adios.h /dump_custom_gz.cpp /dump_custom_gz.h -/dump_custom_mpiio.cpp -/dump_custom_mpiio.h /dump_custom_zstd.cpp /dump_custom_zstd.h /dump_h5md.cpp @@ -703,8 +697,6 @@ /dump_xtc.h /dump_xyz_gz.cpp /dump_xyz_gz.h -/dump_xyz_mpiio.cpp -/dump_xyz_mpiio.h /dump_xyz_zstd.cpp /dump_xyz_zstd.h /dump_yaml.cpp @@ -1455,8 +1447,6 @@ /remap.h /remap_wrap.cpp /remap_wrap.h -/restart_mpiio.cpp -/restart_mpiio.h /rigid_const.h /scafacos.cpp /scafacos.h diff --git a/src/MPIIO/Install.sh b/src/MPIIO/Install.sh deleted file mode 100755 index 03839c2cbe..0000000000 --- a/src/MPIIO/Install.sh +++ /dev/null @@ -1,58 +0,0 @@ -# Install/unInstall package files in LAMMPS -# mode = 0/1/2 for uninstall/install/update - -mode=$1 - -# enforce using portable C locale -LC_ALL=C -export LC_ALL - -# arg1 = file, arg2 = file it depends on - -action () { - if (test $mode = 0) then - rm -f ../$1 - elif (! cmp -s $1 ../$1) then - if (test -z "$2" || test -e ../$2) then - cp $1 .. - if (test $mode = 2) then - echo " updating src/$1" - fi - fi - elif (test -n "$2") then - if (test ! -e ../$2) then - rm -f ../$1 - fi - fi -} - -# force rebuild of files with LMP_MPIIO switch -# also read/write restart so their dependence on changed mpiio.h is rebuilt - -touch ../mpiio.h -touch ../read_restart.cpp -touch ../write_restart.cpp - -# all package files with no dependencies - -for file in *.cpp *.h; do - test -f ${file} && action $file -done - -# edit 2 Makefile.package to include/exclude LMP_MPIIO setting - -if (test $1 = 1) then - - if (test -e ../Makefile.package) then - sed -i -e 's/[^ \t]*MPIIO[^ \t]* //' ../Makefile.package - sed -i -e 's|^PKG_INC =[ \t]*|&-DLMP_MPIIO |' ../Makefile.package - - fi - -elif (test $1 = 0) then - - if (test -e ../Makefile.package) then - sed -i -e 's/[^ \t]*MPIIO[^ \t]* //' ../Makefile.package - fi - -fi diff --git a/src/MPIIO/dump_atom_mpiio.cpp b/src/MPIIO/dump_atom_mpiio.cpp deleted file mode 100644 index 844d1a46e4..0000000000 --- a/src/MPIIO/dump_atom_mpiio.cpp +++ /dev/null @@ -1,728 +0,0 @@ -/* ---------------------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -/* ---------------------------------------------------------------------- - Contributing author: Paul Coffman (IBM) -------------------------------------------------------------------------- */ - -#include "dump_atom_mpiio.h" - -#include "domain.h" -#include "error.h" -#include "memory.h" -#include "update.h" - -#include -#include - -#include "omp_compat.h" -#if defined(_OPENMP) -#include -#endif - -using namespace LAMMPS_NS; - -#define MAX_TEXT_HEADER_SIZE 4096 -#define DUMP_BUF_CHUNK_SIZE 16384 -#define DUMP_BUF_INCREMENT_SIZE 4096 - -/* ---------------------------------------------------------------------- */ - -DumpAtomMPIIO::DumpAtomMPIIO(LAMMPS *lmp, int narg, char **arg) : DumpAtom(lmp, narg, arg) -{ - if (me == 0) - error->warning(FLERR, "MPI-IO output is unmaintained and unreliable. Use with caution."); -} - -/* ---------------------------------------------------------------------- */ - -DumpAtomMPIIO::~DumpAtomMPIIO() -{ - if (multifile == 0) MPI_File_close(&mpifh); -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::openfile() -{ - if (singlefile_opened) { // single file already opened, so just return after resetting filesize - mpifo = currentFileSize; - MPI_File_set_size(mpifh, mpifo + headerSize + sumFileSize); - currentFileSize = mpifo + headerSize + sumFileSize; - return; - } - if (multifile == 0) singlefile_opened = 1; - - // if one file per timestep, replace '*' with current timestep - - filecurrent = filename; - - if (multifile) { - filecurrent = utils::strdup(utils::star_subst(filecurrent, update->ntimestep, padflag)); - if (maxfiles > 0) { - if (numfiles < maxfiles) { - nameslist[numfiles] = utils::strdup(filecurrent); - ++numfiles; - } else { - remove(nameslist[fileidx]); - delete[] nameslist[fileidx]; - nameslist[fileidx] = utils::strdup(filecurrent); - fileidx = (fileidx + 1) % maxfiles; - } - } - } - - if (append_flag) { // append open - int err = MPI_File_open(world, filecurrent, MPI_MODE_CREATE | MPI_MODE_APPEND | MPI_MODE_WRONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) - error->one(FLERR, "Cannot open dump file {}: {}", filecurrent, utils::getsyserror()); - - int myrank; - MPI_Comm_rank(world, &myrank); - if (myrank == 0) MPI_File_get_size(mpifh, &mpifo); - MPI_Bcast(&mpifo, 1, MPI_LMP_BIGINT, 0, world); - MPI_File_set_size(mpifh, mpifo + headerSize + sumFileSize); - currentFileSize = mpifo + headerSize + sumFileSize; - - } else { // replace open - - int err = - MPI_File_open(world, filecurrent, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) - error->one(FLERR, "Cannot open dump file {}: {}", filecurrent, utils::getsyserror()); - - mpifo = 0; - - MPI_File_set_size(mpifh, (MPI_Offset) (headerSize + sumFileSize)); - currentFileSize = (headerSize + sumFileSize); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::write() -{ - if (domain->triclinic == 0) { - boxxlo = domain->boxlo[0]; - boxxhi = domain->boxhi[0]; - boxylo = domain->boxlo[1]; - boxyhi = domain->boxhi[1]; - boxzlo = domain->boxlo[2]; - boxzhi = domain->boxhi[2]; - } else { - boxxlo = domain->boxlo_bound[0]; - boxxhi = domain->boxhi_bound[0]; - boxylo = domain->boxlo_bound[1]; - boxyhi = domain->boxhi_bound[1]; - boxzlo = domain->boxlo_bound[2]; - boxzhi = domain->boxhi_bound[2]; - boxxy = domain->xy; - boxxz = domain->xz; - boxyz = domain->yz; - } - - // nme = # of dump lines this proc contributes to dump - - nme = count(); - - // ntotal = total # of dump lines in snapshot - // nmax = max # of dump lines on any proc - - bigint bnme = nme; - MPI_Allreduce(&bnme, &ntotal, 1, MPI_LMP_BIGINT, MPI_SUM, world); - - int nmax; - MPI_Allreduce(&nme, &nmax, 1, MPI_INT, MPI_MAX, world); - - // write timestep header - // for multiproc, - // nheader = # of lines in this file via Allreduce on clustercomm - - bigint nheader = ntotal; - - // ensure filewriter proc can receive everyone's info - // limit nmax*size_one to int since used as arg in MPI_Rsend() below - // pack my data into buf - // if sorting on IDs also request ID list from pack() - // sort buf as needed - - if (nmax > maxbuf) { - if ((bigint) nmax * size_one > MAXSMALLINT) - error->all(FLERR, "Too much per-proc info for dump"); - maxbuf = nmax; - memory->destroy(buf); - memory->create(buf, (maxbuf * size_one), "dump:buf"); - } - if (sort_flag && sortcol == 0 && nmax > maxids) { - maxids = nmax; - memory->destroy(ids); - memory->create(ids, maxids, "dump:ids"); - } - - if (sort_flag && sortcol == 0) - pack(ids); - else - pack(nullptr); - if (sort_flag) sort(); - - // determine how much data needs to be written for setting the file size and prepocess it prior to writing - performEstimate = 1; - write_header(nheader); - write_data(nme, buf); - MPI_Bcast(&sumFileSize, 1, MPI_LMP_BIGINT, (nprocs - 1), world); - - openfile(); - - performEstimate = 0; - write_header(nheader); // mpifo now points to end of header info - - // now actually write the data - performEstimate = 0; - write_data(nme, buf); - - if (multifile) MPI_File_close(&mpifh); - if (multifile) delete[] filecurrent; -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::init_style() -{ - if (image_flag == 0) - size_one = 5; - else - size_one = 8; - - // format = copy of default or user-specified line format - // default depends on image flags - - delete[] format; - if (format_line_user) { - format = utils::strdup(std::string(format_line_user) + "\n"); - } else { - if (image_flag == 0) - format = utils::strdup(TAGINT_FORMAT " %d %g %g %g\n"); - else - format = utils::strdup(TAGINT_FORMAT " %d %g %g %g %d %d %d\n"); - } - - // setup boundary string - - domain->boundary_string(boundstr); - - // setup column string - - std::string default_columns; - - if (scale_flag == 0 && image_flag == 0) - default_columns = "id type x y z"; - else if (scale_flag == 0 && image_flag == 1) - default_columns = "id type x y z ix iy iz"; - else if (scale_flag == 1 && image_flag == 0) - default_columns = "id type xs ys zs"; - else if (scale_flag == 1 && image_flag == 1) - default_columns = "id type xs ys zs ix iy iz"; - - int icol = 0; - columns.clear(); - for (const auto &item : utils::split_words(default_columns)) { - if (columns.size()) columns += " "; - if (keyword_user[icol].size()) - columns += keyword_user[icol]; - else - columns += item; - ++icol; - } - - // setup function ptrs - - if (binary && domain->triclinic == 0) - header_choice = &DumpAtomMPIIO::header_binary; - else if (binary && domain->triclinic == 1) - header_choice = &DumpAtomMPIIO::header_binary_triclinic; - else if (!binary && domain->triclinic == 0) - header_choice = &DumpAtomMPIIO::header_item; - else if (!binary && domain->triclinic == 1) - header_choice = &DumpAtomMPIIO::header_item_triclinic; - - if (scale_flag == 1 && image_flag == 0 && domain->triclinic == 0) - pack_choice = &DumpAtomMPIIO::pack_scale_noimage; - else if (scale_flag == 1 && image_flag == 1 && domain->triclinic == 0) - pack_choice = &DumpAtomMPIIO::pack_scale_image; - else if (scale_flag == 1 && image_flag == 0 && domain->triclinic == 1) - pack_choice = &DumpAtomMPIIO::pack_scale_noimage_triclinic; - else if (scale_flag == 1 && image_flag == 1 && domain->triclinic == 1) - pack_choice = &DumpAtomMPIIO::pack_scale_image_triclinic; - else if (scale_flag == 0 && image_flag == 0) - pack_choice = &DumpAtomMPIIO::pack_noscale_noimage; - else if (scale_flag == 0 && image_flag == 1) - pack_choice = &DumpAtomMPIIO::pack_noscale_image; - - if (binary) - write_choice = &DumpAtomMPIIO::write_binary; - else if (buffer_flag == 1) - write_choice = &DumpAtomMPIIO::write_string; - else if (image_flag == 0) - write_choice = &DumpAtomMPIIO::write_lines_noimage; - else if (image_flag == 1) - write_choice = &DumpAtomMPIIO::write_lines_image; -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::write_header(bigint ndump) -{ - if (!header_choice) error->all(FLERR, "Must not use 'run pre no' after creating a new dump"); - - (this->*header_choice)(ndump); -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::header_binary(bigint ndump) -{ - if (performEstimate) { - - headerBuffer = (char *) malloc((2 * sizeof(bigint)) + (9 * sizeof(int)) + (6 * sizeof(double))); - - headerSize = 0; - memcpy(headerBuffer + headerSize, &update->ntimestep, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &ndump, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &domain->triclinic, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &domain->boundary[0][0], 6 * sizeof(int)); - headerSize += 6 * sizeof(int); - - memcpy(headerBuffer + headerSize, &boxxlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxylo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &size_one, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &nprocs, sizeof(int)); - headerSize += sizeof(int); - } else { // write data - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); - mpifo += headerSize; - free(headerBuffer); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::header_binary_triclinic(bigint ndump) -{ - if (performEstimate) { - - headerBuffer = (char *) malloc((2 * sizeof(bigint)) + (9 * sizeof(int)) + (9 * sizeof(double))); - - headerSize = 0; - memcpy(headerBuffer + headerSize, &update->ntimestep, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &ndump, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &domain->triclinic, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &domain->boundary[0][0], 6 * sizeof(int)); - headerSize += 6 * sizeof(int); - - memcpy(headerBuffer + headerSize, &boxxlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxylo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxy, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxz, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyz, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &size_one, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &nprocs, sizeof(int)); - headerSize += sizeof(int); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); - mpifo += headerSize; - free(headerBuffer); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::header_item(bigint ndump) -{ - if (performEstimate) { - - auto itemtxt = fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); - itemtxt += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); - itemtxt += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); - itemtxt += fmt::format("{} {}\n{} {}\n{} {}\n", boxxlo, boxxhi, boxylo, boxyhi, boxzlo, boxzhi); - itemtxt += fmt::format("ITEM: ATOMS {}\n", columns); - - headerSize = itemtxt.size(); - headerBuffer = utils::strdup(itemtxt); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_CHAR, MPI_STATUS_IGNORE); - mpifo += headerSize; - delete[] headerBuffer; - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::header_item_triclinic(bigint ndump) -{ - if (performEstimate) { - - auto itemtxt = fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); - itemtxt += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); - itemtxt += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); - itemtxt += fmt::format("{} {} {}\n{} {} {}\n{} {} {}\n", boxxlo, boxxhi, boxxy, boxylo, boxyhi, - boxxz, boxzlo, boxzhi, boxyz); - itemtxt += fmt::format("ITEM: ATOMS {}\n", columns); - - headerSize = itemtxt.size(); - headerBuffer = utils::strdup(itemtxt); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_CHAR, MPI_STATUS_IGNORE); - mpifo += headerSize; - delete[] headerBuffer; - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::write_data(int n, double *mybuf) -{ - (this->*write_choice)(n, mybuf); -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::write_binary(int n, double *mybuf) -{ - n *= size_one; - - if (performEstimate) { - - bigint incPrefix = 0; - bigint bigintNme = (bigint) nme; - MPI_Scan(&bigintNme, &incPrefix, 1, MPI_LMP_BIGINT, MPI_SUM, world); - sumFileSize = (incPrefix * size_one * sizeof(double)) + (nprocs * sizeof(int)); - offsetFromHeader = ((incPrefix - bigintNme) * size_one * sizeof(double)) + (me * sizeof(int)); - } else { - int byteBufSize = (n * sizeof(double)) + sizeof(int); - - char *bufWithSize; - memory->create(bufWithSize, byteBufSize, "dump:bufWithSize"); - memcpy(bufWithSize, (char *) (&n), sizeof(int)); - memcpy(&((char *) bufWithSize)[sizeof(int)], mybuf, (n * sizeof(double))); - MPI_File_write_at_all(mpifh, mpifo + offsetFromHeader, bufWithSize, byteBufSize, MPI_BYTE, - MPI_STATUS_IGNORE); - memory->destroy(bufWithSize); - - if (flush_flag) MPI_File_sync(mpifh); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpAtomMPIIO::write_string(int n, double *mybuf) -{ - if (performEstimate) { - -#if defined(_OPENMP) - int nthreads = omp_get_max_threads(); - if (nthreads > 1) - nsme = convert_string_omp(n, mybuf); - else { - nsme = convert_string(n, mybuf); - } -#else - nsme = convert_string(n, mybuf); -#endif - bigint incPrefix = 0; - bigint bigintNsme = (bigint) nsme; - MPI_Scan(&bigintNsme, &incPrefix, 1, MPI_LMP_BIGINT, MPI_SUM, world); - sumFileSize = (incPrefix * sizeof(char)); - offsetFromHeader = ((incPrefix - bigintNsme) * sizeof(char)); - } else { - MPI_File_write_at_all(mpifh, mpifo + offsetFromHeader, sbuf, nsme, MPI_CHAR, MPI_STATUS_IGNORE); - if (flush_flag) MPI_File_sync(mpifh); - } -} - -/* ---------------------------------------------------------------------- */ - -int DumpAtomMPIIO::convert_string(int n, double *mybuf) -{ - if (image_flag == 0) - return convert_noimage(n, mybuf); - else - return convert_image(n, mybuf); -} - -/* ---------------------------------------------------------------------- */ - -#if defined(_OPENMP) - -int DumpAtomMPIIO::convert_string_omp(int n, double *mybuf) -{ - if (image_flag == 0) - return convert_noimage_omp(n, mybuf); - else - return convert_image_omp(n, mybuf); -} - -/* ---------------------------------------------------------------------- - multithreaded version - convert mybuf of doubles to one big formatted string in sbuf - return -1 if strlen exceeds an int, since used as arg in MPI calls in Dump -------------------------------------------------------------------------- */ - -int DumpAtomMPIIO::convert_image_omp(int n, double *mybuf) -{ - char **mpifh_buffer_line_per_thread; - int mpifhStringCount; - int *mpifhStringCountPerThread, *bufOffset, *bufRange, *bufLength; - - mpifhStringCount = 0; - - int nthreads = omp_get_max_threads(); - if (nthreads > n) { // call serial version - convert_string(n, mybuf); - - } else { - memory->create(mpifhStringCountPerThread, nthreads, "dump:mpifhStringCountPerThread"); - mpifh_buffer_line_per_thread = (char **) malloc(nthreads * sizeof(char *)); - memory->create(bufOffset, nthreads, "dump:bufOffset"); - memory->create(bufRange, nthreads, "dump:bufRange"); - memory->create(bufLength, nthreads, "dump:bufLength"); - - int i = 0; - for (i = 0; i < (nthreads - 1); i++) { - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = (int) (floor((double) n / (double) nthreads)); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - } - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = n - (i * (int) (floor((double) n / (double) nthreads))); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(bufOffset, bufRange, bufLength, \ - mpifhStringCountPerThread, \ - mpifh_buffer_line_per_thread, mybuf) - { - int tid = omp_get_thread_num(); - int m = 0; - - for (int i = 0; i < bufRange[tid]; i++) { - - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc( - mpifh_buffer_line_per_thread[tid], - (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), format, - static_cast(mybuf[bufOffset[tid] + m]), - static_cast(mybuf[bufOffset[tid] + m + 1]), mybuf[bufOffset[tid] + m + 2], - mybuf[bufOffset[tid] + m + 3], mybuf[bufOffset[tid] + m + 4], - static_cast(mybuf[bufOffset[tid] + m + 5]), - static_cast(mybuf[bufOffset[tid] + m + 6]), - static_cast(mybuf[bufOffset[tid] + m + 7])); - m += size_one; - } - } - -#pragma omp barrier - mpifhStringCount = 0; - for (i = 0; i < nthreads; i++) { mpifhStringCount += mpifhStringCountPerThread[i]; } - - memory->destroy(bufOffset); - memory->destroy(bufRange); - memory->destroy(bufLength); - - if (mpifhStringCount > 0) { - if (mpifhStringCount > maxsbuf) { - if (mpifhStringCount > MAXSMALLINT) return -1; - maxsbuf = mpifhStringCount + 1; - memory->grow(sbuf, maxsbuf, "dump:sbuf"); - } - sbuf[0] = '\0'; - } - - for (int i = 0; i < nthreads; i++) { - strcat(sbuf, mpifh_buffer_line_per_thread[i]); - free(mpifh_buffer_line_per_thread[i]); - } - - memory->destroy(mpifhStringCountPerThread); - free(mpifh_buffer_line_per_thread); - } - return mpifhStringCount; -} - -/* ---------------------------------------------------------------------- - multithreaded version - convert mybuf of doubles to one big formatted string in sbuf - return -1 if strlen exceeds an int, since used as arg in MPI calls in Dump -------------------------------------------------------------------------- */ - -int DumpAtomMPIIO::convert_noimage_omp(int n, double *mybuf) -{ - char **mpifh_buffer_line_per_thread; - int mpifhStringCount; - int *mpifhStringCountPerThread, *bufOffset, *bufRange, *bufLength; - - mpifhStringCount = 0; - - int nthreads = omp_get_max_threads(); - if (nthreads > n) { // call serial version - convert_string(n, mybuf); - - } else { - memory->create(mpifhStringCountPerThread, nthreads, "dump:mpifhStringCountPerThread"); - mpifh_buffer_line_per_thread = (char **) malloc(nthreads * sizeof(char *)); - memory->create(bufOffset, nthreads, "dump:bufOffset"); - memory->create(bufRange, nthreads, "dump:bufRange"); - memory->create(bufLength, nthreads, "dump:bufLength"); - - int i = 0; - for (i = 0; i < (nthreads - 1); i++) { - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = (int) (floor((double) n / (double) nthreads)); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - } - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = n - (i * (int) (floor((double) n / (double) nthreads))); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(bufOffset, bufRange, bufLength, \ - mpifhStringCountPerThread, \ - mpifh_buffer_line_per_thread, mybuf) - { - int tid = omp_get_thread_num(); - int m = 0; - - for (int i = 0; i < bufRange[tid]; i++) { - - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc( - mpifh_buffer_line_per_thread[tid], - (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), format, - static_cast(mybuf[bufOffset[tid] + m]), - static_cast(mybuf[bufOffset[tid] + m + 1]), mybuf[bufOffset[tid] + m + 2], - mybuf[bufOffset[tid] + m + 3], mybuf[bufOffset[tid] + m + 4]); - m += size_one; - } - } - -#pragma omp barrier - mpifhStringCount = 0; - for (i = 0; i < nthreads; i++) { mpifhStringCount += mpifhStringCountPerThread[i]; } - - memory->destroy(bufOffset); - memory->destroy(bufRange); - memory->destroy(bufLength); - - if (mpifhStringCount > 0) { - if (mpifhStringCount > maxsbuf) { - if (mpifhStringCount > MAXSMALLINT) return -1; - maxsbuf = mpifhStringCount + 1; - memory->grow(sbuf, maxsbuf, "dump:sbuf"); - } - sbuf[0] = '\0'; - } - - for (int i = 0; i < nthreads; i++) { - strcat(sbuf, mpifh_buffer_line_per_thread[i]); - free(mpifh_buffer_line_per_thread[i]); - } - - memory->destroy(mpifhStringCountPerThread); - free(mpifh_buffer_line_per_thread); - } - - return mpifhStringCount; -} -#endif diff --git a/src/MPIIO/dump_atom_mpiio.h b/src/MPIIO/dump_atom_mpiio.h deleted file mode 100644 index efa2211168..0000000000 --- a/src/MPIIO/dump_atom_mpiio.h +++ /dev/null @@ -1,72 +0,0 @@ -/* -*- c++ -*- ---------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifdef DUMP_CLASS -// clang-format off -DumpStyle(atom/mpiio,DumpAtomMPIIO); -// clang-format on -#else - -#ifndef LMP_DUMP_ATOM_MPIIO_H -#define LMP_DUMP_ATOM_MPIIO_H - -#include "dump_atom.h" - -namespace LAMMPS_NS { - -class DumpAtomMPIIO : public DumpAtom { - - public: - DumpAtomMPIIO(class LAMMPS *, int, char **); - ~DumpAtomMPIIO() override; - - protected: - bigint - sumFileSize; // size in bytes of the file up through this rank offset from the end of the header data - char *headerBuffer; // buffer for holding header data - - MPI_File mpifh; - MPI_Offset mpifo, offsetFromHeader, headerSize, currentFileSize; - int performEstimate; // switch for write_data and write_header methods to use for gathering data and detemining filesize for preallocation vs actually writing the data - char *filecurrent; // name of file for this round (with % and * replaced) - - void openfile() override; - void write_header(bigint) override; - void write() override; - void write_data(int, double *) override; - - void init_style() override; - typedef void (DumpAtomMPIIO::*FnPtrHeader)(bigint); - FnPtrHeader header_choice; // ptr to write header functions - void header_binary(bigint); - void header_binary_triclinic(bigint); - void header_item(bigint); - void header_item_triclinic(bigint); - -#if defined(_OPENMP) - int convert_string_omp(int, double *); // multithreaded version of convert_string - int convert_image_omp(int, double *); // multithreaded version of convert_image - int convert_noimage_omp(int, double *); // multithreaded version of convert_noimage -#endif - - int convert_string(int, double *) override; - typedef void (DumpAtomMPIIO::*FnPtrData)(int, double *); - FnPtrData write_choice; // ptr to write data functions - void write_binary(int, double *); - void write_string(int, double *); -}; - -} // namespace LAMMPS_NS - -#endif -#endif diff --git a/src/MPIIO/dump_cfg_mpiio.cpp b/src/MPIIO/dump_cfg_mpiio.cpp deleted file mode 100644 index 04b962d4c9..0000000000 --- a/src/MPIIO/dump_cfg_mpiio.cpp +++ /dev/null @@ -1,488 +0,0 @@ -// clang-format off -/* ---------------------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -/* ---------------------------------------------------------------------- - Contributing author: Paul Coffman (IBM) -------------------------------------------------------------------------- */ - -#include "omp_compat.h" -#include "dump_cfg_mpiio.h" - -#include "atom.h" -#include "domain.h" -#include "update.h" -#include "memory.h" -#include "error.h" - -#include -#include - -#ifdef LMP_USER_IO_TIMER -#include -#include -long dumpCFGTimestamps[10]; -#endif - -#if defined(_OPENMP) -#include -#endif - -using namespace LAMMPS_NS; - -#define MAX_TEXT_HEADER_SIZE 4096 -#define DUMP_BUF_CHUNK_SIZE 16384 -#define DUMP_BUF_INCREMENT_SIZE 4096 - -#define UNWRAPEXPAND 10.0 -#define ONEFIELD 32 -#define DELTA 1048576 - -/* ---------------------------------------------------------------------- */ - -DumpCFGMPIIO::DumpCFGMPIIO(LAMMPS *lmp, int narg, char **arg) : - DumpCFG(lmp, narg, arg) -{ - if (me == 0) - error->warning(FLERR,"MPI-IO output is unmaintained and unreliable. Use with caution."); -} - -/* ---------------------------------------------------------------------- */ - -DumpCFGMPIIO::~DumpCFGMPIIO() -{ - if (multifile == 0) MPI_File_close(&mpifh); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::openfile() -{ - if (singlefile_opened) { // single file already opened, so just return after resetting filesize - mpifo = currentFileSize; - MPI_File_set_size(mpifh,mpifo+headerSize+sumFileSize); - currentFileSize = mpifo+headerSize+sumFileSize; - return; - } - if (multifile == 0) singlefile_opened = 1; - - // if one file per timestep, replace '*' with current timestep - - filecurrent = filename; - - if (multifile) { - filecurrent = utils::strdup(utils::star_subst(filecurrent, update->ntimestep, padflag)); - if (maxfiles > 0) { - if (numfiles < maxfiles) { - nameslist[numfiles] = utils::strdup(filecurrent); - ++numfiles; - } else { - remove(nameslist[fileidx]); - delete[] nameslist[fileidx]; - nameslist[fileidx] = utils::strdup(filecurrent); - fileidx = (fileidx + 1) % maxfiles; - } - } - } - - if (append_flag) { // append open - int err = MPI_File_open( world, filecurrent, MPI_MODE_CREATE | MPI_MODE_APPEND | - MPI_MODE_WRONLY, MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) error->one(FLERR, "Cannot open dump file {}", filecurrent); - - int myrank; - MPI_Comm_rank(world,&myrank); - if (myrank == 0) - MPI_File_get_size(mpifh,&mpifo); - MPI_Bcast(&mpifo, 1, MPI_LMP_BIGINT, 0, world); - MPI_File_set_size(mpifh,mpifo+headerSize+sumFileSize); - currentFileSize = mpifo+headerSize+sumFileSize; - - } else { // replace open - - int err = MPI_File_open( world, filecurrent, MPI_MODE_CREATE | MPI_MODE_WRONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) error->one(FLERR, "Cannot open dump file {}", filecurrent); - mpifo = 0; - - MPI_File_set_size(mpifh,(MPI_Offset) (headerSize+sumFileSize)); - currentFileSize = (headerSize+sumFileSize); - - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::write() -{ - -#ifdef LMP_USER_IO_TIMER - long startTimeBase, endTimeBase; - MPI_Barrier(world); // timestamp barrier - if (me == 0) - startTimeBase = GetTimeBase(); -#endif - - if (domain->triclinic == 0) { - boxxlo = domain->boxlo[0]; - boxxhi = domain->boxhi[0]; - boxylo = domain->boxlo[1]; - boxyhi = domain->boxhi[1]; - boxzlo = domain->boxlo[2]; - boxzhi = domain->boxhi[2]; - } else { - boxxlo = domain->boxlo_bound[0]; - boxxhi = domain->boxhi_bound[0]; - boxylo = domain->boxlo_bound[1]; - boxyhi = domain->boxhi_bound[1]; - boxzlo = domain->boxlo_bound[2]; - boxzhi = domain->boxhi_bound[2]; - boxxy = domain->xy; - boxxz = domain->xz; - boxyz = domain->yz; - } - - // nme = # of dump lines this proc contributes to dump - - nme = count(); - - // ntotal = total # of dump lines in snapshot - // nmax = max # of dump lines on any proc - - bigint bnme = nme; - MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); - - int nmax; - MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); - - // write timestep header - // for multiproc, - // nheader = # of lines in this file via Allreduce on clustercomm - - bigint nheader = ntotal; - - // ensure filewriter proc can receive everyone's info - // limit nmax*size_one to int since used as arg in MPI_Rsend() below - // pack my data into buf - // if sorting on IDs also request ID list from pack() - // sort buf as needed - - if (nmax > maxbuf) { - if ((bigint) nmax * size_one > MAXSMALLINT) - error->all(FLERR,"Too much per-proc info for dump"); - maxbuf = nmax; - memory->destroy(buf); - memory->create(buf,(maxbuf*size_one),"dump:buf"); - } - if (sort_flag && sortcol == 0 && nmax > maxids) { - maxids = nmax; - memory->destroy(ids); - memory->create(ids,maxids,"dump:ids"); - } - - if (sort_flag && sortcol == 0) pack(ids); - else pack(nullptr); - if (sort_flag) sort(); - - // determine how much data needs to be written for setting the file size and prepocess it prior to writing - performEstimate = 1; - write_header(nheader); - write_data(nme,buf); - MPI_Bcast(&sumFileSize, 1, MPI_LMP_BIGINT, (nprocs-1), world); - -#ifdef LMP_USER_IO_TIMER - MPI_Barrier(world); // timestamp barrier - dumpCFGTimestamps[0] = GetTimeBase(); -#endif - - openfile(); - -#ifdef LMP_USER_IO_TIMER - MPI_Barrier(world); // timestamp barrier - dumpCFGTimestamps[1] = GetTimeBase(); -#endif - - performEstimate = 0; - write_header(nheader); // mpifo now points to end of header info - -#ifdef LMP_USER_IO_TIMER - MPI_Barrier(world); // timestamp barrier - dumpCFGTimestamps[2] = GetTimeBase(); -#endif - - // now actually write the data - performEstimate = 0; - write_data(nme,buf); - -#ifdef LMP_USER_IO_TIMER - MPI_Barrier(world); // timestamp barrier - dumpCFGTimestamps[3] = GetTimeBase(); -#endif - - if (multifile) MPI_File_close(&mpifh); - if (multifile) delete [] filecurrent; - -#ifdef LMP_USER_IO_TIMER - MPI_Barrier(world); // timestamp barrier - dumpCFGTimestamps[4] = GetTimeBase(); - if (me == 0) { - endTimeBase = GetTimeBase(); - printf("total dump cycles: %ld - estimates and setup: %ld openfile: %ld write header: %ld write data: %ld close file: %ld\n",(long) (endTimeBase-startTimeBase),(long) (dumpCFGTimestamps[0]-startTimeBase),(long) (dumpCFGTimestamps[1]-dumpCFGTimestamps[0]),(long) (dumpCFGTimestamps[2]-dumpCFGTimestamps[1]),(long) (dumpCFGTimestamps[3]-dumpCFGTimestamps[2]),(long) (dumpCFGTimestamps[4]-dumpCFGTimestamps[3])); - } -#endif - -} - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::init_style() -{ - if (multifile == 0 && !multifile_override) - error->all(FLERR,"Dump cfg requires one snapshot per file"); - - DumpCFG::init_style(); - - // setup function ptrs - - write_choice = &DumpCFGMPIIO::write_string; - -} - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::write_header(bigint n) -{ - // set scale factor used by AtomEye for CFG viz - // default = 1.0 - // for peridynamics, set to pre-computed PD scale factor - // so PD particles mimic C atoms - // for unwrapped coords, set to UNWRAPEXPAND (10.0) - // so molecules are not split across periodic box boundaries - - double scale = 1.0; - if (atom->peri_flag) scale = atom->pdscale; - else if (unwrapflag == 1) scale = UNWRAPEXPAND; - - auto header = fmt::format("Number of particles = {}\n",n); - header += fmt::format("A = {} Angstrom (basic length-scale)\n",scale); - header += fmt::format("H0(1,1) = {} A\n",domain->xprd); - header += fmt::format("H0(1,2) = 0 A\n"); - header += fmt::format("H0(1,3) = 0 A\n"); - header += fmt::format("H0(2,1) = {} A\n",domain->xy); - header += fmt::format("H0(2,2) = {} A\n",domain->yprd); - header += fmt::format("H0(2,3) = 0 A\n"); - header += fmt::format("H0(3,1) = {} A\n",domain->xz); - header += fmt::format("H0(3,2) = {} A\n",domain->yz); - header += fmt::format("H0(3,3) = {} A\n",domain->zprd); - header += fmt::format(".NO_VELOCITY.\n"); - header += fmt::format("entry_count = {}\n",nfield-2); - for (int i = 0; i < nfield-5; i++) - header += fmt::format("auxiliary[{}] = {}\n",i,auxname[i]); - - if (performEstimate) { - headerSize = header.size(); - } else { // write data - if (me == 0) - MPI_File_write_at(mpifh,mpifo,(void *)header.c_str(),header.size(),MPI_CHAR,MPI_STATUS_IGNORE); - mpifo += header.size(); - } -} - -#if defined(_OPENMP) - -/* ---------------------------------------------------------------------- - convert mybuf of doubles to one big formatted string in sbuf - return -1 if strlen exceeds an int, since used as arg in MPI calls in Dump -------------------------------------------------------------------------- */ - -int DumpCFGMPIIO::convert_string_omp(int n, double *mybuf) -{ - - char **mpifh_buffer_line_per_thread; - int mpifhStringCount; - int *mpifhStringCountPerThread, *bufOffset, *bufRange, *bufLength; - - mpifhStringCount = 0; - - int nthreads = omp_get_max_threads(); - if (nthreads > n) { // call serial version - convert_string(n,mybuf); - - } - else { - memory->create(mpifhStringCountPerThread,nthreads,"dump:mpifhStringCountPerThread"); - mpifh_buffer_line_per_thread = (char **) malloc(nthreads*sizeof(char*)); - memory->create(bufOffset,nthreads,"dump:bufOffset"); - memory->create(bufRange,nthreads,"dump:bufRange"); - memory->create(bufLength,nthreads,"dump:bufLength"); - - int i=0; - for (i=0;i<(nthreads-1);i++) { - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i*(int)(floor((double)n/(double)nthreads))*size_one); - bufRange[i] = (int)(floor((double)n/(double)nthreads)); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - } - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i*(int)(floor((double)n/(double)nthreads))*size_one); - bufRange[i] = n-(i*(int)(floor((double)n/(double)nthreads))); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(bufOffset, bufRange, bufLength, mpifhStringCountPerThread, mpifh_buffer_line_per_thread, mybuf) - { - int tid = omp_get_thread_num(); - int m=0; - - if (unwrapflag == 0) { - for (int i = 0; i < bufRange[tid]; i++) { - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc(mpifh_buffer_line_per_thread[tid],(mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - for (int j = 0; j < size_one; j++) { - if (j == 0) { - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"%f \n",(mybuf[bufOffset[tid]+m])); - } else if (j == 1) { - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"%s \n",typenames[(int) mybuf[bufOffset[tid]+m]]); - } else if (j >= 2) { - if (vtype[j] == Dump::INT) - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],static_cast (mybuf[bufOffset[tid]+m])); - else if (vtype[j] == Dump::DOUBLE) - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],mybuf[bufOffset[tid]+m]); - else if (vtype[j] == Dump::STRING) - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],typenames[(int) mybuf[bufOffset[tid]+m]]); - else if (vtype[j] == Dump::BIGINT) - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],static_cast (mybuf[bufOffset[tid]+m])); - } - m++; - } // for j - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"\n"); - } // for i - } // wrap flag - else if (unwrapflag == 1) { - for (int i = 0; i < bufRange[tid]; i++) { - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc(mpifh_buffer_line_per_thread[tid],(mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - for (int j = 0; j < size_one; j++) { - double unwrap_coord; - if (j == 0) { - //offset += sprintf(&sbuf[offset],"%f \n",mybuf[m]); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"%f \n",mybuf[bufOffset[tid]+m]); - } else if (j == 1) { - // offset += sprintf(&sbuf[offset],"%s \n",typenames[(int) mybuf[m]]); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"%s \n",typenames[(int) mybuf[bufOffset[tid]+m]]); - } else if (j >= 2 && j <= 4) { - unwrap_coord = (mybuf[bufOffset[tid]+m] - 0.5)/UNWRAPEXPAND + 0.5; - //offset += sprintf(&sbuf[offset],vformat[j],unwrap_coord); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],unwrap_coord); - } else if (j >= 5) { - if (vtype[j] == Dump::INT) - //offset += - // sprintf(&sbuf[offset],vformat[j],static_cast (mybuf[m])); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],static_cast (mybuf[bufOffset[tid]+m])); - else if (vtype[j] == Dump::DOUBLE) - // offset += sprintf(&sbuf[offset],vformat[j],mybuf[m]); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],mybuf[bufOffset[tid]+m]); - else if (vtype[j] == Dump::STRING) - // offset += - // sprintf(&sbuf[offset],vformat[j],typenames[(int) mybuf[m]]); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],typenames[(int) mybuf[bufOffset[tid]+m]]); - else if (vtype[j] == Dump::BIGINT) - // offset += - // sprintf(&sbuf[offset],vformat[j],static_cast (mybuf[m])); - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),vformat[j],static_cast (mybuf[bufOffset[tid]+m])); - } - m++; - } // for j - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),"\n"); - } // for i - } // unwrap flag - } // pragma omp parallel - -#pragma omp barrier - mpifhStringCount = 0; - for (i=0;idestroy(bufOffset); - memory->destroy(bufRange); - memory->destroy(bufLength); - - if (mpifhStringCount > 0) { - if (mpifhStringCount > maxsbuf) { - if (mpifhStringCount > MAXSMALLINT) return -1; - maxsbuf = mpifhStringCount+1; - memory->grow(sbuf,maxsbuf,"dump:sbuf"); - } - sbuf[0] = '\0'; - } - - for (int i=0;idestroy(mpifhStringCountPerThread); - free(mpifh_buffer_line_per_thread); - - } // else omp - - return mpifhStringCount; - -} - -#endif - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::write_data(int n, double *mybuf) -{ - (this->*write_choice)(n,mybuf); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCFGMPIIO::write_string(int n, double *mybuf) -{ - if (performEstimate) { - -#if defined(_OPENMP) - int nthreads = omp_get_max_threads(); - if (nthreads > 1) - nsme = convert_string_omp(n,mybuf); - else - nsme = convert_string(n,mybuf); -#else - - nsme = convert_string(n,mybuf); -#endif - bigint incPrefix = 0; - bigint bigintNsme = (bigint) nsme; - MPI_Scan(&bigintNsme,&incPrefix,1,MPI_LMP_BIGINT,MPI_SUM,world); - sumFileSize = (incPrefix*sizeof(char)); - offsetFromHeader = ((incPrefix-bigintNsme)*sizeof(char)); - } - else { - MPI_File_write_at_all(mpifh,mpifo+offsetFromHeader,sbuf,nsme,MPI_CHAR,MPI_STATUS_IGNORE); - if (flush_flag) - MPI_File_sync(mpifh); - } -} - diff --git a/src/MPIIO/dump_cfg_mpiio.h b/src/MPIIO/dump_cfg_mpiio.h deleted file mode 100644 index 7c52d46c4e..0000000000 --- a/src/MPIIO/dump_cfg_mpiio.h +++ /dev/null @@ -1,60 +0,0 @@ -/* -*- c++ -*- ---------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifdef DUMP_CLASS -// clang-format off -DumpStyle(cfg/mpiio,DumpCFGMPIIO); -// clang-format on -#else - -#ifndef LMP_DUMP_CFG_MPIIO_H -#define LMP_DUMP_CFG_MPIIO_H - -#include "dump_cfg.h" - -namespace LAMMPS_NS { - -class DumpCFGMPIIO : public DumpCFG { - public: - DumpCFGMPIIO(class LAMMPS *, int, char **); - ~DumpCFGMPIIO() override; - - protected: - bigint - sumFileSize; // size in bytes of the file up through this rank offset from the end of the header data - char *headerBuffer; // buffer for holding header data - - MPI_File mpifh; - MPI_Offset mpifo, offsetFromHeader, headerSize, currentFileSize; - int performEstimate; // switch for write_data and write_header methods to use for gathering data and detemining filesize for preallocation vs actually writing the data - char *filecurrent; // name of file for this round (with % and * replaced) - -#if defined(_OPENMP) - int convert_string_omp(int, double *); // multithreaded version of convert_string -#endif - - void openfile() override; - void init_style() override; - void write_header(bigint) override; - void write() override; - void write_data(int, double *) override; - - typedef void (DumpCFGMPIIO::*FnPtrData)(int, double *); - FnPtrData write_choice; // ptr to write data functions - void write_string(int, double *); -}; - -} // namespace LAMMPS_NS - -#endif -#endif diff --git a/src/MPIIO/dump_custom_mpiio.cpp b/src/MPIIO/dump_custom_mpiio.cpp deleted file mode 100644 index 80adefa325..0000000000 --- a/src/MPIIO/dump_custom_mpiio.cpp +++ /dev/null @@ -1,648 +0,0 @@ -/* ---------------------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -/* ---------------------------------------------------------------------- - Contributing author: Paul Coffman (IBM) -------------------------------------------------------------------------- */ - -#include "dump_custom_mpiio.h" - -#include "domain.h" -#include "error.h" -#include "fix.h" -#include "input.h" -#include "memory.h" -#include "modify.h" -#include "update.h" -#include "variable.h" - -#include -#include - -#include "omp_compat.h" -#if defined(_OPENMP) -#include -#endif - -using namespace LAMMPS_NS; - -#define DUMP_BUF_CHUNK_SIZE 16384 -#define DUMP_BUF_INCREMENT_SIZE 4096 - -/* ---------------------------------------------------------------------- */ - -DumpCustomMPIIO::DumpCustomMPIIO(LAMMPS *lmp, int narg, char **arg) : DumpCustom(lmp, narg, arg) -{ - if (me == 0) - error->warning(FLERR, "MPI-IO output is unmaintained and unreliable. Use with caution."); -} - -/* ---------------------------------------------------------------------- */ - -DumpCustomMPIIO::~DumpCustomMPIIO() -{ - if (multifile == 0) MPI_File_close(&mpifh); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::openfile() -{ - if (singlefile_opened) { // single file already opened, so just return after resetting filesize - mpifo = currentFileSize; - MPI_File_set_size(mpifh, mpifo + headerSize + sumFileSize); - currentFileSize = mpifo + headerSize + sumFileSize; - return; - } - if (multifile == 0) singlefile_opened = 1; - - // if one file per timestep, replace '*' with current timestep - - filecurrent = filename; - - if (multifile) { - filecurrent = utils::strdup(utils::star_subst(filecurrent, update->ntimestep, padflag)); - if (maxfiles > 0) { - if (numfiles < maxfiles) { - nameslist[numfiles] = utils::strdup(filecurrent); - ++numfiles; - } else { - remove(nameslist[fileidx]); - delete[] nameslist[fileidx]; - nameslist[fileidx] = utils::strdup(filecurrent); - fileidx = (fileidx + 1) % maxfiles; - } - } - } - - if (append_flag) { // append open - int err = MPI_File_open(world, filecurrent, MPI_MODE_CREATE | MPI_MODE_APPEND | MPI_MODE_WRONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) - error->one(FLERR, "Cannot open dump file {}: {}", filecurrent, utils::getsyserror()); - - int myrank; - MPI_Comm_rank(world, &myrank); - if (myrank == 0) MPI_File_get_size(mpifh, &mpifo); - MPI_Bcast(&mpifo, 1, MPI_LMP_BIGINT, 0, world); - MPI_File_set_size(mpifh, mpifo + headerSize + sumFileSize); - currentFileSize = mpifo + headerSize + sumFileSize; - - } else { // replace open - - int err = - MPI_File_open(world, filecurrent, MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) - error->one(FLERR, "Cannot open dump file {}: {}", filecurrent, utils::getsyserror()); - - mpifo = 0; - - MPI_File_set_size(mpifh, (MPI_Offset) (headerSize + sumFileSize)); - currentFileSize = (headerSize + sumFileSize); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::write() -{ - if (domain->triclinic == 0) { - boxxlo = domain->boxlo[0]; - boxxhi = domain->boxhi[0]; - boxylo = domain->boxlo[1]; - boxyhi = domain->boxhi[1]; - boxzlo = domain->boxlo[2]; - boxzhi = domain->boxhi[2]; - } else { - boxxlo = domain->boxlo_bound[0]; - boxxhi = domain->boxhi_bound[0]; - boxylo = domain->boxlo_bound[1]; - boxyhi = domain->boxhi_bound[1]; - boxzlo = domain->boxlo_bound[2]; - boxzhi = domain->boxhi_bound[2]; - boxxy = domain->xy; - boxxz = domain->xz; - boxyz = domain->yz; - } - - // nme = # of dump lines this proc contributes to dump - - nme = count(); - - // ntotal = total # of dump lines in snapshot - // nmax = max # of dump lines on any proc - - bigint bnme = nme; - MPI_Allreduce(&bnme, &ntotal, 1, MPI_LMP_BIGINT, MPI_SUM, world); - - int nmax; - MPI_Allreduce(&nme, &nmax, 1, MPI_INT, MPI_MAX, world); - - // write timestep header - // for multiproc, - // nheader = # of lines in this file via Allreduce on clustercomm - - bigint nheader = ntotal; - - // ensure filewriter proc can receive everyone's info - // limit nmax*size_one to int since used as arg in MPI_Rsend() below - // pack my data into buf - // if sorting on IDs also request ID list from pack() - // sort buf as needed - - if (nmax > maxbuf) { - if ((bigint) nmax * size_one > MAXSMALLINT) - error->all(FLERR, "Too much per-proc info for dump"); - maxbuf = nmax; - memory->destroy(buf); - memory->create(buf, (maxbuf * size_one), "dump:buf"); - } - if (sort_flag && sortcol == 0 && nmax > maxids) { - maxids = nmax; - memory->destroy(ids); - memory->create(ids, maxids, "dump:ids"); - } - - if (sort_flag && sortcol == 0) - pack(ids); - else - pack(nullptr); - if (sort_flag) sort(); - - // determine how much data needs to be written for setting the file size and prepocess it prior to writing - performEstimate = 1; - write_header(nheader); - write_data(nme, buf); - MPI_Bcast(&sumFileSize, 1, MPI_LMP_BIGINT, (nprocs - 1), world); - - openfile(); - - performEstimate = 0; - write_header(nheader); // mpifo now points to end of header info - - // now actually write the data - performEstimate = 0; - write_data(nme, buf); - - if (multifile) MPI_File_close(&mpifh); - if (multifile) delete[] filecurrent; -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::init_style() -{ - // assemble ITEMS: column string from defaults and user values - - delete[] columns; - std::string combined; - int icol = 0; - for (const auto &item : utils::split_words(columns_default)) { - if (combined.size()) combined += " "; - if (keyword_user[icol].size()) - combined += keyword_user[icol]; - else - combined += item; - ++icol; - } - columns = utils::strdup(combined); - - // format = copy of default or user-specified line format - - delete[] format; - if (format_line_user) - format = utils::strdup(format_line_user); - else - format = utils::strdup(format_default); - - // tokenize the format string and add space at end of each format element - // if user-specified int/float format exists, use it instead - // if user-specified column format exists, use it instead - // lo priority = line, medium priority = int/float, hi priority = column - - auto words = utils::split_words(format); - if ((int) words.size() < nfield) - error->all(FLERR, "Dump_modify format line is too short: {}", format); - - int i = 0; - for (const auto &word : words) { - if (i >= nfield) break; - delete[] vformat[i]; - - if (format_column_user[i]) - vformat[i] = utils::strdup(std::string(format_column_user[i]) + " "); - else if (vtype[i] == Dump::INT && format_int_user) - vformat[i] = utils::strdup(std::string(format_int_user) + " "); - else if (vtype[i] == Dump::DOUBLE && format_float_user) - vformat[i] = utils::strdup(std::string(format_float_user) + " "); - else if (vtype[i] == Dump::BIGINT && format_bigint_user) - vformat[i] = utils::strdup(std::string(format_bigint_user) + " "); - else - vformat[i] = utils::strdup(word + " "); - - // remove trailing blank on last column's format - if (i == nfield - 1) vformat[i][strlen(vformat[i]) - 1] = '\0'; - - ++i; - } - - // setup boundary string - - domain->boundary_string(boundstr); - - // setup function ptrs - - if (binary && domain->triclinic == 0) - header_choice = &DumpCustomMPIIO::header_binary; - else if (binary && domain->triclinic == 1) - header_choice = &DumpCustomMPIIO::header_binary_triclinic; - else if (!binary && domain->triclinic == 0) - header_choice = &DumpCustomMPIIO::header_item; - else if (!binary && domain->triclinic == 1) - header_choice = &DumpCustomMPIIO::header_item_triclinic; - - if (binary) - write_choice = &DumpCustomMPIIO::write_binary; - else - write_choice = &DumpCustomMPIIO::write_string; - - // find current ptr for each compute,fix,variable - // check that fix frequency is acceptable - - for (i = 0; i < ncompute; i++) { - compute[i] = modify->get_compute_by_id(id_compute[i]); - if (!compute[i]) - error->all(FLERR, "Could not find dump custom/mpiio compute ID {}", id_compute[i]); - } - - for (i = 0; i < nfix; i++) { - fix[i] = modify->get_fix_by_id(id_fix[i]); - if (!fix[i]) error->all(FLERR, "Could not find dump custom/mpiio fix ID {}", id_fix[i]); - if (nevery % fix[i]->peratom_freq) - error->all(FLERR, "dump custom/mpiio and fix not computed at compatible times"); - } - - for (i = 0; i < nvariable; i++) { - int ivariable = input->variable->find(id_variable[i]); - if (ivariable < 0) - error->all(FLERR, "Could not find dump custom/mpiio variable name {}", id_variable[i]); - variable[i] = ivariable; - } - - // set index and check validity of region - - if (idregion && !domain->get_region_by_id(idregion)) - error->all(FLERR, "Region {} for dump custom/mpiio does not exist", idregion); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::write_header(bigint ndump) -{ - if (!header_choice) error->all(FLERR, "Must not use 'run pre no' after creating a new dump"); - - (this->*header_choice)(ndump); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::header_binary(bigint ndump) -{ - if (performEstimate) { - - headerBuffer = (char *) malloc((2 * sizeof(bigint)) + (9 * sizeof(int)) + (6 * sizeof(double))); - - headerSize = 0; - memcpy(headerBuffer + headerSize, &update->ntimestep, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &ndump, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &domain->triclinic, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &domain->boundary[0][0], 6 * sizeof(int)); - headerSize += 6 * sizeof(int); - - memcpy(headerBuffer + headerSize, &boxxlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxylo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &size_one, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &nprocs, sizeof(int)); - headerSize += sizeof(int); - } else { // write data - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); - mpifo += headerSize; - free(headerBuffer); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::header_binary_triclinic(bigint ndump) -{ - if (performEstimate) { - - headerBuffer = (char *) malloc((2 * sizeof(bigint)) + (9 * sizeof(int)) + (9 * sizeof(double))); - - headerSize = 0; - memcpy(headerBuffer + headerSize, &update->ntimestep, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &ndump, sizeof(bigint)); - headerSize += sizeof(bigint); - - memcpy(headerBuffer + headerSize, &domain->triclinic, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &domain->boundary[0][0], 6 * sizeof(int)); - headerSize += 6 * sizeof(int); - - memcpy(headerBuffer + headerSize, &boxxlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxylo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzlo, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxzhi, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxy, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxxz, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &boxyz, sizeof(double)); - headerSize += sizeof(double); - - memcpy(headerBuffer + headerSize, &size_one, sizeof(int)); - headerSize += sizeof(int); - - memcpy(headerBuffer + headerSize, &nprocs, sizeof(int)); - headerSize += sizeof(int); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_BYTE, MPI_STATUS_IGNORE); - mpifo += headerSize; - free(headerBuffer); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::header_item(bigint ndump) -{ - if (performEstimate) { - - auto itemtxt = fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); - itemtxt += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); - itemtxt += fmt::format("ITEM: BOX BOUNDS {}\n", boundstr); - itemtxt += fmt::format("{} {}\n{} {}\n{} {}\n", boxxlo, boxxhi, boxylo, boxyhi, boxzlo, boxzhi); - itemtxt += fmt::format("ITEM: ATOMS {}\n", columns); - - headerSize = itemtxt.size(); - headerBuffer = utils::strdup(itemtxt); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_CHAR, MPI_STATUS_IGNORE); - mpifo += headerSize; - delete[] headerBuffer; - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::header_item_triclinic(bigint ndump) -{ - if (performEstimate) { - - auto itemtxt = fmt::format("ITEM: TIMESTEP\n{}\n", update->ntimestep); - itemtxt += fmt::format("ITEM: NUMBER OF ATOMS\n{}\n", ndump); - itemtxt += fmt::format("ITEM: BOX BOUNDS xy xz yz {}\n", boundstr); - itemtxt += fmt::format("{} {} {}\n{} {} {}\n{} {} {}\n", boxxlo, boxxhi, boxxy, boxylo, boxyhi, - boxxz, boxzlo, boxzhi, boxyz); - itemtxt += fmt::format("ITEM: ATOMS {}\n", columns); - - headerSize = itemtxt.size(); - headerBuffer = utils::strdup(itemtxt); - - } else { // write data - - if (me == 0) - MPI_File_write_at(mpifh, mpifo, headerBuffer, headerSize, MPI_CHAR, MPI_STATUS_IGNORE); - mpifo += headerSize; - delete[] headerBuffer; - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::write_data(int n, double *mybuf) -{ - (this->*write_choice)(n, mybuf); -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::write_binary(int n, double *mybuf) -{ - n *= size_one; - - if (performEstimate) { - - bigint incPrefix = 0; - bigint bigintNme = (bigint) nme; - MPI_Scan(&bigintNme, &incPrefix, 1, MPI_LMP_BIGINT, MPI_SUM, world); - sumFileSize = (incPrefix * size_one * sizeof(double)) + (nprocs * sizeof(int)); - offsetFromHeader = ((incPrefix - bigintNme) * size_one * sizeof(double)) + (me * sizeof(int)); - } else { - int byteBufSize = (n * sizeof(double)) + sizeof(int); - - char *bufWithSize; - memory->create(bufWithSize, byteBufSize, "dump:bufWithSize"); - memcpy(bufWithSize, (char *) (&n), sizeof(int)); - memcpy(&((char *) bufWithSize)[sizeof(int)], mybuf, (n * sizeof(double))); - MPI_File_write_at_all(mpifh, mpifo + offsetFromHeader, bufWithSize, byteBufSize, MPI_BYTE, - MPI_STATUS_IGNORE); - memory->destroy(bufWithSize); - - if (flush_flag) MPI_File_sync(mpifh); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpCustomMPIIO::write_string(int n, double *mybuf) -{ - if (performEstimate) { - -#if defined(_OPENMP) - int nthreads = omp_get_max_threads(); - if ((nthreads > 1) && !(lmp->kokkos)) - nsme = convert_string_omp(n, mybuf); // not (yet) compatible with Kokkos - else - nsme = convert_string(n, mybuf); -#else - nsme = convert_string(n, mybuf); -#endif - bigint incPrefix = 0; - bigint bigintNsme = (bigint) nsme; - MPI_Scan(&bigintNsme, &incPrefix, 1, MPI_LMP_BIGINT, MPI_SUM, world); - sumFileSize = (incPrefix * sizeof(char)); - offsetFromHeader = ((incPrefix - bigintNsme) * sizeof(char)); - } else { - MPI_File_write_at_all(mpifh, mpifo + offsetFromHeader, sbuf, nsme, MPI_CHAR, MPI_STATUS_IGNORE); - if (flush_flag) MPI_File_sync(mpifh); - } -} - -#if defined(_OPENMP) - -/* ---------------------------------------------------------------------- - multithreaded version - convert mybuf of doubles to one big formatted string in sbuf - return -1 if strlen exceeds an int, since used as arg in MPI calls in Dump -------------------------------------------------------------------------- */ - -int DumpCustomMPIIO::convert_string_omp(int n, double *mybuf) -{ - char **mpifh_buffer_line_per_thread; - int mpifhStringCount; - int *mpifhStringCountPerThread, *bufOffset, *bufRange, *bufLength; - - mpifhStringCount = 0; - - int nthreads = omp_get_max_threads(); - if (nthreads > n) { // call serial version - convert_string(n, mybuf); - - } else { - memory->create(mpifhStringCountPerThread, nthreads, "dump:mpifhStringCountPerThread"); - mpifh_buffer_line_per_thread = (char **) malloc(nthreads * sizeof(char *)); - memory->create(bufOffset, nthreads, "dump:bufOffset"); - memory->create(bufRange, nthreads, "dump:bufRange"); - memory->create(bufLength, nthreads, "dump:bufLength"); - - int i = 0; - for (i = 0; i < (nthreads - 1); i++) { - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = (int) (floor((double) n / (double) nthreads)); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - } - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i * (int) (floor((double) n / (double) nthreads)) * size_one); - bufRange[i] = n - (i * (int) (floor((double) n / (double) nthreads))); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(bufOffset, bufRange, bufLength, \ - mpifhStringCountPerThread, \ - mpifh_buffer_line_per_thread, mybuf) - { - int tid = omp_get_thread_num(); - int m = 0; - - for (int i = 0; i < bufRange[tid]; i++) { - - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc( - mpifh_buffer_line_per_thread[tid], - (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid] + DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - for (int j = 0; j < size_one; j++) { - - if (vtype[j] == Dump::INT) - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), - vformat[j], static_cast(mybuf[bufOffset[tid] + m])); - else if (vtype[j] == Dump::DOUBLE) - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), - vformat[j], mybuf[bufOffset[tid] + m]); - else if (vtype[j] == Dump::STRING) - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), - vformat[j], typenames[(int) mybuf[bufOffset[tid] + m]]); - m++; - } - mpifhStringCountPerThread[tid] += - sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]), "\n"); - } - } - -#pragma omp barrier - mpifhStringCount = 0; - for (i = 0; i < nthreads; i++) { mpifhStringCount += mpifhStringCountPerThread[i]; } - - memory->destroy(bufOffset); - memory->destroy(bufRange); - memory->destroy(bufLength); - - if (mpifhStringCount > 0) { - if (mpifhStringCount > maxsbuf) { - if (mpifhStringCount > MAXSMALLINT) return -1; - maxsbuf = mpifhStringCount + 1; - memory->grow(sbuf, maxsbuf, "dump:sbuf"); - } - sbuf[0] = '\0'; - } - - for (int i = 0; i < nthreads; i++) { - strcat(sbuf, mpifh_buffer_line_per_thread[i]); - free(mpifh_buffer_line_per_thread[i]); - } - - memory->destroy(mpifhStringCountPerThread); - free(mpifh_buffer_line_per_thread); - } - - return mpifhStringCount; -} -#endif diff --git a/src/MPIIO/dump_custom_mpiio.h b/src/MPIIO/dump_custom_mpiio.h deleted file mode 100644 index 3df72cb6e0..0000000000 --- a/src/MPIIO/dump_custom_mpiio.h +++ /dev/null @@ -1,68 +0,0 @@ -/* -*- c++ -*- ---------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifdef DUMP_CLASS -// clang-format off -DumpStyle(custom/mpiio,DumpCustomMPIIO); -// clang-format on -#else - -#ifndef LMP_DUMP_CUSTOM_MPIIO_H -#define LMP_DUMP_CUSTOM_MPIIO_H - -#include "dump_custom.h" - -namespace LAMMPS_NS { - -class DumpCustomMPIIO : public DumpCustom { - public: - DumpCustomMPIIO(class LAMMPS *, int, char **); - ~DumpCustomMPIIO() override; - - protected: - bigint - sumFileSize; // size in bytes of the file up through this rank offset from the end of the header data - char *headerBuffer; // buffer for holding header data - - MPI_File mpifh; - MPI_Offset mpifo, offsetFromHeader, headerSize, currentFileSize; - int performEstimate; // switch for write_data and write_header methods to use for gathering data and detemining filesize for preallocation vs actually writing the data - char *filecurrent; // name of file for this round (with % and * replaced) - -#if defined(_OPENMP) - int convert_string_omp(int, double *); // multithreaded version of convert_string -#endif - - void openfile() override; - void write_header(bigint) override; - void write() override; - void write_data(int, double *) override; - - void init_style() override; - typedef void (DumpCustomMPIIO::*FnPtrHeader)(bigint); - FnPtrHeader header_choice; // ptr to write header functions - void header_binary(bigint); - void header_binary_triclinic(bigint); - void header_item(bigint); - void header_item_triclinic(bigint); - - typedef void (DumpCustomMPIIO::*FnPtrData)(int, double *); - FnPtrData write_choice; // ptr to write data functions - void write_binary(int, double *); - void write_string(int, double *); -}; - -} // namespace LAMMPS_NS - -#endif -#endif diff --git a/src/MPIIO/dump_xyz_mpiio.cpp b/src/MPIIO/dump_xyz_mpiio.cpp deleted file mode 100644 index f1b00034ab..0000000000 --- a/src/MPIIO/dump_xyz_mpiio.cpp +++ /dev/null @@ -1,369 +0,0 @@ -// clang-format off -/* ---------------------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -/* ---------------------------------------------------------------------- - Contributing author: Paul Coffman (IBM) -------------------------------------------------------------------------- */ - -#include "dump_xyz_mpiio.h" - -#include "compute.h" -#include "domain.h" -#include "error.h" -#include "memory.h" -#include "update.h" - -#include -#include - -#include "omp_compat.h" -#if defined(_OPENMP) -#include -#endif - -using namespace LAMMPS_NS; - -#define MAX_TEXT_HEADER_SIZE 4096 -#define DUMP_BUF_CHUNK_SIZE 16384 -#define DUMP_BUF_INCREMENT_SIZE 4096 - -/* ---------------------------------------------------------------------- */ - -DumpXYZMPIIO::DumpXYZMPIIO(LAMMPS *lmp, int narg, char **arg) : - DumpXYZ(lmp, narg, arg) { - if (me == 0) - error->warning(FLERR,"MPI-IO output is unmaintained and unreliable. Use with caution."); -} - -/* ---------------------------------------------------------------------- */ - -DumpXYZMPIIO::~DumpXYZMPIIO() -{ - if (multifile == 0) MPI_File_close(&mpifh); -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::openfile() -{ - if (singlefile_opened) { // single file already opened, so just return after resetting filesize - mpifo = currentFileSize; - MPI_File_set_size(mpifh,mpifo+headerSize+sumFileSize); - currentFileSize = mpifo+headerSize+sumFileSize; - return; - } - if (multifile == 0) singlefile_opened = 1; - - // if one file per timestep, replace '*' with current timestep - - filecurrent = filename; - - if (multifile) { - filecurrent = utils::strdup(utils::star_subst(filecurrent, update->ntimestep, padflag)); - if (maxfiles > 0) { - if (numfiles < maxfiles) { - nameslist[numfiles] = utils::strdup(filecurrent); - ++numfiles; - } else { - remove(nameslist[fileidx]); - delete[] nameslist[fileidx]; - nameslist[fileidx] = utils::strdup(filecurrent); - fileidx = (fileidx + 1) % maxfiles; - } - } - } - - if (append_flag) { // append open - int err = MPI_File_open( world, filecurrent, MPI_MODE_CREATE | MPI_MODE_APPEND | - MPI_MODE_WRONLY , MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) error->one(FLERR, "Cannot open dump file {}", filecurrent); - - int myrank; - MPI_Comm_rank(world,&myrank); - if (myrank == 0) MPI_File_get_size(mpifh,&mpifo); - MPI_Bcast(&mpifo, 1, MPI_LMP_BIGINT, 0, world); - MPI_File_set_size(mpifh,mpifo+headerSize+sumFileSize); - currentFileSize = mpifo+headerSize+sumFileSize; - } else { // replace open - - int err = MPI_File_open( world, filecurrent, MPI_MODE_CREATE | MPI_MODE_WRONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) error->one(FLERR, "Cannot open dump file {}",filecurrent); - mpifo = 0; - - MPI_File_set_size(mpifh,(MPI_Offset) (headerSize+sumFileSize)); - currentFileSize = (headerSize+sumFileSize); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::write() -{ - if (domain->triclinic == 0) { - boxxlo = domain->boxlo[0]; - boxxhi = domain->boxhi[0]; - boxylo = domain->boxlo[1]; - boxyhi = domain->boxhi[1]; - boxzlo = domain->boxlo[2]; - boxzhi = domain->boxhi[2]; - } else { - boxxlo = domain->boxlo_bound[0]; - boxxhi = domain->boxhi_bound[0]; - boxylo = domain->boxlo_bound[1]; - boxyhi = domain->boxhi_bound[1]; - boxzlo = domain->boxlo_bound[2]; - boxzhi = domain->boxhi_bound[2]; - boxxy = domain->xy; - boxxz = domain->xz; - boxyz = domain->yz; - } - - // nme = # of dump lines this proc contributes to dump - - nme = count(); - - // ntotal = total # of dump lines in snapshot - // nmax = max # of dump lines on any proc - - bigint bnme = nme; - MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); - - int nmax; - MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); - - // write timestep header - // for multiproc, - // nheader = # of lines in this file via Allreduce on clustercomm - - bigint nheader = ntotal; - - // ensure filewriter proc can receive everyone's info - // limit nmax*size_one to int since used as arg in MPI_Rsend() below - // pack my data into buf - // if sorting on IDs also request ID list from pack() - // sort buf as needed - - if (nmax > maxbuf) { - if ((bigint) nmax * size_one > MAXSMALLINT) - error->all(FLERR,"Too much per-proc info for dump"); - maxbuf = nmax; - memory->destroy(buf); - memory->create(buf,(maxbuf*size_one),"dump:buf"); - } - if (sort_flag && sortcol == 0 && nmax > maxids) { - maxids = nmax; - memory->destroy(ids); - memory->create(ids,maxids,"dump:ids"); - } - - if (sort_flag && sortcol == 0) pack(ids); - else pack(nullptr); - if (sort_flag) sort(); - - // determine how much data needs to be written for setting the file size and prepocess it prior to writing - performEstimate = 1; - write_header(nheader); - write_data(nme,buf); - MPI_Bcast(&sumFileSize, 1, MPI_LMP_BIGINT, (nprocs-1), world); - - openfile(); - - performEstimate = 0; - write_header(nheader); // actually write the header - mpifo now points to end of header info - - // now actually write the data - performEstimate = 0; - write_data(nme,buf); - - if (multifile) MPI_File_close(&mpifh); - if (multifile) delete[] filecurrent; -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::init_style() -{ - // format = copy of default or user-specified line format - - delete[] format; - char *str; - if (format_line_user) str = format_line_user; - else str = format_default; - - int n = strlen(str) + 2; - format = new char[n]; - strcpy(format,str); - strcat(format,"\n"); - - // initialize typenames array to be backward compatible by default - // a 32-bit int can be maximally 10 digits plus sign - - if (typenames == nullptr) { - typenames = new char*[ntypes+1]; - for (int itype = 1; itype <= ntypes; itype++) - typenames[itype] = utils::strdup(std::to_string(itype)); - } - - // setup function ptr - write_choice = &DumpXYZMPIIO::write_string; -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::write_header(bigint n) -{ - auto header = fmt::format("{}\n Atoms. Timestep: {}", n, update->ntimestep); - if (time_flag) header += fmt::format(" Time: {:.6f}", compute_time()); - header += "\n"; - - if (performEstimate) { - headerSize = header.size(); - } else { // write data - if (me == 0) - MPI_File_write_at(mpifh,mpifo,(void *)header.c_str(),header.size(),MPI_CHAR,MPI_STATUS_IGNORE); - mpifo += header.size(); - } -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::write_data(int n, double *mybuf) -{ - (this->*write_choice)(n,mybuf); -} - -/* ---------------------------------------------------------------------- */ - -void DumpXYZMPIIO::write_string(int n, double *mybuf) -{ - if (performEstimate) { - -#if defined(_OPENMP) - int nthreads = omp_get_max_threads(); - if (nthreads > 1) - nsme = convert_string_omp(n,mybuf); - else - nsme = convert_string(n,mybuf); -#else - - nsme = convert_string(n,mybuf); -#endif - bigint incPrefix = 0; - bigint bigintNsme = (bigint) nsme; - MPI_Scan(&bigintNsme,&incPrefix,1,MPI_LMP_BIGINT,MPI_SUM,world); - sumFileSize = (incPrefix*sizeof(char)); - offsetFromHeader = ((incPrefix-bigintNsme)*sizeof(char)); - } - else { // write data - MPI_File_write_at_all(mpifh,mpifo+offsetFromHeader,sbuf,nsme,MPI_CHAR,MPI_STATUS_IGNORE); - if (flush_flag) - MPI_File_sync(mpifh); - } -} - -#if defined(_OPENMP) - -/* ---------------------------------------------------------------------- - multithreaded version - convert mybuf of doubles to one big formatted string in sbuf - return -1 if strlen exceeds an int, since used as arg in MPI calls in Dump -------------------------------------------------------------------------- */ - -int DumpXYZMPIIO::convert_string_omp(int n, double *mybuf) -{ - char **mpifh_buffer_line_per_thread; - int mpifhStringCount; - int *mpifhStringCountPerThread, *bufOffset, *bufRange, *bufLength; - - mpifhStringCount = 0; - - int nthreads = omp_get_max_threads(); - if (nthreads > n) { // call serial version - convert_string(n,mybuf); - - } - else { - memory->create(mpifhStringCountPerThread,nthreads,"dump:mpifhStringCountPerThread"); - mpifh_buffer_line_per_thread = (char **) malloc(nthreads*sizeof(char*)); - memory->create(bufOffset,nthreads,"dump:bufOffset"); - memory->create(bufRange,nthreads,"dump:bufRange"); - memory->create(bufLength,nthreads,"dump:bufLength"); - - int i=0; - for (i=0;i<(nthreads-1);i++) { - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i*(int)(floor((double)n/(double)nthreads))*size_one); - bufRange[i] = (int)(floor((double)n/(double)nthreads)); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - } - mpifhStringCountPerThread[i] = 0; - bufOffset[i] = (int) (i*(int)(floor((double)n/(double)nthreads))*size_one); - bufRange[i] = n-(i*(int)(floor((double)n/(double)nthreads))); - bufLength[i] = DUMP_BUF_CHUNK_SIZE; - mpifh_buffer_line_per_thread[i] = (char *) malloc(DUMP_BUF_CHUNK_SIZE * sizeof(char)); - mpifh_buffer_line_per_thread[i][0] = '\0'; - -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(bufOffset, bufRange, bufLength, mpifhStringCountPerThread, mpifh_buffer_line_per_thread, mybuf) - { - int tid = omp_get_thread_num(); - int m=0; - - for (int i = 0; i < bufRange[tid]; i++) { - - if ((bufLength[tid] - mpifhStringCountPerThread[tid]) < DUMP_BUF_INCREMENT_SIZE) { - mpifh_buffer_line_per_thread[tid] = (char *) realloc(mpifh_buffer_line_per_thread[tid],(mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char)); - bufLength[tid] = (mpifhStringCountPerThread[tid]+DUMP_BUF_CHUNK_SIZE) * sizeof(char); - } - - mpifhStringCountPerThread[tid] += sprintf(&(mpifh_buffer_line_per_thread[tid][mpifhStringCountPerThread[tid]]),format,typenames[static_cast (mybuf[bufOffset[tid]+m+1])],mybuf[bufOffset[tid]+m+2],mybuf[bufOffset[tid]+m+3],mybuf[bufOffset[tid]+m+4]); - m += size_one; - - } - } - -#pragma omp barrier - mpifhStringCount = 0; - for (i=0;idestroy(bufOffset); - memory->destroy(bufRange); - memory->destroy(bufLength); - - if (mpifhStringCount > 0) { - if (mpifhStringCount > maxsbuf) { - if (mpifhStringCount > MAXSMALLINT) return -1; - maxsbuf = mpifhStringCount+1; - memory->grow(sbuf,maxsbuf,"dump:sbuf"); - } - sbuf[0] = '\0'; - } - - - for (int i=0;idestroy(mpifhStringCountPerThread); - free(mpifh_buffer_line_per_thread); - - } - - return mpifhStringCount; -} -#endif diff --git a/src/MPIIO/dump_xyz_mpiio.h b/src/MPIIO/dump_xyz_mpiio.h deleted file mode 100644 index f1e41fabcd..0000000000 --- a/src/MPIIO/dump_xyz_mpiio.h +++ /dev/null @@ -1,61 +0,0 @@ -/* -*- c++ -*- --------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifdef DUMP_CLASS -// clang-format off -DumpStyle(xyz/mpiio,DumpXYZMPIIO); -// clang-format on -#else - -#ifndef LMP_DUMP_XYZ_MPIIO_H -#define LMP_DUMP_XYZ_MPIIO_H - -#include "dump_xyz.h" - -namespace LAMMPS_NS { - -class DumpXYZMPIIO : public DumpXYZ { - public: - DumpXYZMPIIO(class LAMMPS *, int, char **); - ~DumpXYZMPIIO() override; - - protected: - bigint - sumFileSize; // size in bytes of the file up through this rank offset from the end of the header data - char *headerBuffer; // buffer for holding header data - - MPI_File mpifh; - MPI_Offset mpifo, offsetFromHeader, headerSize, currentFileSize; - int performEstimate; // switch for write_data and write_header methods to use for gathering data and detemining filesize for preallocation vs actually writing the data - char *filecurrent; // name of file for this round (with % and * replaced) - -#if defined(_OPENMP) - int convert_string_omp(int, double *); // multithreaded version of convert_string -#endif - - void openfile() override; - void write_header(bigint) override; - void write() override; - void write_data(int, double *) override; - - void init_style() override; - - typedef void (DumpXYZMPIIO::*FnPtrData)(int, double *); - FnPtrData write_choice; // ptr to write data functions - void write_string(int, double *); -}; - -} // namespace LAMMPS_NS - -#endif -#endif diff --git a/src/MPIIO/restart_mpiio.cpp b/src/MPIIO/restart_mpiio.cpp deleted file mode 100644 index 0d40c2162c..0000000000 --- a/src/MPIIO/restart_mpiio.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// clang-format off -/* ---------------------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -/* ---------------------------------------------------------------------- - Contributing author: Paul Coffman (IBM) -------------------------------------------------------------------------- */ - -#include "restart_mpiio.h" - -#include "error.h" - -using namespace LAMMPS_NS; - -// the (rather old) version of ROMIO in MPICH for Windows -// uses "char *" instead of "const char *". This works around it. -#if defined(_WIN32) -#define ROMIO_COMPAT_CAST (char *) -#else -#define ROMIO_COMPAT_CAST -#endif - -/* ---------------------------------------------------------------------- */ - -RestartMPIIO::RestartMPIIO(LAMMPS *lmp) : Pointers(lmp) -{ - mpiio_exists = 1; - MPI_Comm_size(world,&nprocs); - MPI_Comm_rank(world,&myrank); -} - -/* ---------------------------------------------------------------------- - calls MPI_File_open in read-only mode, read_restart should call this - for some file servers it is most efficient to only read or only write -------------------------------------------------------------------------- */ - -void RestartMPIIO::openForRead(const char *filename) -{ - int err = MPI_File_open(world, ROMIO_COMPAT_CAST filename, MPI_MODE_RDONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot open restart file for reading - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } -} - -/* ---------------------------------------------------------------------- - calls MPI_File_open in write-only mode, write_restart should call this - for some file servers it is most efficient to only read or only write -------------------------------------------------------------------------- */ - -void RestartMPIIO::openForWrite(const char *filename) -{ - int err = MPI_File_open(world, ROMIO_COMPAT_CAST filename, MPI_MODE_WRONLY, - MPI_INFO_NULL, &mpifh); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot open restart file for writing - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } -} - -/* ---------------------------------------------------------------------- - determine the absolute offset for the data to be written with - MPI_Scan of the send sizes - compute the file size based off the MPI_Scan send size value on the last rank - set the filesize with ftruncate via MPI_File_set_size - write the data via collective MPI-IO by calling MPI_File_write_at_all -------------------------------------------------------------------------- */ - -void RestartMPIIO::write(MPI_Offset headerOffset, int send_size, double *buf) -{ - bigint incPrefix = 0; - bigint bigintSendSize = (bigint) send_size; - MPI_Scan(&bigintSendSize,&incPrefix,1,MPI_LMP_BIGINT,MPI_SUM,world); - - bigint largestIncPrefix = incPrefix; - MPI_Bcast(&largestIncPrefix, 1, MPI_LMP_BIGINT, (nprocs-1), world); - - int err = MPI_File_set_size(mpifh, - (headerOffset+(largestIncPrefix*sizeof(double)))); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot set restart file size - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } - - err = MPI_File_write_at_all(mpifh,headerOffset + - ((incPrefix-bigintSendSize)*sizeof(double)), - buf,send_size,MPI_DOUBLE,MPI_STATUS_IGNORE); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot write to restart file - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } -} - -/* ---------------------------------------------------------------------- - read the data into buf via collective MPI-IO by calling MPI_File_read_at_all - with the chunkOffset and chunkSize provided - if the consolidated chunksize is greater than INT_MAX - can only happen in extreme situation of reading restart file on - much fewer ranks than written and with relatively large data sizes - follow the collective IO call with rank independent IO to read remaining data -------------------------------------------------------------------------- */ - -void RestartMPIIO::read(MPI_Offset chunkOffset, bigint chunkSize, double *buf) -{ - int intChunkSize; - bigint remainingSize = 0; - if (chunkSize > INT_MAX) { - intChunkSize = INT_MAX; - remainingSize = chunkSize - INT_MAX; - } - else intChunkSize = (int) chunkSize; - - int err = MPI_File_read_at_all(mpifh,chunkOffset,buf,intChunkSize, - MPI_DOUBLE,MPI_STATUS_IGNORE); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot read from restart file - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } - - MPI_Offset currentOffset = chunkOffset+intChunkSize; - MPI_Offset bufOffset = intChunkSize; - while (remainingSize > 0) { - int currentChunkSize; - if (remainingSize > INT_MAX) { - currentChunkSize = INT_MAX; - remainingSize -= INT_MAX; - } - else { - currentChunkSize = remainingSize; - remainingSize = 0; - } - int err = MPI_File_read_at(mpifh,currentOffset,&buf[bufOffset], - currentChunkSize,MPI_DOUBLE,MPI_STATUS_IGNORE); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot read from restart file - MPI error: %s", - mpiErrorString); - error->one(FLERR,str); - } - currentOffset += currentChunkSize; - bufOffset += currentChunkSize; - } -} - -/* ---------------------------------------------------------------------- - calls MPI_File_close -------------------------------------------------------------------------- */ - -void RestartMPIIO::close() -{ - int err = MPI_File_close(&mpifh); - if (err != MPI_SUCCESS) { - char str[MPI_MAX_ERROR_STRING+128]; - char mpiErrorString[MPI_MAX_ERROR_STRING]; - int mpiErrorStringLength; - MPI_Error_string(err, mpiErrorString, &mpiErrorStringLength); - sprintf(str,"Cannot close restart file - MPI error: %s",mpiErrorString); - error->one(FLERR,str); - } -} diff --git a/src/MPIIO/restart_mpiio.h b/src/MPIIO/restart_mpiio.h deleted file mode 100644 index f9a7540ab1..0000000000 --- a/src/MPIIO/restart_mpiio.h +++ /dev/null @@ -1,40 +0,0 @@ -/* -*- c++ -*- ---------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifndef LMP_RESTART_MPIIO_H -#define LMP_RESTART_MPIIO_H - -#include "pointers.h" - -namespace LAMMPS_NS { - -class RestartMPIIO : protected Pointers { - private: - MPI_File mpifh; - int nprocs, myrank; - - public: - int mpiio_exists; - - RestartMPIIO(class LAMMPS *); - - void openForRead(const char *); - void openForWrite(const char *); - void write(MPI_Offset, int, double *); - void read(MPI_Offset, bigint, double *); - void close(); -}; - -} // namespace LAMMPS_NS - -#endif diff --git a/src/Makefile b/src/Makefile index 8aa7103376..5f905ee145 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ endif # PACKLIB = all packages that require an additional lib # should be PACKSYS + PACKINT + PACKEXT # PACKSYS = subset that reqiure a common system library -# include MPIIO and LB b/c require full MPI, not just STUBS +# include LATBOLTZ b/c it requires a full MPI, not just STUBS # PACKINT = subset that require an internal (provided) library # PACKEXT = subset that require an external (downloaded) library @@ -106,7 +106,6 @@ PACKAGE = \ mofff \ molecule \ molfile \ - mpiio \ netcdf \ openmp \ opt \ @@ -212,7 +211,6 @@ PACKLIB = \ kim \ kokkos \ lepton \ - mpiio \ poems \ python \ voronoi \ @@ -237,7 +235,7 @@ PACKLIB = \ vtk \ electrode -PACKSYS = compress latboltz mpiio python +PACKSYS = compress latboltz python PACKINT = atc awpmd colvars electrode gpu kokkos lepton ml-pod poems diff --git a/src/Purge.list b/src/Purge.list index 34ec934816..3052c3af1d 100644 --- a/src/Purge.list +++ b/src/Purge.list @@ -52,6 +52,17 @@ lmpgitversion.h mliap_model_python_couple.cpp mliap_model_python_couple.h # removed in August 2023 +dump_atom_mpiio.cpp +dump_atom_mpiio.h +dump_cfg_mpiio.cpp +dump_cfg_mpiio.h +dump_custom_mpiio.cpp +dump_custom_mpiio.h +dump_xyz_mpiio.cpp +dump_xyz_mpiio.h +restart_mpiio.cpp +restart_mpiio.h +# removed on 3 August 2023 fix_mscg.cpp fix_mscg.h # removed on 29 March 2023 diff --git a/src/dump.cpp b/src/dump.cpp index 39d8989c41..a231b367d5 100644 --- a/src/dump.cpp +++ b/src/dump.cpp @@ -120,8 +120,6 @@ Dump::Dump(LAMMPS *lmp, int /*narg*/, char **arg) : char *ptr; if ((ptr = strchr(filename,'%'))) { - if (strstr(style,"mpiio")) - error->all(FLERR,"Dump file MPI-IO output not allowed with % in filename"); multiproc = 1; nclusterprocs = 1; filewriter = 1; diff --git a/src/mpiio.h b/src/mpiio.h deleted file mode 100644 index 923aff0bbe..0000000000 --- a/src/mpiio.h +++ /dev/null @@ -1,51 +0,0 @@ -/* -*- c++ -*- ---------------------------------------------------------- - LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator - https://www.lammps.org/, Sandia National Laboratories - LAMMPS development team: developers@lammps.org - - Copyright (2003) Sandia Corporation. Under the terms of Contract - DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains - certain rights in this software. This software is distributed under - the GNU General Public License. - - See the README file in the top-level LAMMPS directory. -------------------------------------------------------------------------- */ - -#ifndef LMP_MPIIO_H -#define LMP_MPIIO_H - -// true interface to MPIIO package -// used when MPIIO package is installed - -#ifdef LMP_MPIIO - -#if defined(MPI_STUBS) -#error "The MPIIO package cannot be compiled in serial with MPI STUBS" -#endif - -#include "restart_mpiio.h" // IWYU pragma: export - -#else - -// dummy interface to MPIIO package -// needed for compiling when MPIIO package is not installed - -namespace LAMMPS_NS { - -class RestartMPIIO { - public: - int mpiio_exists; - - RestartMPIIO(class LAMMPS *) { mpiio_exists = 0; } - ~RestartMPIIO() {} - void openForRead(const char *) {} - void openForWrite(const char *) {} - void write(MPI_Offset, int, double *) {} - void read(MPI_Offset, long, double *) {} - void close() {} -}; - -} // namespace LAMMPS_NS - -#endif -#endif diff --git a/src/output.cpp b/src/output.cpp index 6e57122ffe..57518c9ec8 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -994,22 +994,12 @@ void Output::create_restart(int narg, char **arg) error->all(FLERR,"Both restart files must use % or neither"); } - int mpiioflag; - if (utils::strmatch(arg[1],"\\.mpiio$")) mpiioflag = 1; - else mpiioflag = 0; - if (nfile == 2) { - if (mpiioflag && !utils::strmatch(arg[2],"\\.mpiio$")) - error->all(FLERR,"Both restart files must use MPI-IO or neither"); - if (!mpiioflag && utils::strmatch(arg[2],"\\.mpiio$")) - error->all(FLERR,"Both restart files must use MPI-IO or neither"); - } - // setup output style and process optional args delete restart; restart = new WriteRestart(lmp); int iarg = nfile+1; - restart->multiproc_options(multiproc,mpiioflag,narg-iarg,&arg[iarg]); + restart->multiproc_options(multiproc,narg-iarg,&arg[iarg]); } /* ---------------------------------------------------------------------- diff --git a/src/read_restart.cpp b/src/read_restart.cpp index 0de36ebe1e..6925bd6096 100644 --- a/src/read_restart.cpp +++ b/src/read_restart.cpp @@ -30,7 +30,6 @@ #include "label_map.h" #include "memory.h" #include "modify.h" -#include "mpiio.h" #include "pair.h" #include "special.h" #include "update.h" @@ -43,7 +42,7 @@ using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -ReadRestart::ReadRestart(LAMMPS *lmp) : Command(lmp), mpiio(nullptr) {} +ReadRestart::ReadRestart(LAMMPS *lmp) : Command(lmp) {} /* ---------------------------------------------------------------------- */ @@ -88,17 +87,8 @@ void ReadRestart::command(int narg, char **arg) if (strchr(arg[0],'%')) multiproc = 1; else multiproc = 0; - if (strstr(arg[0],".mpiio")) mpiioflag = 1; - else mpiioflag = 0; - - if (multiproc && mpiioflag) - error->all(FLERR,"Read restart MPI-IO input not allowed with % in filename"); - - if (mpiioflag) { - mpiio = new RestartMPIIO(lmp); - if (!mpiio->mpiio_exists) - error->all(FLERR,"Reading from MPI-IO filename when MPIIO package is not installed"); - } + if (utils::strmatch(arg[0],"\\.mpiio")) + error->all(FLERR,"MPI-IO files are no longer supported by LAMMPS"); // open single restart file or base file for multiproc case @@ -182,28 +172,6 @@ void ReadRestart::command(int narg, char **arg) double *buf = nullptr; int m,flag; - // MPI-IO input from single file - - if (mpiioflag) { - mpiio->openForRead(file); - memory->create(buf,assignedChunkSize,"read_restart:buf"); - mpiio->read((headerOffset+assignedChunkOffset),assignedChunkSize,buf); - mpiio->close(); - - // can calculate number of atoms from assignedChunkSize - - if (!nextra) { - atom->nlocal = 1; // temporarily claim there is one atom... - int perAtomSize = avec->size_restart(); // ...so we can get its size - atom->nlocal = 0; // restore nlocal to zero atoms - int atomCt = (int) (assignedChunkSize / perAtomSize); - if (atomCt > atom->nmax) avec->grow(atomCt); - } - - m = 0; - while (m < assignedChunkSize) m += avec->unpack_restart(&buf[m]); - } - // input of single native file // nprocs_file = # of chunks in file // proc 0 reads a chunk and bcasts it to other procs @@ -211,7 +179,7 @@ void ReadRestart::command(int narg, char **arg) // if remapflag set, remap the atom to box before checking sub-domain // check for atom in sub-domain differs for orthogonal vs triclinic box - else if (multiproc == 0) { + if (multiproc == 0) { int triclinic = domain->triclinic; imageint *iptr; @@ -410,7 +378,7 @@ void ReadRestart::command(int narg, char **arg) // for multiproc or MPI-IO files: // perform irregular comm to migrate atoms to correct procs - if (multiproc || mpiioflag) { + if (multiproc) { // if remapflag set, remap all atoms I read back to box before migrating @@ -419,8 +387,7 @@ void ReadRestart::command(int narg, char **arg) imageint *image = atom->image; int nlocal = atom->nlocal; - for (int i = 0; i < nlocal; i++) - domain->remap(x[i],image[i]); + for (int i = 0; i < nlocal; i++) domain->remap(x[i],image[i]); } // create a temporary fix to hold and migrate extra atom info @@ -528,8 +495,6 @@ void ReadRestart::command(int narg, char **arg) if (comm->me == 0) utils::logmesg(lmp," read_restart CPU = {:.3f} seconds\n",platform::walltime()-time1); - - delete mpiio; } /* ---------------------------------------------------------------------- @@ -989,120 +954,9 @@ void ReadRestart::file_layout() error->all(FLERR,"Restart file is not a multi-proc file"); if (multiproc && multiproc_file == 0) error->all(FLERR,"Restart file is a multi-proc file"); - - } else if (flag == MPIIO) { - int mpiioflag_file = read_int(); - if (mpiioflag == 0 && mpiioflag_file) - error->all(FLERR,"Restart file is a MPI-IO file"); - if (mpiioflag && mpiioflag_file == 0) - error->all(FLERR,"Restart file is not a MPI-IO file"); - - if (mpiioflag) { - bigint *nproc_chunk_offsets; - memory->create(nproc_chunk_offsets,nprocs, - "write_restart:nproc_chunk_offsets"); - bigint *nproc_chunk_sizes; - memory->create(nproc_chunk_sizes,nprocs, - "write_restart:nproc_chunk_sizes"); - - // on rank 0 read in the chunk sizes that were written out - // then consolidate them and compute offsets relative to the - // end of the header info to fit the current partition size - // if the number of ranks that did the writing is different - - if (me == 0) { - int ndx; - int *all_written_send_sizes; - memory->create(all_written_send_sizes,nprocs_file, - "write_restart:all_written_send_sizes"); - int *nproc_chunk_number; - memory->create(nproc_chunk_number,nprocs, - "write_restart:nproc_chunk_number"); - - utils::sfread(FLERR,all_written_send_sizes,sizeof(int),nprocs_file,fp,nullptr,error); - - if ((nprocs != nprocs_file) && !(atom->nextra_store)) { - // nprocs differ, but atom sizes are fixed length, yeah! - atom->nlocal = 1; // temporarily claim there is one atom... - int perAtomSize = atom->avec->size_restart(); // ...so we can get its size - atom->nlocal = 0; // restore nlocal to zero atoms - - bigint total_size = 0; - for (int i = 0; i < nprocs_file; ++i) { - total_size += all_written_send_sizes[i]; - } - bigint total_ct = total_size / perAtomSize; - - bigint base_ct = total_ct / nprocs; - bigint leftover_ct = total_ct - (base_ct * nprocs); - bigint current_ByteOffset = 0; - base_ct += 1; - bigint base_ByteOffset = base_ct * (perAtomSize * sizeof(double)); - for (ndx = 0; ndx < leftover_ct; ++ndx) { - nproc_chunk_offsets[ndx] = current_ByteOffset; - nproc_chunk_sizes[ndx] = base_ct * perAtomSize; - current_ByteOffset += base_ByteOffset; - } - base_ct -= 1; - base_ByteOffset -= (perAtomSize * sizeof(double)); - for (; ndx < nprocs; ++ndx) { - nproc_chunk_offsets[ndx] = current_ByteOffset; - nproc_chunk_sizes[ndx] = base_ct * perAtomSize; - current_ByteOffset += base_ByteOffset; - } - } else { // we have to read in based on how it was written - int init_chunk_number = nprocs_file/nprocs; - int num_extra_chunks = nprocs_file - (nprocs*init_chunk_number); - - for (int i = 0; i < nprocs; i++) { - if (i < num_extra_chunks) - nproc_chunk_number[i] = init_chunk_number+1; - else - nproc_chunk_number[i] = init_chunk_number; - } - - int all_written_send_sizes_index = 0; - bigint current_offset = 0; - for (int i=0;idestroy(all_written_send_sizes); - memory->destroy(nproc_chunk_number); - } - - // scatter chunk sizes and offsets to all procs - - MPI_Scatter(nproc_chunk_sizes, 1, MPI_LMP_BIGINT, - &assignedChunkSize , 1, MPI_LMP_BIGINT, 0,world); - MPI_Scatter(nproc_chunk_offsets, 1, MPI_LMP_BIGINT, - &assignedChunkOffset , 1, MPI_LMP_BIGINT, 0,world); - - memory->destroy(nproc_chunk_sizes); - memory->destroy(nproc_chunk_offsets); - } } - flag = read_int(); } - - // if MPI-IO file, broadcast the end of the header offset - // this allows all ranks to compute offset to their data - - if (mpiioflag) { - if (me == 0) headerOffset = platform::ftell(fp); - MPI_Bcast(&headerOffset,1,MPI_LMP_BIGINT,0,world); - } } // ---------------------------------------------------------------------- diff --git a/src/read_restart.h b/src/read_restart.h index 573f49016a..ffc4d00e91 100644 --- a/src/read_restart.h +++ b/src/read_restart.h @@ -39,13 +39,6 @@ class ReadRestart : public Command { int nprocs_file; // total # of procs that wrote restart file int revision; // revision number of the restart file format - // MPI-IO values - - int mpiioflag; // 1 for MPIIO output, else 0 - class RestartMPIIO *mpiio; // MPIIO for restart file input - bigint assignedChunkSize; - MPI_Offset assignedChunkOffset, headerOffset; - std::string file_search(const std::string &); void header(); void type_arrays(); diff --git a/src/write_restart.cpp b/src/write_restart.cpp index 62cbe0078c..a996532687 100644 --- a/src/write_restart.cpp +++ b/src/write_restart.cpp @@ -29,7 +29,6 @@ #include "label_map.h" #include "memory.h" #include "modify.h" -#include "mpiio.h" #include "neighbor.h" #include "output.h" #include "pair.h" @@ -74,16 +73,13 @@ void WriteRestart::command(int narg, char **arg) if (strchr(arg[0],'%')) multiproc = nprocs; else multiproc = 0; - if (utils::strmatch(arg[0],"\\.mpiio$")) mpiioflag = 1; - else mpiioflag = 0; - - if ((comm->me == 0) && mpiioflag) - error->warning(FLERR,"MPI-IO output is unmaintained and unreliable. Use with caution."); + if (utils::strmatch(arg[0],"\\.mpiio$")) + error->all(FLERR,"MPI-IO files are no longer supported by LAMMPS"); // setup output style and process optional args // also called by Output class for periodic restart files - multiproc_options(multiproc,mpiioflag,narg-1,&arg[1]); + multiproc_options(multiproc,narg-1,&arg[1]); // init entire system since comm->exchange is done // comm::init needs neighbor::init needs pair::init needs kspace::init, etc @@ -119,21 +115,9 @@ void WriteRestart::command(int narg, char **arg) /* ---------------------------------------------------------------------- */ -void WriteRestart::multiproc_options(int multiproc_caller, int mpiioflag_caller, int narg, char **arg) +void WriteRestart::multiproc_options(int multiproc_caller, int narg, char **arg) { multiproc = multiproc_caller; - mpiioflag = mpiioflag_caller; - - // error checks - - if (multiproc && mpiioflag) - error->all(FLERR,"Restart file MPI-IO output not allowed with % in filename"); - - if (mpiioflag) { - mpiio = new RestartMPIIO(lmp); - if (!mpiio->mpiio_exists) - error->all(FLERR,"Writing to MPI-IO filename when MPIIO package is not installed"); - } // defaults for multiproc file writing @@ -354,49 +338,34 @@ void WriteRestart::write(const std::string &file) } } - // MPI-IO output to single file + // output of one or more native files + // filewriter = 1 = this proc writes to file + // ping each proc in my cluster, receive its data, write data to file + // else wait for ping from fileproc, send my data to fileproc - if (mpiioflag) { - if (me == 0 && fp) { - magic_string(); - if (ferror(fp)) io_error = 1; - fclose(fp); - fp = nullptr; + int tmp,recv_size; + + if (filewriter) { + MPI_Status status; + MPI_Request request; + for (int iproc = 0; iproc < nclusterprocs; iproc++) { + if (iproc) { + MPI_Irecv(buf,max_size,MPI_DOUBLE,me+iproc,0,world,&request); + MPI_Send(&tmp,0,MPI_INT,me+iproc,0,world); + MPI_Wait(&request,&status); + MPI_Get_count(&status,MPI_DOUBLE,&recv_size); + } else recv_size = send_size; + + write_double_vec(PERPROC,recv_size,buf); } - mpiio->openForWrite(file.c_str()); - mpiio->write(headerOffset,send_size,buf); - mpiio->close(); + magic_string(); + if (ferror(fp)) io_error = 1; + fclose(fp); + fp = nullptr; + } else { - - // output of one or more native files - // filewriter = 1 = this proc writes to file - // ping each proc in my cluster, receive its data, write data to file - // else wait for ping from fileproc, send my data to fileproc - - int tmp,recv_size; - - if (filewriter) { - MPI_Status status; - MPI_Request request; - for (int iproc = 0; iproc < nclusterprocs; iproc++) { - if (iproc) { - MPI_Irecv(buf,max_size,MPI_DOUBLE,me+iproc,0,world,&request); - MPI_Send(&tmp,0,MPI_INT,me+iproc,0,world); - MPI_Wait(&request,&status); - MPI_Get_count(&status,MPI_DOUBLE,&recv_size); - } else recv_size = send_size; - - write_double_vec(PERPROC,recv_size,buf); - } - magic_string(); - if (ferror(fp)) io_error = 1; - fclose(fp); - fp = nullptr; - - } else { - MPI_Recv(&tmp,0,MPI_INT,fileproc,0,world,MPI_STATUS_IGNORE); - MPI_Rsend(buf,send_size,MPI_DOUBLE,fileproc,0,world); - } + MPI_Recv(&tmp,0,MPI_INT,fileproc,0,world,MPI_STATUS_IGNORE); + MPI_Rsend(buf,send_size,MPI_DOUBLE,fileproc,0,world); } // check for I/O error status @@ -578,18 +547,7 @@ void WriteRestart::force_fields() void WriteRestart::file_layout(int send_size) { - if (me == 0) { - write_int(MULTIPROC,multiproc); - write_int(MPIIO,mpiioflag); - } - - if (mpiioflag) { - int *all_send_sizes; - memory->create(all_send_sizes,nprocs,"write_restart:all_send_sizes"); - MPI_Gather(&send_size, 1, MPI_INT, all_send_sizes, 1, MPI_INT, 0,world); - if (me == 0) fwrite(all_send_sizes,sizeof(int),nprocs,fp); - memory->destroy(all_send_sizes); - } + if (me == 0) write_int(MULTIPROC,multiproc); // -1 flag signals end of file layout info @@ -597,14 +555,6 @@ void WriteRestart::file_layout(int send_size) int flag = -1; fwrite(&flag,sizeof(int),1,fp); } - - // if MPI-IO file, broadcast the end of the header offste - // this allows all ranks to compute offset to their data - - if (mpiioflag) { - if (me == 0) headerOffset = platform::ftell(fp); - MPI_Bcast(&headerOffset,1,MPI_LMP_BIGINT,0,world); - } } // ---------------------------------------------------------------------- diff --git a/src/write_restart.h b/src/write_restart.h index 390e143ed5..1dcb003a4b 100644 --- a/src/write_restart.h +++ b/src/write_restart.h @@ -28,7 +28,7 @@ class WriteRestart : public Command { public: WriteRestart(class LAMMPS *); void command(int, char **) override; - void multiproc_options(int, int, int, char **); + void multiproc_options(int, int, char **); void write(const std::string &); private: @@ -44,12 +44,6 @@ class WriteRestart : public Command { int fileproc; // ID of proc in my cluster who writes to file int icluster; // which cluster I am in - // MPI-IO values - - int mpiioflag; // 1 for MPIIO output, else 0 - class RestartMPIIO *mpiio; // MPIIO for restart file output - MPI_Offset headerOffset; - void header(); void type_arrays(); void force_fields(); @@ -66,8 +60,6 @@ class WriteRestart : public Command { void write_int_vec(int, int, int *); void write_double_vec(int, int, double *); }; - } // namespace LAMMPS_NS - #endif #endif diff --git a/unittest/formats/test_file_operations.cpp b/unittest/formats/test_file_operations.cpp index c05a71a494..19d7a0999b 100644 --- a/unittest/formats/test_file_operations.cpp +++ b/unittest/formats/test_file_operations.cpp @@ -324,7 +324,6 @@ TEST_F(FileOperationsTest, write_restart) command("write_restart multi-%.restart"); command("write_restart multi2-%.restart fileper 2"); command("write_restart multi3-%.restart nfile 1"); - // if (Info::has_package("MPIIO")) command("write_restart test.restart.mpiio"); END_HIDE_OUTPUT(); ASSERT_FILE_EXISTS("noinit.restart"); @@ -336,19 +335,6 @@ TEST_F(FileOperationsTest, write_restart) ASSERT_FILE_EXISTS("multi2-0.restart"); ASSERT_FILE_EXISTS("multi3-base.restart"); ASSERT_FILE_EXISTS("multi3-0.restart"); -#if 0 - if (Info::has_package("MPIIO")) { - ASSERT_FILE_EXISTS("test.restart.mpiio"); - } - - if (!Info::has_package("MPIIO")) { - TEST_FAILURE(".*ERROR: Writing to MPI-IO filename when MPIIO package is not inst.*", - command("write_restart test.restart.mpiio");); - } else { - TEST_FAILURE(".*ERROR: Restart file MPI-IO output not allowed with % in filename.*", - command("write_restart test.restart-%.mpiio");); - } -#endif TEST_FAILURE(".*ERROR: Illegal write_restart command.*", command("write_restart");); TEST_FAILURE(".*ERROR: Unknown write_restart keyword: xxxx.*", command("write_restart test.restart xxxx");); @@ -402,7 +388,6 @@ TEST_F(FileOperationsTest, write_restart) delete_file("multi3-base.restart"); delete_file("multi3-0.restart"); delete_file("triclinic.restart"); - //if (Info::has_package("MPIIO")) delete_file("test.restart.mpiio"); } TEST_F(FileOperationsTest, write_data)