diff --git a/src/EXTRA-FIX/fix_ttm_grid.cpp b/src/EXTRA-FIX/fix_ttm_grid.cpp index 862e487d1b..958c2d2dc1 100644 --- a/src/EXTRA-FIX/fix_ttm_grid.cpp +++ b/src/EXTRA-FIX/fix_ttm_grid.cpp @@ -56,9 +56,10 @@ FixTTMGrid::FixTTMGrid(LAMMPS *lmp, int narg, char **arg) : { pergrid_flag = 1; pergrid_freq = 1; + restart_file = 1; if (outfile) error->all(FLERR,"Fix ttm/grid does not support outfile option - " - "use dump grid command instead"); + "use dump grid command or restart files instead"); skin_original = neighbor->skin; } @@ -275,16 +276,13 @@ void FixTTMGrid::end_of_step() /* ---------------------------------------------------------------------- read electron temperatures on grid from a user-specified file - proc 0 reads one chunk at a time, broadcasts to other procs - each proc stores values for grid points it owns ------------------------------------------------------------------------- */ void FixTTMGrid::read_electron_temperatures(const std::string &filename) { - int ***T_initial_set; - memory->create3d_offset(T_initial_set, nzlo_in, nzhi_in, nylo_in, nyhi_in, nxlo_in, nxhi_in, - "ttm/grid:T_initial_set"); - memset(&T_initial_set[nzlo_in][nylo_in][nxlo_in], 0, ngridown * sizeof(int)); + memory->create3d_offset(T_electron_read, nzlo_in, nzhi_in, nylo_in, nyhi_in, nxlo_in, nxhi_in, + "ttm/grid:T_electron_read"); + memset(&T_electron_read[nzlo_in][nylo_in][nxlo_in], 0, ngridown * sizeof(int)); // proc 0 opens file @@ -294,71 +292,149 @@ void FixTTMGrid::read_electron_temperatures(const std::string &filename) if (!fp) error->one(FLERR, "Cannot open grid file: {}: {}", filename, utils::getsyserror()); } - // read electron temperature values from file, one chunk at a time + // read the file + // Grid3d::read_file() calls back to read_grid_lines() with chunks of lines - auto buffer = new char[CHUNK * MAXLINE]; - bigint ntotal = (bigint) nxgrid * nygrid * nzgrid; - bigint nread = 0; - - while (nread < ntotal) { - int nchunk = MIN(ntotal - nread, CHUNK); - int eof = utils::read_lines_from_file(fp, nchunk, MAXLINE, buffer, comm->me, world); - if (eof) error->all(FLERR, "Unexpected end of data file"); - - // loop over lines of grid point values - // tokenize the line into ix,iy,iz grid index plus temperature value - // if I own grid point, store the value - - for (const auto &line : utils::split_lines(buffer)) { - try { - ValueTokenizer values(utils::trim_comment(line)); - if (values.count() == 0) { - ; // ignore comment only lines - } else if (values.count() == 4) { - ++nread; - - int ix = values.next_int(); - int iy = values.next_int(); - int iz = values.next_int(); - - if (ix < 0 || ix >= nxgrid || iy < 0 || iy >= nygrid || iz < 0 || iz >= nzgrid) - throw TokenizerException("Fix ttm/grid invalid grid index in input", ""); - - if (ix >= nxlo_in && ix <= nxhi_in && iy >= nylo_in && iy <= nyhi_in && iz >= nzlo_in && - iz <= nzhi_in) { - T_electron[iz][iy][ix] = values.next_double(); - T_initial_set[iz][iy][ix] = 1; - } - } else { - throw TokenizerException("Incorrect format in fix ttm electron grid file", ""); - } - } catch (std::exception &e) { - error->one(FLERR, e.what()); - } - } - } + grid->read_file(Grid3d::FIX,this,fp,CHUNK,MAXLINE); // close file if (comm->me == 0) fclose(fp); - // clean up - - delete[] buffer; - // check completeness of input data int flag = 0; for (int iz = nzlo_in; iz <= nzhi_in; iz++) for (int iy = nylo_in; iy <= nyhi_in; iy++) for (int ix = nxlo_in; ix <= nxhi_in; ix++) - if (T_initial_set[iz][iy][ix] == 0) flag = 1; + if (T_electron_read[iz][iy][ix] == 0) flag = 1; int flagall; MPI_Allreduce(&flag, &flagall, 1, MPI_INT, MPI_SUM, world); if (flagall) error->all(FLERR, "Fix ttm/grid infile did not set all temperatures"); - memory->destroy3d_offset(T_initial_set, nzlo_in, nylo_in, nxlo_in); + memory->destroy3d_offset(T_electron_read, nzlo_in, nylo_in, nxlo_in); +} + +/* ---------------------------------------------------------------------- + process a chunk of lines in buffer + each proc stores values for grid points it owns + called back to from Grid3d::read_file() +------------------------------------------------------------------------- */ + +int FixTTMGrid::unpack_read_grid(char *buffer) +{ + // loop over chunk of lines of grid point values + // skip comment lines + // tokenize the line into ix,iy,iz grid index plus temperature value + // if I own grid point, store the value + + int nread = 0; + + for (const auto &line : utils::split_lines(buffer)) { + try { + ValueTokenizer values(utils::trim_comment(line)); + if (values.count() == 0) { + ; // ignore comment only lines + } else if (values.count() == 4) { + ++nread; + + int ix = values.next_int(); + int iy = values.next_int(); + int iz = values.next_int(); + + if (ix < 0 || ix >= nxgrid || iy < 0 || iy >= nygrid || iz < 0 || iz >= nzgrid) + throw TokenizerException("Fix ttm/grid invalid grid index in input", ""); + + if (ix >= nxlo_in && ix <= nxhi_in && iy >= nylo_in && iy <= nyhi_in && iz >= nzlo_in && + iz <= nzhi_in) { + T_electron[iz][iy][ix] = values.next_double(); + T_electron_read[iz][iy][ix] = 1; + } + } else { + throw TokenizerException("Incorrect format in fix ttm electron grid file", ""); + } + } catch (std::exception &e) { + error->one(FLERR, e.what()); + } + } + + return nread; +} + +/* ---------------------------------------------------------------------- + write electron temperatures on grid to file + identical format to infile option, so info can be read in when restarting + each proc contributes info for its portion of grid +------------------------------------------------------------------------- */ + +void FixTTMGrid::write_restart_file(const char *file) +{ + // proc 0 opens file and writes header + + if (comm->me == 0) { + auto outfile = std::string(file) + ".ttm"; + fpout = fopen(outfile.c_str(),"w"); + if (fpout == nullptr) + error->one(FLERR,"Cannot open fix ttm/grid restart file {}: {}",outfile,utils::getsyserror()); + + bigint ngrid = (bigint) nxgrid * nygrid * nzgrid; + fmt::print(fpout,"# fix ttm electron temperature on grid for " + "{} grid points on timestep {}\n\n",ngrid,update->ntimestep); + } + + // write file + // Grid3d::write_file() calls back to pack_write_file() and unpack_write_file() + + grid->write_file(Grid3d::FIX,this,0,1,sizeof(double), MPI_DOUBLE); + + // close file + + if (comm->me == 0) fclose(fpout); +} + +/* ---------------------------------------------------------------------- + pack values from local grid into buf +------------------------------------------------------------------------- */ + +void FixTTMGrid::pack_write_grid(int /*which*/, void *vbuf) +{ + int ix, iy, iz; + + auto buf = (double *) vbuf; + + int m = 0; + for (iz = nzlo_in; iz <= nzhi_in; iz++) + for (iy = nylo_in; iy <= nyhi_in; iy++) + for (ix = nxlo_in; ix <= nxhi_in; ix++) + buf[m++] = T_electron[iz][iy][ix]; +} + +/* ---------------------------------------------------------------------- + unpcak values from buf and write them to restart file +------------------------------------------------------------------------- */ + +void FixTTMGrid::unpack_write_grid(int /*which*/, void *vbuf, int *bounds) +{ + int ix, iy, iz; + + int xlo = bounds[0]; + int xhi = bounds[1]; + int ylo = bounds[2]; + int yhi = bounds[3]; + int zlo = bounds[4]; + int zhi = bounds[5]; + + auto buf = (double *) vbuf; + double value; + + int m = 0; + for (iz = zlo; iz <= zhi; iz++) + for (iy = ylo; iy <= yhi; iy++) + for (ix = xlo; ix <= xhi; ix++) { + value = buf[m++]; + fprintf(fpout, "%d %d %d %20.16g\n", ix, iy, iz, value); + } } /* ---------------------------------------------------------------------- @@ -540,82 +616,6 @@ void FixTTMGrid::deallocate_grid() memory->destroy3d_offset(net_energy_transfer, nzlo_out, nylo_out, nxlo_out); } -/* ---------------------------------------------------------------------- - pack entire state of Fix into one write -------------------------------------------------------------------------- */ - -void FixTTMGrid::write_restart(FILE *fp) -{ - error->all(FLERR,"Fix ttm/grid command does not yet support " - "writing a distributed grid to a restart file"); -} - -/* ---------------------------------------------------------------------- - use state info from restart file to restart the Fix -------------------------------------------------------------------------- */ - -void FixTTMGrid::restart(char *buf) -{ - error->all(FLERR,"Fix ttm/grid command does not yet support " - "reading a distributed grid from a restart file"); -} - -/* ---------------------------------------------------------------------- - pack values from local grid into buf - used by which = 0 and 1 - NOTE: remove this function when ready to release -------------------------------------------------------------------------- */ - -void FixTTMGrid::pack_gather_grid(int /*which*/, void *vbuf) -{ - int ix, iy, iz; - - auto buf = (double *) vbuf; - - int m = 0; - for (iz = nzlo_in; iz <= nzhi_in; iz++) - for (iy = nylo_in; iy <= nyhi_in; iy++) - for (ix = nxlo_in; ix <= nxhi_in; ix++) buf[m++] = T_electron[iz][iy][ix]; -} - -/* ---------------------------------------------------------------------- - which = 0: unpack values from buf into global gbuf based on their indices - which = 1: print values from buf to FPout file - NOTE: remove this function when ready to release -------------------------------------------------------------------------- */ - -void FixTTMGrid::unpack_gather_grid(int which, void *vbuf, void *vgbuf, int xlo, int xhi, int ylo, - int yhi, int zlo, int zhi) -{ - int ix, iy, iz; - - auto buf = (double *) vbuf; - auto gbuf = (double *) vgbuf; - - if (which == 0) { - int iglobal; - int ilocal = 0; - - for (iz = zlo; iz <= zhi; iz++) - for (iy = ylo; iy <= yhi; iy++) - for (ix = xlo; ix <= xhi; ix++) { - iglobal = nygrid * nxgrid * iz + nxgrid * iy + ix; - gbuf[iglobal] = buf[ilocal++]; - } - - } else if (which == 1) { - int ilocal = 0; - double value; - - for (iz = zlo; iz <= zhi; iz++) - for (iy = ylo; iy <= yhi; iy++) - for (ix = xlo; ix <= xhi; ix++) { - value = buf[ilocal++]; - fprintf(FPout, "%d %d %d %20.16g\n", ix, iy, iz, value); - } - } -} - /* ---------------------------------------------------------------------- return index of grid associated with name this class can store M named grids, indexed 0 to M-1 diff --git a/src/EXTRA-FIX/fix_ttm_grid.h b/src/EXTRA-FIX/fix_ttm_grid.h index 5486dbb39b..cbc4804d95 100644 --- a/src/EXTRA-FIX/fix_ttm_grid.h +++ b/src/EXTRA-FIX/fix_ttm_grid.h @@ -32,6 +32,9 @@ class FixTTMGrid : public FixTTM { void init() override; void post_force(int) override; void end_of_step() override; + void write_restart_file(const char *) override; + double compute_vector(int) override; + double memory_usage() override; // grid communication @@ -43,27 +46,22 @@ class FixTTMGrid : public FixTTM { void unpack_reverse_grid(int, void *, int, int *) override; void pack_remap_grid(void *, int, int *) override; void unpack_remap_grid(void *, int, int *) override; - void pack_gather_grid(int, void *) override; - void unpack_gather_grid(int, void *, void *, int, int, int, int, int, int) override; - - void write_restart(FILE *) override; - void restart(char *) override; + int unpack_read_grid(char *) override; + void pack_write_grid(int, void *) override; + void unpack_write_grid(int, void *, int *) override; int get_grid_by_name(const std::string &, int &) override; void *get_grid_by_index(int) override; int get_griddata_by_name(int, const std::string &, int &) override; void *get_griddata_by_index(int) override; - double compute_vector(int) override; - double memory_usage() override; - private: int ngridown, ngridout; int nxlo_in, nxhi_in, nylo_in, nyhi_in, nzlo_in, nzhi_in; int nxlo_out, nxhi_out, nylo_out, nyhi_out, nzlo_out, nzhi_out; double delxinv, delyinv, delzinv; double skin_original; - FILE *FPout; + FILE *fpout; class Grid3d *grid; class Grid3d *grid_previous; @@ -71,6 +69,8 @@ class FixTTMGrid : public FixTTM { int ngrid_buf1, ngrid_buf2; double *grid_buf1, *grid_buf2; + double ***T_electron_read; + void allocate_grid() override; void deallocate_grid() override; void read_electron_temperatures(const std::string &) override; diff --git a/src/fix.h b/src/fix.h index dc2d7f933f..6dfb383091 100644 --- a/src/fix.h +++ b/src/fix.h @@ -219,9 +219,10 @@ class Fix : protected Pointers { virtual void unpack_reverse_grid(int, void *, int, int *){}; virtual void pack_remap_grid(void *, int, int *){}; virtual void unpack_remap_grid(void *, int, int *){}; - virtual void pack_gather_grid(int, void *){}; - virtual void unpack_gather_grid(int, void *, void *, int, int, int, int, int, int){}; - + virtual int unpack_read_grid(char *) {return 0;}; + virtual void pack_write_grid(int, void *){}; + virtual void unpack_write_grid(int, void *, int *){}; + virtual int get_grid_by_name(const std::string &, int &) { return -1; }; virtual void *get_grid_by_index(int) { return nullptr; }; virtual int get_griddata_by_name(int, const std::string &, int &) { return -1; }; diff --git a/src/grid2d.cpp b/src/grid2d.cpp index 0696162e02..edbfc5623d 100644 --- a/src/grid2d.cpp +++ b/src/grid2d.cpp @@ -1312,40 +1312,84 @@ void Grid2d::remap_style(T *ptr, int nper, int nbyte, } // ---------------------------------------------------------------------- -// gather/scatter grid data between one and many procs for I/O purposes +// grid I/O methods // ---------------------------------------------------------------------- /* ---------------------------------------------------------------------- - gather global grid values to proc 0, one grid chunk at a time - proc 0 pings each proc for its grid chunk - pack/unpack operations are performed by caller via callbacks - caller can decide whether to store chunks, output them, etc + read grid values from a file ------------------------------------------------------------------------- */ -void Grid2d::gather(int /*caller*/, void *ptr, int nper, int nbyte, - int which, void *buf, MPI_Datatype datatype) +void Grid2d::read_file(int caller, void *ptr, FILE *fp, int nchunk, int maxline) +{ + if (caller == FIX) + read_file_style((Fix *) ptr,fp,nchunk,maxline); +} + +/* ---------------------------------------------------------------------- + proc 0 reads one chunk of lines at a time from file + broadcast chunk buffer to other procs + call back to caller so it can process the chunk of lines + caller returns count of grid-value lines in chunk +------------------------------------------------------------------------- */ + +template < class T > +void Grid2d::read_file_style(T *ptr, FILE *fp, int nchunk, int maxline) +{ + auto buffer = new char[nchunk * maxline]; + bigint ntotal = (bigint) ngrid[0] * ngrid[1]; + bigint nread = 0; + + while (nread < ntotal) { + int nchunk = MIN(ntotal - nread, nchunk); + int eof = utils::read_lines_from_file(fp, nchunk, maxline, buffer, comm->me, world); + if (eof) error->all(FLERR, "Unexpected end of grid data file"); + + nread += ptr->unpack_read_grid(buffer); + } +} + +/* ---------------------------------------------------------------------- + write grid values to a file +------------------------------------------------------------------------- */ + +void Grid2d::write_file(int caller, void *ptr, int which, + int nper, int nbyte, MPI_Datatype datatype) +{ + if (caller == FIX) + write_file_style((Fix *) ptr, which, nper, nbyte, datatype); +} + +/* ---------------------------------------------------------------------- + proc 0 reads one chunk of lines at a time from file + broadcast chunk buffer to other procs + call back to caller so it can process the chunk of lines + caller returns count of grid-value lines in chunk +------------------------------------------------------------------------- */ + +template < class T > +void Grid2d::write_file_style(T *ptr, int which, + int nper, int nbyte, MPI_Datatype datatype) { int me = comm->me; - Fix *fptr = (Fix *) ptr; - // maxsize = max grid data owned by any proc + // maxsize = max size of grid data owned by any proc int mysize = (inxhi-inxlo+1) * (inyhi-inylo+1); mysize *= nper; int maxsize; MPI_Allreduce(&mysize,&maxsize,1,MPI_INT,MPI_MAX,world); - // pack my data via callback to caller + // pack my grid data via callback to caller - char *mybuf; - if (me == 0) memory->create(mybuf,maxsize*nbyte,"grid2d:mybuf"); - else memory->create(mybuf,mysize*nbyte,"grid2d:mybuf"); - fptr->pack_gather_grid(which,mybuf); + char *onebuf; + if (me == 0) memory->create(onebuf,maxsize*nbyte,"grid3d:onebuf"); + else memory->create(onebuf,mysize*nbyte,"grid3d:onebuf"); + ptr->pack_write_grid(which,onebuf); - // ping each proc for its data - // unpack into full buffer via callback to caller + // ping each proc for its grid data + // call back to caller with each proc's grid data - int xlo,xhi,ylo,yhi,tmp; + int tmp; int bounds[4]; if (me == 0) { @@ -1354,27 +1398,23 @@ void Grid2d::gather(int /*caller*/, void *ptr, int nper, int nbyte, for (int iproc = 0; iproc < nprocs; iproc++) { if (iproc) { - MPI_Irecv(mybuf,maxsize,datatype,iproc,0,world,&request); + MPI_Irecv(onebuf,maxsize,datatype,iproc,0,world,&request); MPI_Send(&tmp,0,MPI_INT,iproc,0,world); MPI_Wait(&request,&status); MPI_Recv(bounds,4,MPI_INT,iproc,0,world,&status); - xlo = bounds[0]; - xhi = bounds[1]; - ylo = bounds[2]; - yhi = bounds[3]; } else { - xlo = inxlo; - xhi = inxhi; - ylo = inylo; - yhi = inyhi; + bounds[0] = inxlo; + bounds[1] = inxhi; + bounds[2] = inylo; + bounds[3] = inyhi; } - fptr->unpack_gather_grid(which,mybuf,buf,xlo,xhi,ylo,yhi,0,0); + ptr->unpack_write_grid(which,onebuf,bounds); } } else { MPI_Recv(&tmp,0,MPI_INT,0,0,world,MPI_STATUS_IGNORE); - MPI_Rsend(mybuf,mysize,datatype,0,0,world); + MPI_Rsend(onebuf,mysize,datatype,0,0,world); bounds[0] = inxlo; bounds[1] = inxhi; bounds[2] = inylo; @@ -1382,7 +1422,9 @@ void Grid2d::gather(int /*caller*/, void *ptr, int nper, int nbyte, MPI_Send(bounds,4,MPI_INT,0,0,world); } - memory->destroy(mybuf); + // clean up + + memory->destroy(onebuf); } // ---------------------------------------------------------------------- diff --git a/src/grid2d.h b/src/grid2d.h index da2c09dbf3..ca450a7610 100644 --- a/src/grid2d.h +++ b/src/grid2d.h @@ -43,9 +43,10 @@ class Grid2d : protected Pointers { void setup_remap(Grid2d *, int &, int &); void remap(int, void *, int, int, void *, void *, MPI_Datatype); - void gather(int, void *, int, int, int, void *, MPI_Datatype); + void read_file(int, void *, FILE *, int, int); + void write_file(int, void *, int, int, int, MPI_Datatype); - protected: +protected: int me, nprocs; int layout; // REGULAR or TILED MPI_Comm gridcomm; // communicator for this class @@ -217,6 +218,9 @@ class Grid2d : protected Pointers { void setup_remap_tiled(Grid2d *, int &, int &); template void remap_style(T *, int, int, void *, void *, MPI_Datatype); + template void read_file_style(T *, FILE *, int, int); + template void write_file_style(T *, int, int, int, MPI_Datatype); + int compute_overlap(int *, int *, Overlap *&); void clean_overlap(); void box_drop(int *, int *); diff --git a/src/grid3d.cpp b/src/grid3d.cpp index a6a5c631e5..4656cec4ef 100644 --- a/src/grid3d.cpp +++ b/src/grid3d.cpp @@ -1448,40 +1448,84 @@ void Grid3d::remap_style(T *ptr, int nper, int nbyte, } // ---------------------------------------------------------------------- -// gather/scatter grid data between one and many procs for I/O purposes +// grid I/O methods // ---------------------------------------------------------------------- /* ---------------------------------------------------------------------- - gather global grid values to proc 0, one grid chunk at a time - proc 0 pings each proc for its grid chunk - pack/unpack operations are performed by caller via callbacks - caller can decide whether to store chunks, output them, etc + read grid values from a file ------------------------------------------------------------------------- */ -void Grid3d::gather(int /*caller*/, void *ptr, int nper, int nbyte, - int which, void *buf, MPI_Datatype datatype) +void Grid3d::read_file(int caller, void *ptr, FILE *fp, int nchunk, int maxline) +{ + if (caller == FIX) + read_file_style((Fix *) ptr,fp,nchunk,maxline); +} + +/* ---------------------------------------------------------------------- + proc 0 reads one chunk of lines at a time from file + broadcast chunk buffer to other procs + call back to caller so it can process the chunk of lines + caller returns count of grid-value lines in chunk +------------------------------------------------------------------------- */ + +template < class T > +void Grid3d::read_file_style(T *ptr, FILE *fp, int nchunk, int maxline) +{ + auto buffer = new char[nchunk * maxline]; + bigint ntotal = (bigint) ngrid[0] * ngrid[1] * ngrid[2]; + bigint nread = 0; + + while (nread < ntotal) { + int nchunk = MIN(ntotal - nread, nchunk); + int eof = utils::read_lines_from_file(fp, nchunk, maxline, buffer, comm->me, world); + if (eof) error->all(FLERR, "Unexpected end of grid data file"); + + nread += ptr->unpack_read_grid(buffer); + } +} + +/* ---------------------------------------------------------------------- + write grid values to a file +------------------------------------------------------------------------- */ + +void Grid3d::write_file(int caller, void *ptr, int which, + int nper, int nbyte, MPI_Datatype datatype) +{ + if (caller == FIX) + write_file_style((Fix *) ptr, which, nper, nbyte, datatype); +} + +/* ---------------------------------------------------------------------- + proc 0 reads one chunk of lines at a time from file + broadcast chunk buffer to other procs + call back to caller so it can process the chunk of lines + caller returns count of grid-value lines in chunk +------------------------------------------------------------------------- */ + +template < class T > +void Grid3d::write_file_style(T *ptr, int which, + int nper, int nbyte, MPI_Datatype datatype) { int me = comm->me; - Fix *fptr = (Fix *) ptr; - // maxsize = max grid data owned by any proc + // maxsize = max size of grid data owned by any proc int mysize = (inxhi-inxlo+1) * (inyhi-inylo+1) * (inzhi-inzlo+1); mysize *= nper; int maxsize; MPI_Allreduce(&mysize,&maxsize,1,MPI_INT,MPI_MAX,world); - // pack my data via callback to caller + // pack my grid data via callback to caller - char *mybuf; - if (me == 0) memory->create(mybuf,maxsize*nbyte,"grid3d:mybuf"); - else memory->create(mybuf,mysize*nbyte,"grid3d:mybuf"); - fptr->pack_gather_grid(which,mybuf); + char *onebuf; + if (me == 0) memory->create(onebuf,maxsize*nbyte,"grid3d:onebuf"); + else memory->create(onebuf,mysize*nbyte,"grid3d:nebuf"); + ptr->pack_write_grid(which,onebuf); - // ping each proc for its data - // unpack into full buffer via callback to caller + // ping each proc for its grid data + // call back to caller with each proc's grid data - int xlo,xhi,ylo,yhi,zlo,zhi,tmp; + int tmp; int bounds[6]; if (me == 0) { @@ -1490,31 +1534,25 @@ void Grid3d::gather(int /*caller*/, void *ptr, int nper, int nbyte, for (int iproc = 0; iproc < nprocs; iproc++) { if (iproc) { - MPI_Irecv(mybuf,maxsize,datatype,iproc,0,world,&request); + MPI_Irecv(onebuf,maxsize,datatype,iproc,0,world,&request); MPI_Send(&tmp,0,MPI_INT,iproc,0,world); MPI_Wait(&request,&status); MPI_Recv(bounds,6,MPI_INT,iproc,0,world,&status); - xlo = bounds[0]; - xhi = bounds[1]; - ylo = bounds[2]; - yhi = bounds[3]; - zlo = bounds[4]; - zhi = bounds[5]; } else { - xlo = inxlo; - xhi = inxhi; - ylo = inylo; - yhi = inyhi; - zlo = inzlo; - zhi = inzhi; + bounds[0] = inxlo; + bounds[1] = inxhi; + bounds[2] = inylo; + bounds[3] = inyhi; + bounds[4] = inzlo; + bounds[5] = inzhi; } - fptr->unpack_gather_grid(which,mybuf,buf,xlo,xhi,ylo,yhi,zlo,zhi); + ptr->unpack_write_grid(which,onebuf,bounds); } } else { MPI_Recv(&tmp,0,MPI_INT,0,0,world,MPI_STATUS_IGNORE); - MPI_Rsend(mybuf,mysize,datatype,0,0,world); + MPI_Rsend(onebuf,mysize,datatype,0,0,world); bounds[0] = inxlo; bounds[1] = inxhi; bounds[2] = inylo; @@ -1524,7 +1562,9 @@ void Grid3d::gather(int /*caller*/, void *ptr, int nper, int nbyte, MPI_Send(bounds,6,MPI_INT,0,0,world); } - memory->destroy(mybuf); + // clean up + + memory->destroy(onebuf); } // ---------------------------------------------------------------------- diff --git a/src/grid3d.h b/src/grid3d.h index 7ffa7eb5c0..3d653cd3b7 100644 --- a/src/grid3d.h +++ b/src/grid3d.h @@ -45,7 +45,8 @@ class Grid3d : protected Pointers { void setup_remap(Grid3d *, int &, int &); void remap(int, void *, int, int, void *, void *, MPI_Datatype); - void gather(int, void *, int, int, int, void *, MPI_Datatype); + void read_file(int, void *, FILE *, int, int); + void write_file(int, void *, int, int, int, MPI_Datatype); protected: int me, nprocs; @@ -221,6 +222,9 @@ class Grid3d : protected Pointers { template void remap_style(T *, int, int, void *, void *, MPI_Datatype); + template void read_file_style(T *, FILE *, int, int); + template void write_file_style(T *, int, int, int, MPI_Datatype); + int compute_overlap(int *, int *, Overlap *&); void clean_overlap(); void box_drop(int *, int *); diff --git a/src/write_restart.cpp b/src/write_restart.cpp index 2fe2f8d373..511df4c58f 100644 --- a/src/write_restart.cpp +++ b/src/write_restart.cpp @@ -399,7 +399,7 @@ void WriteRestart::write(const std::string &file) } } - // Check for I/O error status + // check for I/O error status int io_all = 0; MPI_Allreduce(&io_error,&io_all,1,MPI_INT,MPI_MAX,world);