diff --git a/src/output.cpp b/src/output.cpp index 7c97b85b1a..c865145769 100644 --- a/src/output.cpp +++ b/src/output.cpp @@ -762,13 +762,13 @@ void Output::create_restart(int narg, char **arg) error->all(FLERR,"Both restart files must have '%' or neither"); } - int mpiio; - if (strstr(arg[1],".mpi")) mpiio = 1; - else mpiio = 0; + int mpiioflag; + if (strstr(arg[1],".mpi")) mpiioflag = 1; + else mpiioflag = 0; if (nfile == 2) { - if (mpiio && !strstr(arg[2],".mpi")) + if (mpiioflag && !strstr(arg[2],".mpi")) error->all(FLERR,"Both restart files must use MPI-IO or neither"); - if (!mpiio && strstr(arg[2],".mpi")) + if (!mpiioflag && strstr(arg[2],".mpi")) error->all(FLERR,"Both restart files must use MPI-IO or neither"); } @@ -777,7 +777,7 @@ void Output::create_restart(int narg, char **arg) delete restart; restart = new WriteRestart(lmp); int iarg = nfile+1; - restart->multiproc_options(multiproc,mpiio,narg-iarg,&arg[iarg]); + restart->multiproc_options(multiproc,mpiioflag,narg-iarg,&arg[iarg]); } /* ---------------------------------------------------------------------- diff --git a/src/read_restart.cpp b/src/read_restart.cpp index 29135fa01b..e6f2330844 100644 --- a/src/read_restart.cpp +++ b/src/read_restart.cpp @@ -35,6 +35,7 @@ #include "improper.h" #include "special.h" #include "universe.h" +#include "mpiio.h" #include "memory.h" #include "error.h" @@ -96,13 +97,20 @@ void ReadRestart::command(int narg, char **arg) if (strchr(arg[0],'%')) multiproc = 1; else multiproc = 0; - if (strstr(arg[0],".mpi")) mpiio = 1; - else mpiio = 0; + if (strstr(arg[0],".mpi")) mpiioflag = 1; + else mpiioflag = 0; - if (multiproc && mpiio) + if (multiproc && mpiioflag) error->all(FLERR, "Read restart MPI-IO output not allowed with '%' in filename"); + if (mpiioflag) { + mpiio = new RestartMPIIO(lmp); + if (!mpiio->mpiio_exists) + error->all(FLERR,"Reading from MPI-IO filename when " + "MPIIO package is not installed"); + } + // open single restart file or base file for multiproc case if (me == 0) { @@ -170,19 +178,37 @@ void ReadRestart::command(int narg, char **arg) if (me == 0) fclose(fp); - // single file: - // nprocs_file = # of chunks in file - // proc 0 reads a chunk and bcasts it to other procs - // each proc unpacks the atoms, saving ones in it's sub-domain - // check for atom in sub-domain differs for orthogonal vs triclinic box - AtomVec *avec = atom->avec; int maxbuf = 0; double *buf = NULL; int m; - if (multiproc == 0) { + // MPI-IO input from single file + + if (mpiioflag) { + // add calls to RestartMPIIO class + // reopen header file + // perform reads + // allow for different # of procs reading than wrote the file + + // mpiio->open(file); + // mpiio->read(); + // mpiio->close(); + + // then process atom info as + + //m = 0; + //while (m < n) m += avec->unpack_restart(&buf[m]); + } + + // input of single native file + // nprocs_file = # of chunks in file + // proc 0 reads a chunk and bcasts it to other procs + // each proc unpacks the atoms, saving ones in it's sub-domain + // check for atom in sub-domain differs for orthogonal vs triclinic box + + else if (multiproc == 0) { int triclinic = domain->triclinic; double *x,lamda[3]; @@ -225,13 +251,14 @@ void ReadRestart::command(int narg, char **arg) } if (me == 0) fclose(fp); + } - // multiple files with procs <= files + // input of multiple native files with procs <= files // # of files = multiproc_file // each proc reads a subset of files, striding by nprocs // each proc keeps all atoms in all perproc chunks in its files - } else if (nprocs <= multiproc_file) { + else if (nprocs <= multiproc_file) { char *procfile = new char[strlen(file) + 16]; char *ptr = strchr(file,'%'); @@ -275,15 +302,16 @@ void ReadRestart::command(int narg, char **arg) } delete [] procfile; + } - // multiple files with procs > files + // input of multiple native files with procs > files // # of files = multiproc_file // cluster procs based on # of files // 1st proc in each cluster reads per-proc chunks from file // sends chunks round-robin to other procs in its cluster // each proc keeps all atoms in its perproc chunks in file - } else { + else { // nclusterprocs = # of procs in my cluster that read from one file // filewriter = 1 if this proc reads file, else 0 @@ -382,10 +410,10 @@ void ReadRestart::command(int narg, char **arg) delete [] file; memory->destroy(buf); - // for multiproc files: + // for multiproc or MPI-IO files: // perform irregular comm to migrate atoms to correct procs - if (multiproc) { + if (multiproc || mpiioflag) { // create a temporary fix to hold and migrate extra atom info // necessary b/c irregular will migrate atoms @@ -899,13 +927,16 @@ void ReadRestart::file_layout() error->all(FLERR,"Restart file is a multi-proc file"); } else if (flag = MPIIO) { - int mpiio_file = read_int(); - if (mpiio == 0 && mpiio_file) + int mpiioflag_file = read_int(); + if (mpiioflag == 0 && mpiioflag_file) error->all(FLERR,"Restart file is a MPI-IO file"); - if (mpiio && mpiio_file == 0) + if (mpiioflag && mpiioflag_file == 0) error->all(FLERR,"Restart file is not a MPI-IO file"); } + // NOTE: could add reading of MPI-IO specific fields to header here + // e.g. read vector of PERPROCSIZE values + flag = read_int(); } } diff --git a/src/read_restart.h b/src/read_restart.h index b96516ab6d..e074d46f71 100644 --- a/src/read_restart.h +++ b/src/read_restart.h @@ -35,10 +35,12 @@ class ReadRestart : protected Pointers { FILE *fp; int nfix_restart_global,nfix_restart_peratom; - int mpiio; // 1 for MPIIO output, else 0 int multiproc; // 0 = proc 0 writes for all // else # of procs writing files + int mpiioflag; // 1 for MPIIO output, else 0 + class RestartMPIIO *mpiio; // MPIIO for restart file input + void file_search(char *, char *); void header(int); void type_arrays(); diff --git a/src/write_restart.cpp b/src/write_restart.cpp index 78821e377c..f5dc666dce 100644 --- a/src/write_restart.cpp +++ b/src/write_restart.cpp @@ -34,6 +34,7 @@ #include "comm.h" #include "output.h" #include "thermo.h" +#include "mpiio.h" #include "memory.h" #include "error.h" @@ -96,17 +97,24 @@ void WriteRestart::command(int narg, char **arg) if (strchr(arg[0],'%')) multiproc = nprocs; else multiproc = 0; - if (strstr(arg[0],".mpi")) mpiio = 1; - else mpiio = 0; + if (strstr(arg[0],".mpi")) mpiioflag = 1; + else mpiioflag = 0; - if (multiproc && mpiio) + if (multiproc && mpiioflag) error->all(FLERR, "Write restart MPI-IO output not allowed with '%' in filename"); + if (mpiioflag) { + mpiio = new RestartMPIIO(lmp); + if (!mpiio->mpiio_exists) + error->all(FLERR,"Writing to MPI-IO filename when " + "MPIIO package is not installed"); + } + // setup output style and process optional args // also called by Output class for periodic restart files - multiproc_options(multiproc,mpiio,narg-1,&arg[1]); + multiproc_options(multiproc,mpiioflag,narg-1,&arg[1]); // init entire system since comm->exchange is done // comm::init needs neighbor::init needs pair::init needs kspace::init, etc @@ -136,11 +144,11 @@ void WriteRestart::command(int narg, char **arg) /* ---------------------------------------------------------------------- ------------------------------------------------------------------------- */ -void WriteRestart::multiproc_options(int multiproc_caller, int mpiio_caller, +void WriteRestart::multiproc_options(int multiproc_caller, int mpiioflag_caller, int narg, char **arg) { multiproc = multiproc_caller; - mpiio = mpiio_caller; + mpiioflag = mpiioflag_caller; // defaults for multiproc file writing @@ -363,33 +371,50 @@ void WriteRestart::write(char *file) } } + // MPI-IO output to single file + + if (mpiioflag) { + // add calls to RestartMPIIO class + // reopen header file in append mode + // perform writes + + // mpiio->open(file); + // mpiio->write(send_size,buf); + // mpiio->close(); + } + + // output of one or more native files // filewriter = 1 = this proc writes to file // ping each proc in my cluster, receive its data, write data to file // else wait for ping from fileproc, send my data to fileproc - int tmp,recv_size; - MPI_Status status; - MPI_Request request; + else { + int tmp,recv_size; + MPI_Status status; + MPI_Request request; - if (filewriter) { - write_int(PROCSPERFILE,nclusterprocs); - for (int iproc = 0; iproc < nclusterprocs; iproc++) { - if (iproc) { - MPI_Irecv(buf,max_size,MPI_DOUBLE,me+iproc,0,world,&request); - MPI_Send(&tmp,0,MPI_INT,me+iproc,0,world); - MPI_Wait(&request,&status); - MPI_Get_count(&status,MPI_DOUBLE,&recv_size); - } else recv_size = send_size; + if (filewriter) { + write_int(PROCSPERFILE,nclusterprocs); + for (int iproc = 0; iproc < nclusterprocs; iproc++) { + if (iproc) { + MPI_Irecv(buf,max_size,MPI_DOUBLE,me+iproc,0,world,&request); + MPI_Send(&tmp,0,MPI_INT,me+iproc,0,world); + MPI_Wait(&request,&status); + MPI_Get_count(&status,MPI_DOUBLE,&recv_size); + } else recv_size = send_size; + + write_double_vec(PERPROC,recv_size,buf); + } + fclose(fp); - write_double_vec(PERPROC,recv_size,buf); + } else { + MPI_Recv(&tmp,0,MPI_INT,fileproc,0,world,&status); + MPI_Rsend(buf,send_size,MPI_DOUBLE,fileproc,0,world); } - fclose(fp); - - } else { - MPI_Recv(&tmp,0,MPI_INT,fileproc,0,world,&status); - MPI_Rsend(buf,send_size,MPI_DOUBLE,fileproc,0,world); } + // clean up + memory->destroy(buf); // invoke any fixes that write their own restart file @@ -528,9 +553,12 @@ void WriteRestart::file_layout(int send_size) { if (me == 0) { write_int(MULTIPROC,multiproc); - write_int(MPIIO,mpiio); + write_int(MPIIO,mpiioflag); } + // NOTE: could add MPI-IO specific fields to header here + // e.g. gather send_size across all procs and call write_int_vec() + // -1 flag signals end of file layout info if (me == 0) { diff --git a/src/write_restart.h b/src/write_restart.h index d658f1d722..be8f716cd5 100644 --- a/src/write_restart.h +++ b/src/write_restart.h @@ -37,7 +37,6 @@ class WriteRestart : protected Pointers { FILE *fp; bigint natoms; // natoms (sum of nlocal) to write into file - int mpiio; // 1 for MPIIO output, else 0 int multiproc; // 0 = proc 0 writes for all // else # of procs writing files int nclusterprocs; // # of procs in my cluster that write to one file @@ -45,6 +44,9 @@ class WriteRestart : protected Pointers { int fileproc; // ID of proc in my cluster who writes to file int icluster; // which cluster I am in + int mpiioflag; // 1 for MPIIO output, else 0 + class RestartMPIIO *mpiio; // MPIIO for restart file output + void header(); void type_arrays(); void force_fields();