diff --git a/src/dump.cpp b/src/dump.cpp index 7c7f2cd3a2..2947b4c3fb 100644 --- a/src/dump.cpp +++ b/src/dump.cpp @@ -90,19 +90,25 @@ Dump::Dump(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) compressed = 0; binary = 0; multifile = 0; + multiproc = 0; + nclusterprocs = nprocs; + filewriter = 0; + if (me == 0) filewriter = 1; + fileproc = 0; + multiname = NULL; char *ptr; if (ptr = strchr(filename,'%')) { multiproc = 1; - char *extend = new char[strlen(filename) + 16]; + nclusterprocs = 1; + filewriter = 1; + fileproc = me; + MPI_Comm_split(world,me,0,&dumpcomm); + multiname = new char[strlen(filename) + 16]; *ptr = '\0'; - sprintf(extend,"%s%d%s",filename,me,ptr+1); - delete [] filename; - n = strlen(extend) + 1; - filename = new char[n]; - strcpy(filename,extend); - delete [] extend; + sprintf(multiname,"%s%d%s",filename,me,ptr+1); + *ptr = '%'; } if (strchr(filename,'*')) multifile = 1; @@ -120,6 +126,7 @@ Dump::~Dump() delete [] id; delete [] style; delete [] filename; + delete [] multiname; delete [] format; delete [] format_default; @@ -133,15 +140,15 @@ Dump::~Dump() memory->destroy(proclist); delete irregular; + if (multiproc) MPI_Comm_free(&dumpcomm); + // XTC style sets fp to NULL since it closes file in its destructor if (multifile == 0 && fp != NULL) { if (compressed) { - if (multiproc) pclose(fp); - else if (me == 0) pclose(fp); + if (filewriter) pclose(fp); } else { - if (multiproc) fclose(fp); - else if (me == 0) fclose(fp); + if (filewriter) fclose(fp); } } } @@ -167,6 +174,8 @@ void Dump::init() } if (sort_flag) { + if (multiproc > 1) + error->all(FLERR,"Cannot dump sort when multiple procs write the dump file"); if (sortcol == 0 && atom->tag_enable == 0) error->all(FLERR,"Cannot dump sort on atom IDs with no atom IDs defined"); if (sortcol && sortcol > size_one) @@ -266,27 +275,30 @@ void Dump::write() boxyz = domain->yz; } - // nme = # of dump lines this proc will contribute to dump + // nme = # of dump lines this proc contributes to dump nme = count(); - bigint bnme = nme; - // ntotal = total # of dump lines + // ntotal = total # of dump lines in snapshot // nmax = max # of dump lines on any proc + bigint bnme = nme; + MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); + int nmax; - if (multiproc) nmax = nme; - else { - MPI_Allreduce(&bnme,&ntotal,1,MPI_LMP_BIGINT,MPI_SUM,world); - MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); - } + if (multiproc != nprocs) MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); + else nmax = nme; // write timestep header + // for multiproc, nheader = # of lines in this file via Allreduce on dumpcomm - if (multiproc) write_header(bnme); - else write_header(ntotal); + bigint nheader = ntotal; + if (multiproc) + MPI_Allreduce(&bnme,&nheader,1,MPI_LMP_BIGINT,MPI_SUM,dumpcomm); - // insure proc 0 can receive everyone's info + if (filewriter) write_header(nheader); + + // insure filewriter proc can receive everyone's info // limit nmax*size_one to int since used as arg in MPI_Rsend() below // pack my data into buf // if sorting on IDs also request ID list from pack() @@ -309,46 +321,40 @@ void Dump::write() else pack(NULL); if (sort_flag) sort(); - // multiproc = 1 = each proc writes own data to own file - // multiproc = 0 = all procs write to one file thru proc 0 - // proc 0 pings each proc, receives it's data, writes to file - // all other procs wait for ping, send their data to proc 0 + // filewriter = 1 = this proc writes to file + // ping each proc in my cluster, receive its data, write data to file + // else wait for ping from fileproc, send my data to fileproc - if (multiproc) write_data(nme,buf); - else { - int tmp,nlines; - MPI_Status status; - MPI_Request request; + int tmp,nlines; + MPI_Status status; + MPI_Request request; - if (me == 0) { - for (int iproc = 0; iproc < nprocs; iproc++) { - if (iproc) { - MPI_Irecv(buf,maxbuf*size_one,MPI_DOUBLE,iproc,0,world,&request); - MPI_Send(&tmp,0,MPI_INT,iproc,0,world); - MPI_Wait(&request,&status); - MPI_Get_count(&status,MPI_DOUBLE,&nlines); - nlines /= size_one; - } else nlines = nme; - - write_data(nlines,buf); - } - if (flush_flag) fflush(fp); - - } else { - MPI_Recv(&tmp,0,MPI_INT,0,0,world,&status); - MPI_Rsend(buf,nme*size_one,MPI_DOUBLE,0,0,world); + if (filewriter) { + for (int iproc = 0; iproc < nclusterprocs; iproc++) { + if (iproc) { + MPI_Irecv(buf,maxbuf*size_one,MPI_DOUBLE,me+iproc,0,world,&request); + MPI_Send(&tmp,0,MPI_INT,me+iproc,0,world); + MPI_Wait(&request,&status); + MPI_Get_count(&status,MPI_DOUBLE,&nlines); + nlines /= size_one; + } else nlines = nme; + + write_data(nlines,buf); } + if (flush_flag) fflush(fp); + + } else { + MPI_Recv(&tmp,0,MPI_INT,fileproc,0,world,&status); + MPI_Rsend(buf,nme*size_one,MPI_DOUBLE,fileproc,0,world); } - // if file per timestep, close file + // if file per timestep, close file if I am filewriter if (multifile) { if (compressed) { - if (multiproc) pclose(fp); - else if (me == 0) pclose(fp); + if (filewriter) pclose(fp); } else { - if (multiproc) fclose(fp); - else if (me == 0) fclose(fp); + if (filewriter) fclose(fp); } } } @@ -368,27 +374,29 @@ void Dump::openfile() // if one file per timestep, replace '*' with current timestep - char *filecurrent; - if (multifile == 0) filecurrent = filename; - else { - filecurrent = new char[strlen(filename) + 16]; - char *ptr = strchr(filename,'*'); + char *filecurrent = filename; + if (multiproc) filecurrent = multiname; + + if (multifile) { + char *filestar = filecurrent; + filecurrent = new char[strlen(filestar) + 16]; + char *ptr = strchr(filestar,'*'); *ptr = '\0'; if (padflag == 0) sprintf(filecurrent,"%s" BIGINT_FORMAT "%s", - filename,update->ntimestep,ptr+1); + filestar,update->ntimestep,ptr+1); else { char bif[8],pad[16]; strcpy(bif,BIGINT_FORMAT); sprintf(pad,"%%s%%0%d%s%%s",padflag,&bif[1]); - sprintf(filecurrent,pad,filename,update->ntimestep,ptr+1); + sprintf(filecurrent,pad,filestar,update->ntimestep,ptr+1); } *ptr = '*'; } - // open one file on proc 0 or file on every proc + // each proc with filewriter = 1 opens a file - if (me == 0 || multiproc) { + if (filewriter) { if (compressed) { #ifdef LAMMPS_GZIP char gzip[128]; @@ -552,8 +560,7 @@ void Dump::sort() // this insures proc 0 can receive everyone's info int nmax; - if (multiproc) nmax = nme; - else MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); + MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,world); if (nmax > maxbuf) { maxbuf = nmax; @@ -669,6 +676,36 @@ void Dump::modify_params(int narg, char **arg) else if (strcmp(arg[iarg+1],"no") == 0) first_flag = 0; else error->all(FLERR,"Illegal dump_modify command"); iarg += 2; + + } else if (strcmp(arg[iarg],"fileper") == 0) { + if (iarg+2 > narg) error->all(FLERR,"Illegal dump_modify command"); + if (!multiproc) + error->all(FLERR,"Cannot use dump_modify fileper " + "without % in dump file name"); + int nper = atoi(arg[iarg+1]); + if (nper <= 0) error->all(FLERR,"Illegal dump_modify command"); + + multiproc = nprocs/nper; + if (nprocs % nper) multiproc++; + fileproc = me/nper * nper; + int fileprocnext = MIN(fileproc+nper,nprocs); + nclusterprocs = fileprocnext - fileproc; + if (me == fileproc) filewriter = 1; + else filewriter = 0; + int icluster = fileproc/nper; + + MPI_Comm_free(&dumpcomm); + MPI_Comm_split(world,icluster,0,&dumpcomm); + + delete [] multiname; + multiname = new char[strlen(filename) + 16]; + char *ptr = strchr(filename,'%'); + *ptr = '\0'; + sprintf(multiname,"%s%d%s",filename,icluster,ptr+1); + *ptr = '%'; + iarg += 2; + + } else if (strcmp(arg[iarg],"flush") == 0) { if (iarg+2 > narg) error->all(FLERR,"Illegal dump_modify command"); if (strcmp(arg[iarg+1],"yes") == 0) flush_flag = 1; @@ -685,6 +722,40 @@ void Dump::modify_params(int narg, char **arg) strcpy(format_user,arg[iarg+1]); } iarg += 2; + + } else if (strcmp(arg[iarg],"nfile") == 0) { + if (iarg+2 > narg) error->all(FLERR,"Illegal dump_modify command"); + if (!multiproc) + error->all(FLERR,"Cannot use dump_modify nfile " + "without % in dump file name"); + int nfile = atoi(arg[iarg+1]); + if (nfile <= 0) error->all(FLERR,"Illegal dump_modify command"); + nfile = MIN(nfile,nprocs); + + multiproc = nfile; + int icluster = static_cast ((bigint) me * nfile/nprocs); + fileproc = static_cast ((bigint) icluster * nprocs/nfile); + int fcluster = static_cast ((bigint) fileproc * nfile/nprocs); + if (fcluster < icluster) fileproc++; + int fileprocnext = + static_cast ((bigint) (icluster+1) * nprocs/nfile); + fcluster = static_cast ((bigint) fileprocnext * nfile/nprocs); + if (fcluster < icluster+1) fileprocnext++; + nclusterprocs = fileprocnext - fileproc; + if (me == fileproc) filewriter = 1; + else filewriter = 0; + + MPI_Comm_free(&dumpcomm); + MPI_Comm_split(world,icluster,0,&dumpcomm); + + delete [] multiname; + multiname = new char[strlen(filename) + 16]; + char *ptr = strchr(filename,'%'); + *ptr = '\0'; + sprintf(multiname,"%s%d%s",filename,icluster,ptr+1); + *ptr = '%'; + iarg += 2; + } else if (strcmp(arg[iarg],"pad") == 0) { if (iarg+2 > narg) error->all(FLERR,"Illegal dump_modify command"); padflag = atoi(arg[iarg+1]); diff --git a/src/dump.h b/src/dump.h index b54705aa93..06b331d113 100644 --- a/src/dump.h +++ b/src/dump.h @@ -14,6 +14,7 @@ #ifndef LMP_DUMP_H #define LMP_DUMP_H +#include "mpi.h" #include "stdio.h" #include "pointers.h" @@ -55,7 +56,14 @@ class Dump : protected Pointers { int compressed; // 1 if dump file is written compressed, 0 no int binary; // 1 if dump file is written binary, 0 no int multifile; // 0 = one big file, 1 = one file per timestep - int multiproc; // 0 = proc 0 writes for all, 1 = one file/proc + + int multiproc; // 0 = proc 0 writes for all, + // else # of procs writing files + int nclusterprocs; // # of procs in my cluster that write to one file + int filewriter; // 1 if this proc writes a file, else 0 + int fileproc; // ID of proc in my cluster who writes to file + char *multiname; // dump filename with % converted to cluster ID + MPI_Comm dumpcomm; int header_flag; // 0 = item, 2 = xyz int flush_flag; // 0 if no flush, 1 if flush every dump @@ -80,7 +88,7 @@ class Dump : protected Pointers { double boxzlo,boxzhi; double boxxy,boxxz,boxyz; - bigint ntotal; // # of per-atom lines in snapshot + bigint ntotal; // total # of per-atom lines in snapshot int reorderflag; // 1 if OK to reorder instead of sort int ntotal_reorder; // # of atoms that must be in snapshot int nme_reorder; // # of atoms I must own in snapshot