ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make read_restart evenly divide the work of reading when using mpiio.

Browse Source 
Currently only affects restart files written without any per-atom fix data.
This commit is contained in:
Tim Mattox
2017-03-27 14:53:48 -05:00
parent 0463923e33
commit 661bd37e15
1 changed files with 69 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/read_restart.cpp
View File

@ -212,7 +212,8 @@ void ReadRestart::command(int narg, char **arg)
int perAtomSize = avec->size_restart(); // ...so we can get its size
atom->nlocal = 0; // restore nlocal to zero atoms
int atomCt = (int) (assignedChunkSize / perAtomSize);
#ifdef DEBUG_ME_NOTNOW
//#define DEBUG_PRE_GROW
#ifdef DEBUG_PRE_GROW
fprintf(stdout, "ReadRestart::command %04d: pAS %d, aCt %d, nmax %d, chunckSize %12.0f, %12.0f\n"
,me
,perAtomSize
@ -1026,6 +1027,7 @@ void ReadRestart::file_layout()
// if the number of ranks that did the writing is different
if (me == 0) {
int ndx;
int *all_written_send_sizes;
memory->create(all_written_send_sizes,nprocs_file,
"write_restart:all_written_send_sizes");
@ -1035,30 +1037,76 @@ void ReadRestart::file_layout()
fread(all_written_send_sizes,sizeof(int),nprocs_file,fp);
int init_chunk_number = nprocs_file/nprocs;
int num_extra_chunks = nprocs_file - (nprocs*init_chunk_number);
if ((nprocs != nprocs_file) && !(atom->nextra_store)) {
// nprocs differ, but atom sizes are fixed length, yeah!
atom->nlocal = 1; // temporarily claim there is one atom...
int perAtomSize = atom->avec->size_restart(); // ...so we can get its size
atom->nlocal = 0; // restore nlocal to zero atoms
for (int i = 0; i < nprocs; i++) {
if (i < num_extra_chunks)
nproc_chunk_number[i] = init_chunk_number+1;
else
nproc_chunk_number[i] = init_chunk_number;
}
bigint total_size = 0;
for (int i = 0; i < nprocs_file; ++i) {
total_size += all_written_send_sizes[i];
}
bigint total_ct = total_size / perAtomSize;
int all_written_send_sizes_index = 0;
bigint current_offset = 0;
for (int i=0;i<nprocs;i++) {
nproc_chunk_offsets[i] = current_offset;
nproc_chunk_sizes[i] = 0;
for (int j=0;j<nproc_chunk_number[i];j++) {
nproc_chunk_sizes[i] +=
all_written_send_sizes[all_written_send_sizes_index];
current_offset +=
(all_written_send_sizes[all_written_send_sizes_index] *
sizeof(double));
all_written_send_sizes_index++;
bigint base_ct = total_ct / nprocs;
bigint leftover_ct = total_ct - (base_ct * nprocs);
bigint current_ByteOffset = 0;
base_ct += 1;
bigint base_ByteOffset = base_ct * (perAtomSize * sizeof(double));
for (ndx = 0; ndx < leftover_ct; ++ndx) {
nproc_chunk_offsets[ndx] = current_ByteOffset;
nproc_chunk_sizes[ndx] = base_ct * perAtomSize;
current_ByteOffset += base_ByteOffset;
}
base_ct -= 1;
base_ByteOffset -= (perAtomSize * sizeof(double));
for (; ndx < nprocs; ++ndx) {
nproc_chunk_offsets[ndx] = current_ByteOffset;
nproc_chunk_sizes[ndx] = base_ct * perAtomSize;
current_ByteOffset += base_ByteOffset;
}
//#define DEBUG_FILE_LAYOUT
#ifdef DEBUG_FILE_LAYOUT
fprintf(stdout, "ReadRestart::file_layout: %15.0f/%d = %15.0f totCt, %15.0f natoms, %12.0f baseCt, %12.0f leftover, %d np != %d npf %c%c\n"
,(double) total_size
,perAtomSize
,(double) total_ct
,(double) atom->natoms
,(double) base_ct
,(double) leftover_ct
,nprocs
,nprocs_file
,(total_size == (total_ct * perAtomSize)) ? ' ' : 'E'
,(total_ct == (base_ct * nprocs + leftover_ct)) ? ' ' : 'F'
);
#endif
} else { // Bummer, we have to read in based on how it was written
int init_chunk_number = nprocs_file/nprocs;
int num_extra_chunks = nprocs_file - (nprocs*init_chunk_number);
for (int i = 0; i < nprocs; i++) {
if (i < num_extra_chunks)
nproc_chunk_number[i] = init_chunk_number+1;
else
nproc_chunk_number[i] = init_chunk_number;
}
int all_written_send_sizes_index = 0;
bigint current_offset = 0;
for (int i=0;i<nprocs;i++) {
nproc_chunk_offsets[i] = current_offset;
nproc_chunk_sizes[i] = 0;
for (int j=0;j<nproc_chunk_number[i];j++) {
nproc_chunk_sizes[i] +=
all_written_send_sizes[all_written_send_sizes_index];
current_offset +=
(all_written_send_sizes[all_written_send_sizes_index] *
sizeof(double));
all_written_send_sizes_index++;
}
}
}
memory->destroy(all_written_send_sizes);
memory->destroy(nproc_chunk_number);