git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@8604 f3b2605a-c512-4ea7-a41b-209d697bcdaa

This commit is contained in:
sjplimp
2012-08-11 19:05:13 +00:00
parent ee7f12334d
commit 58443251c3
65 changed files with 112430 additions and 0 deletions

View File

@ -0,0 +1,42 @@
# Makefile for coupling library
SHELL = /bin/sh
# System-specific settings
CC = g++
CCFLAGS = -g -O -DMPICH_IGNORE_CXX_SEEK
DEPFLAGS = -M
LINK = g++
LINKFLAGS = -g -O
ARCHIVE = ar
ARFLAGS = -rc
SIZE = size
# Files
LIB = libcouple.a
SRC = $(wildcard *.cpp)
INC = $(wildcard *.h)
OBJ = $(SRC:.cpp=.o)
# Targets
lib: $(OBJ)
$(ARCHIVE) $(ARFLAGS) $(LIB) $(OBJ)
clean:
rm $(LIB) *.o *.d
# Compilation rules
%.o:%.cpp
$(CC) $(CCFLAGS) -c $<
%.d:%.cpp
$(CC) $(CCFLAGS) $(DEPFLAGS) $< > $@
# Individual dependencies
DEPENDS = $(OBJ:.o=.d)
include $(DEPENDS)

View File

@ -0,0 +1,16 @@
This directory has a small collection of routines, useful for
exchanging data between 2 codes being run together as a coupled
application. It is used by the LAMMPS <-> Quest and LAMMPS <->
SPPARKS applications in 2 sister directories.
The library dir has a Makefile (which you may need to edit for your
box). If you type
g++ -f Makefile.g++
you should create libcouple.a, which the other coupled applications
link to.
Note that the library uses MPI, so the Makefile you use needs to
include a path to the MPI include file, if it is not someplace
the compiler will find it.

View File

@ -0,0 +1,42 @@
#include "mpi.h"
#include "stdlib.h"
#include "stdio.h"
#include "error.h"
/* ---------------------------------------------------------------------- */
Error::Error(MPI_Comm caller)
{
comm = caller;
MPI_Comm_rank(comm,&me);
}
/* ----------------------------------------------------------------------
called by all procs
------------------------------------------------------------------------- */
void Error::all(const char *str)
{
if (me == 0) printf("ERROR: %s\n",str);
MPI_Finalize();
exit(1);
}
/* ----------------------------------------------------------------------
called by one proc
------------------------------------------------------------------------- */
void Error::one(const char *str)
{
printf("ERROR on proc %d: %s\n",me,str);
MPI_Abort(comm,1);
}
/* ----------------------------------------------------------------------
called by one proc
------------------------------------------------------------------------- */
void Error::warning(const char *str)
{
printf("WARNING: %s\n",str);
}

View File

@ -0,0 +1,19 @@
error.o: error.cpp /usr/local/include/mpi.h /usr/local/include/mpio.h \
/usr/local/include/mpi.h /usr/local/include/mpicxx.h \
/usr/include/stdlib.h /usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/bits/types.h \
/usr/include/bits/wordsize.h /usr/include/bits/typesizes.h \
/usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h /usr/include/stdio.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h error.h

View File

@ -0,0 +1,19 @@
#ifndef ERROR_H
#define ERROR_H
#include "mpi.h"
class Error {
public:
Error(MPI_Comm);
void all(const char *);
void one(const char *);
void warning(const char *);
private:
MPI_Comm comm;
int me;
};
#endif

View File

@ -0,0 +1,48 @@
#include "stdio.h"
#include "string.h"
#include "files.h"
#define MAXLINE 256
/* ---------------------------------------------------------------------- */
void replace(char *file, char *header, int n, char **lines)
{
FILE *fpr = fopen(file,"r");
FILE *fpw = fopen("tmp.file","w");
char line[MAXLINE];
while (fgets(line,MAXLINE,fpr)) {
if (strstr(line,header)) {
fprintf(fpw,"%s",line);
for (int i = 0; i < n; i++) {
fgets(line,MAXLINE,fpr);
fprintf(fpw,"%s",lines[i]);
}
} else fprintf(fpw,"%s",line);
}
fclose(fpr);
fclose(fpw);
rename("tmp.file",file);
}
/* ---------------------------------------------------------------------- */
char **extract(char *file, char *header, int n, char **lines)
{
FILE *fp = fopen(file,"r");
char line[MAXLINE];
while (fgets(line,MAXLINE,fp)) {
if (strstr(line,header)) {
for (int i = 0; i < n; i++) {
fgets(line,MAXLINE,fp);
sprintf(lines[i],"%s",line);
}
break;
}
}
fclose(fp);
}

View File

@ -0,0 +1,11 @@
files.o: files.cpp /usr/include/stdio.h /usr/include/features.h \
/usr/include/sys/cdefs.h /usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/types.h /usr/include/bits/wordsize.h \
/usr/include/bits/typesizes.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h /usr/include/string.h /usr/include/xlocale.h \
files.h

View File

@ -0,0 +1,2 @@
void replace(char *, char *, int, char **);
char **extract(char *, char *, int, char **);

View File

@ -0,0 +1,393 @@
#include "stdio.h"
#include "stdlib.h"
#include "string.h"
#include "irregular.h"
#include "memory.h"
#include "error.h"
#define MAX(A,B) ((A) > (B)) ? (A) : (B)
enum{UNSET,SET};
enum{NONE,SAME,VARYING};
/* ---------------------------------------------------------------------- */
Irregular::Irregular(MPI_Comm caller)
{
comm = caller;
MPI_Comm_rank(comm,&me);
MPI_Comm_size(comm,&nprocs);
memory = new Memory(comm);
error = new Error(comm);
init();
patternflag = UNSET;
sizestyle = NONE;
}
/* ---------------------------------------------------------------------- */
Irregular::~Irregular()
{
delete memory;
delete error;
deallocate();
}
/* ----------------------------------------------------------------------
n = # of datums contributed by this proc
proclist = which proc each datum is to be sent to
------------------------------------------------------------------------- */
void Irregular::pattern(int n, int *proclist)
{
// free any previous irregular info
deallocate();
init();
patternflag = SET;
sizestyle = NONE;
ndatumsend = n;
// list = 1 for procs I send to, including self
// nrecv = # of messages I receive, not including self
// self = 0 if no data for self, 1 if there is
int *list = new int[nprocs];
int *counts = new int[nprocs];
for (int i = 0; i < nprocs; i++) {
list[i] = 0;
counts[i] = 1;
}
for (int i = 0; i < n; i++) list[proclist[i]] = 1;
MPI_Reduce_scatter(list,&nrecv,counts,MPI_INT,MPI_SUM,comm);
self = 0;
if (list[me]) self = 1;
if (self) nrecv--;
// storage for recv info, not including self
recvproc = new int[nrecv];
recvcount = new int[nrecv];
recvsize = new int[nrecv];
request = new MPI_Request[nrecv];
status = new MPI_Status[nrecv];
// list = # of datums to send to each proc, including self
// nsend = # of messages I send, not including self
for (int i = 0; i < nprocs; i++) list[i] = 0;
for (int i = 0; i < n; i++) list[proclist[i]]++;
nsend = 0;
for (int i = 0; i < nprocs; i++) if (list[i] > 0) nsend++;
if (self) nsend--;
// storage for send info, including self
sendproc = new int[nsend+self];
sendcount = new int[nsend+self];
sendsize = new int[nsend+self];
sendindices = (int *) memory->smalloc(n*sizeof(int),"sendindices");
// setup sendprocs and sendcounts, including self
// each proc begins with iproc > me, and continues until iproc = me
// list ends up with pointer to which send that proc is associated with
int iproc = me;
int isend = 0;
for (int i = 0; i < nprocs; i++) {
iproc++;
if (iproc == nprocs) iproc = 0;
if (list[iproc] > 0) {
sendproc[isend] = iproc;
sendcount[isend] = list[iproc];
list[iproc] = isend;
isend++;
}
}
// post all receives for datum counts
for (int i = 0; i < nrecv; i++)
MPI_Irecv(&recvcount[i],1,MPI_INT,MPI_ANY_SOURCE,0,comm,&request[i]);
// barrier to insure receives are posted
MPI_Barrier(comm);
// send each datum count, packing buf with needed datums
for (int i = 0; i < nsend; i++)
MPI_Send(&sendcount[i],1,MPI_INT,sendproc[i],0,comm);
// insure all MPI_ANY_SOURCE messages are received
// set recvproc
if (nrecv) MPI_Waitall(nrecv,request,status);
for (int i = 0; i < nrecv; i++) recvproc[i] = status[i].MPI_SOURCE;
// ndatumrecv = total datums received, including self
ndatumrecv = 0;
for (int i = 0; i < nrecv; i++)
ndatumrecv += recvcount[i];
if (self) ndatumrecv += sendcount[nsend];
// setup sendindices, including self
// counts = offset into sendindices for each proc I send to
// let sc0 = sendcount[0], sc1 = sendcount[1], etc
// sendindices[0:sc0-1] = indices of datums in 1st message
// sendindices[sc0:sc0+sc1-1] = indices of datums in 2nd message, etc
counts[0] = 0;
for (int i = 1; i < nsend+self; i++)
counts[i] = counts[i-1] + sendcount[i-1];
for (int i = 0; i < n; i++) {
isend = list[proclist[i]];
sendindices[counts[isend]++] = i;
}
// clean up
delete [] counts;
delete [] list;
}
/* ----------------------------------------------------------------------
n = size of each received datum
return total size in bytes of received data on this proc
------------------------------------------------------------------------- */
int Irregular::size(int n)
{
if (patternflag == UNSET) error->all("Cannot size without pattern");
sizestyle = SAME;
nsize = n;
nsendmax = 0;
for (int i = 0; i < nsend+self; i++) {
sendsize[i] = nsize * sendcount[i];
if (i < nsend) nsendmax = MAX(nsendmax,sendsize[i]);
}
for (int i = 0; i < nrecv; i++) recvsize[i] = nsize * recvcount[i];
nbytesrecv = nsize * ndatumrecv;
return nbytesrecv;
}
/* ----------------------------------------------------------------------
slength,rlength = size of each datum to send and recv
soffset = offset into eventual buffer of send data for each datum
soffset can be NULL, in which case will build sendoffset from slength
return total size in bytes of received data on this proc
------------------------------------------------------------------------- */
int Irregular::size(int *slength, int *soffset, int *rlength)
{
if (patternflag == UNSET) error->all("Cannot size without pattern");
sizestyle = VARYING;
// store local copy of pointers to send lengths/offsets
// if soffset not provided, create local copy from slength
sendsizedatum = slength;
if (soffset == NULL) {
sendoffsetflag = 1;
sendoffset = (int *) memory->smalloc(ndatumsend*sizeof(int),"sendoffset");
if (ndatumsend) sendoffset[0] = 0;
for (int i = 1; i < ndatumsend; i++)
sendoffset[i] = sendoffset[i-1] + sendsizedatum[i-1];
} else {
if (sendoffsetflag) memory->sfree(sendoffset);
sendoffsetflag = 0;
sendoffset = soffset;
}
nsendmax = 0;
int m = 0;
for (int i = 0; i < nsend+self; i++) {
sendsize[i] = 0;
for (int j = 0; j < sendcount[i]; j++)
sendsize[i] += sendsizedatum[sendindices[m++]];
if (i < nsend) nsendmax = MAX(nsendmax,sendsize[i]);
}
nbytesrecv = 0;
m = 0;
for (int i = 0; i < nrecv; i++) {
recvsize[i] = 0;
for (int j = 0; j < recvcount[i]; j++) recvsize[i] += rlength[m++];
nbytesrecv += recvsize[i];
}
if (self) nbytesrecv += sendsize[nsend];
return nbytesrecv;
}
/* ----------------------------------------------------------------------
wrapper on 2 versions of exchange
------------------------------------------------------------------------- */
void Irregular::exchange(char *sendbuf, char *recvbuf)
{
if (sizestyle == SAME) exchange_same(sendbuf,recvbuf);
else if (sizestyle == VARYING) exchange_varying(sendbuf,recvbuf);
else error->all("Irregular size was not set");
}
/* ----------------------------------------------------------------------
sendbuf = data to send
recvbuf = buffer to recv all data into
requires nsize,nsendmax,recvsize,sendsize be setup by size(int)
------------------------------------------------------------------------- */
void Irregular::exchange_same(char *sendbuf, char *recvbuf)
{
// post all receives
int recvoffset = 0;
for (int irecv = 0; irecv < nrecv; irecv++) {
MPI_Irecv(&recvbuf[recvoffset],recvsize[irecv],MPI_BYTE,
recvproc[irecv],0,comm,&request[irecv]);
recvoffset += recvsize[irecv];
}
// malloc buf for largest send
char *buf = (char *) memory->smalloc(nsendmax,"buf");
// barrier to insure receives are posted
MPI_Barrier(comm);
// send each message, packing buf with needed datums
int m = 0;
for (int isend = 0; isend < nsend; isend++) {
int bufoffset = 0;
for (int i = 0; i < sendcount[isend]; i++) {
memcpy(&buf[bufoffset],&sendbuf[nsize*sendindices[m++]],nsize);
bufoffset += nsize;
}
MPI_Send(buf,sendsize[isend],MPI_BYTE,sendproc[isend],0,comm);
}
// copy self data directly from sendbuf to recvbuf
if (self)
for (int i = 0; i < sendcount[nsend]; i++) {
memcpy(&recvbuf[recvoffset],&sendbuf[nsize*sendindices[m++]],nsize);
recvoffset += nsize;
}
// free send buffer
memory->sfree(buf);
// wait on all incoming messages
if (nrecv) MPI_Waitall(nrecv,request,status);
}
/* ----------------------------------------------------------------------
sendbuf = data to send
recvbuf = buffer to recv all data into
requires nsendmax,recvsize,sendsize,sendoffset,sendsizedatum
be setup by size(int *, int *)
------------------------------------------------------------------------- */
void Irregular::exchange_varying(char *sendbuf, char *recvbuf)
{
// post all receives
int recvoffset = 0;
for (int irecv = 0; irecv < nrecv; irecv++) {
MPI_Irecv(&recvbuf[recvoffset],recvsize[irecv],MPI_BYTE,
recvproc[irecv],0,comm,&request[irecv]);
recvoffset += recvsize[irecv];
}
// malloc buf for largest send
char *buf = (char *) memory->smalloc(nsendmax,"buf");
// barrier to insure receives are posted
MPI_Barrier(comm);
// send each message, packing buf with needed datums
int index;
int m = 0;
for (int isend = 0; isend < nsend; isend++) {
int bufoffset = 0;
for (int i = 0; i < sendcount[isend]; i++) {
index = sendindices[m++];
memcpy(&buf[bufoffset],&sendbuf[sendoffset[index]],sendsizedatum[index]);
bufoffset += sendsizedatum[index];
}
MPI_Send(buf,sendsize[isend],MPI_BYTE,sendproc[isend],0,comm);
}
// copy self data directly from sendbuf to recvbuf
if (self)
for (int i = 0; i < sendcount[nsend]; i++) {
index = sendindices[m++];
memcpy(&recvbuf[recvoffset],&sendbuf[sendoffset[index]],
sendsizedatum[index]);
recvoffset += sendsizedatum[index];
}
// free send buffer
memory->sfree(buf);
// wait on all incoming messages
if (nrecv) MPI_Waitall(nrecv,request,status);
}
/* ---------------------------------------------------------------------- */
void Irregular::init()
{
sendoffsetflag = 0;
sendproc = sendcount = sendsize = sendindices = NULL;
sendoffset = NULL;
recvproc = recvcount = recvsize = NULL;
request = NULL;
status = NULL;
}
/* ---------------------------------------------------------------------- */
void Irregular::deallocate()
{
delete [] sendproc;
delete [] sendcount;
delete [] sendsize;
memory->sfree(sendindices);
if (sendoffsetflag) memory->sfree(sendoffset);
delete [] recvproc;
delete [] recvcount;
delete [] recvsize;
delete [] request;
delete [] status;
}

View File

@ -0,0 +1,19 @@
irregular.o: irregular.cpp /usr/include/stdio.h /usr/include/features.h \
/usr/include/sys/cdefs.h /usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/types.h /usr/include/bits/wordsize.h \
/usr/include/bits/typesizes.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h /usr/include/stdlib.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h /usr/include/string.h irregular.h \
/usr/local/include/mpi.h /usr/local/include/mpio.h \
/usr/local/include/mpi.h /usr/local/include/mpicxx.h memory.h error.h

View File

@ -0,0 +1,58 @@
#ifndef IRREGULAR_H
#define IRREGULAR_H
#include "mpi.h"
class Irregular {
public:
Irregular(MPI_Comm);
~Irregular();
void pattern(int, int *);
int size(int);
int size(int *, int *, int *);
void exchange(char *, char *);
private:
int me,nprocs;
int patternflag; // UNSET,SET
int sizestyle; // NONE,SAME,VARYING
int self; // 0 = no data to copy to self, 1 = yes
int ndatumsend; // # of datums to send w/ self
int ndatumrecv; // # of datums to recv w/ self
int nbytesrecv; // total bytes in received data w/ self
int nsend; // # of messages to send w/out self
int nrecv; // # of messages to recv w/out self
int nsendmax; // # of bytes in largest send message, w/out self
int *sendproc; // list of procs to send to w/out self
int *sendcount; // # of datums to send to each proc w/ self
int *sendsize; // # of bytes to send to each proc w/ self
int *sendindices; // indices of datums to send to each proc w/ self
int nsize; // size of every datum in bytes (SAME)
int *sendsizedatum; // bytes in each datum to send w/ self (VARYING)
int *sendoffset; // byte offset to where each datum starts w/ self
int sendoffsetflag; // 1 if allocated sendoffset, 0 if passed in
int *recvproc; // list of procs to recv from w/out self
int *recvcount; // # of datums to recv from each proc w/out self
int *recvsize; // # of bytes to recv from each proc w/out self
MPI_Request *request; // MPI requests for posted recvs
MPI_Status *status; // MPI statuses for Waitall
MPI_Comm comm; // MPI communicator for all communication
class Memory *memory;
class Error *error;
void exchange_same(char *, char *);
void exchange_varying(char *, char *);
void init();
void deallocate();
};
#endif

View File

@ -0,0 +1,249 @@
#include "stdlib.h"
#include "string.h"
#include "lammps_data_write.h"
#include "memory.h"
#include "error.h"
#define DELTA 4;
enum{INT,DOUBLE,DOUBLE2};
/* ---------------------------------------------------------------------- */
LAMMPSDataWrite::LAMMPSDataWrite(MPI_Comm caller_comm) : Send2One(caller_comm)
{
outfile = NULL;
nheader = maxheader = 0;
format = NULL;
headtype = NULL;
ihead = NULL;
dhead = NULL;
ddhead = NULL;
nper = maxper = 0;
atomtype = NULL;
ivec = NULL;
dvec = NULL;
stride = NULL;
}
/* ---------------------------------------------------------------------- */
LAMMPSDataWrite::~LAMMPSDataWrite()
{
delete [] outfile;
for (int i = 0; i < nheader; i++) delete [] format[i];
memory->sfree(format);
memory->sfree(headtype);
memory->sfree(ihead);
memory->sfree(dhead);
memory->destroy_2d_double_array(ddhead);
memory->sfree(atomtype);
memory->sfree(ivec);
memory->sfree(dvec);
memory->sfree(stride);
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::pre()
{
if (me == 0) {
fp = fopen(outfile,"w");
if (fp == NULL)
error->one("Could not open data_write output file");
}
if (me == 0) {
fprintf(fp,"%s","LAMMPS data file\n\n");
for (int i = 0; i < nheader; i++)
if (headtype[i] == INT) fprintf(fp,format[i],ihead[i]);
else if (headtype[i] == DOUBLE) fprintf(fp,format[i],dhead[i]);
else if (headtype[i] == DOUBLE2) fprintf(fp,format[i],
ddhead[i][0],ddhead[i][1]);
fprintf(fp,"\nAtoms\n\n");
}
}
/* ---------------------------------------------------------------------- */
int LAMMPSDataWrite::size()
{
return nper*nlocal*sizeof(double);
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::pack(char *cbuf)
{
int i,j;
double *dbuf = (double *) cbuf;
int m = 0;
for (i = 0; i < nlocal; i++)
for (j = 0; j < nper; j++) {
if (i == 0) {
if (atomtype[j] == 0)
printf("AT %d %d %p %d\n",
atomtype[j],stride[j],ivec[j],ivec[j][0]);
else
printf("AT %d %d %p %g\n",
atomtype[j],stride[j],dvec[j],dvec[j][0]);
}
if (atomtype[j] == INT) dbuf[m++] = ivec[j][i*stride[j]];
else if (atomtype[j] == DOUBLE) dbuf[m++] = dvec[j][i*stride[j]];
}
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::process(int nbuf, char *cbuf)
{
int i,j;
double *dbuf = (double *) cbuf;
int n = nbuf/nper/sizeof(double);
int m = 0;
for (i = 0; i < n; i++) {
for (j = 0; j < nper; j++) {
double dvalue = dbuf[m++];
if (atomtype[j] == INT) fprintf(fp,"%d ",static_cast<int> (dvalue));
else if (atomtype[j] == DOUBLE) fprintf(fp,"%g ",dvalue);
}
fprintf(fp,"\n");
}
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::post()
{
if (me == 0) fclose(fp);
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::file(char *fname)
{
int n = strlen(fname) + 1;
outfile = new char[n];
strcpy(outfile,fname);
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::header(char *str, int ivalue)
{
if (nheader == maxheader) grow_header();
int n = strlen(str) + 2;
format[nheader] = new char[n];
strcpy(format[nheader],str);
format[nheader][n-2] = '\n';
format[nheader][n-1] = '\0';
headtype[nheader] = INT;
ihead[nheader] = ivalue;
nheader++;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::header(char *str, double dvalue)
{
if (nheader == maxheader) grow_header();
int n = strlen(str) + 2;
format[nheader] = new char[n];
strcpy(format[nheader],str);
format[nheader][n-2] = '\n';
format[nheader][n-1] = '\0';
headtype[nheader] = DOUBLE;
dhead[nheader] = dvalue;
nheader++;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::header(char *str, double dvalue1, double dvalue2)
{
if (nheader == maxheader) grow_header();
int n = strlen(str) + 2;
format[nheader] = new char[n];
strcpy(format[nheader],str);
format[nheader][n-2] = '\n';
format[nheader][n-1] = '\0';
headtype[nheader] = DOUBLE2;
ddhead[nheader][0] = dvalue1;
ddhead[nheader][1] = dvalue2;
nheader++;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::atoms(int n)
{
nlocal = n;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::atoms(int *vec)
{
if (nper == maxper) grow_peratom();
atomtype[nper] = INT;
ivec[nper] = vec;
stride[nper] = 1;
nper++;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::atoms(double *vec)
{
if (nper == maxper) grow_peratom();
atomtype[nper] = DOUBLE;
dvec[nper] = vec;
stride[nper] = 1;
nper++;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::atoms(int n, double **vec)
{
if (nper+n >= maxper) grow_peratom();
for (int i = 0; i < n; i++) {
atomtype[nper] = DOUBLE;
dvec[nper] = &vec[0][i];
stride[nper] = n;
nper++;
}
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::grow_header()
{
int n = maxheader + DELTA;
format = (char **) memory->srealloc(format,n*sizeof(char *),"ldw:format");
headtype = (int *) memory->srealloc(headtype,n*sizeof(int),"ldw:headtype");
ihead = (int *) memory->srealloc(ihead,n*sizeof(int),"ldw:ihead");
dhead = (double *) memory->srealloc(dhead,n*sizeof(double),"ldw:dhead");
ddhead = memory->grow_2d_double_array(ddhead,n,2,"ldw:ddhead");
maxheader = n;
}
/* ---------------------------------------------------------------------- */
void LAMMPSDataWrite::grow_peratom()
{
int n = maxper + DELTA;
atomtype = (int *) memory->srealloc(atomtype,n*sizeof(int *),"ldw:atomtype");
ivec = (int **) memory->srealloc(ivec,n*sizeof(int *),"ldw:ihead");
dvec = (double **) memory->srealloc(dvec,n*sizeof(double *),"ldw:dhead");
stride = (int *) memory->srealloc(stride,n*sizeof(int *),"ldw:stride");
maxper = n;
}

View File

@ -0,0 +1,20 @@
lammps_data_write.o: lammps_data_write.cpp /usr/include/stdlib.h \
/usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/bits/types.h \
/usr/include/bits/wordsize.h /usr/include/bits/typesizes.h \
/usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h /usr/include/string.h lammps_data_write.h \
send2one.h /usr/local/include/mpi.h /usr/local/include/mpio.h \
/usr/local/include/mpi.h /usr/local/include/mpicxx.h \
/usr/include/stdio.h /usr/include/libio.h /usr/include/_G_config.h \
/usr/include/wchar.h /usr/include/bits/wchar.h /usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h memory.h error.h

View File

@ -0,0 +1,48 @@
#ifndef LAMMPS_DATA_WRITE_H
#define LAMMPS_DATA_WRITE_H
#include "send2one.h"
#include "stdio.h"
class LAMMPSDataWrite : public Send2One {
public:
LAMMPSDataWrite(MPI_Comm);
~LAMMPSDataWrite();
void pre();
int size();
void pack(char *);
void process(int, char *);
void post();
void file(char *);
void header(char *, int);
void header(char *, double);
void header(char *, double, double);
void atoms(int);
void atoms(int *);
void atoms(double *);
void atoms(int, double **);
private:
char *outfile;
int nlocal;
FILE *fp;
int nheader,maxheader;
char **format;
int *headtype,*ihead;
double *dhead;
double **ddhead;
int nper,maxper;
int *atomtype;
int **ivec;
double **dvec;
int *stride;
void grow_header();
void grow_peratom();
};
#endif

View File

@ -0,0 +1,316 @@
#include "mpi.h"
#include "many2many.h"
#include "irregular.h"
#include "memory.h"
#include "error.h"
#include <map>
#define MAX(A,B) ((A) > (B)) ? (A) : (B)
/* ---------------------------------------------------------------------- */
Many2Many::Many2Many(MPI_Comm caller)
{
comm = caller;
MPI_Comm_rank(comm,&me);
MPI_Comm_size(comm,&nprocs);
memory = new Memory(comm);
error = new Error(comm);
src_own = dest_own = NULL;
src_off = dest_off = NULL;
src_iwork = dest_iwork = NULL;
src_dwork = dest_dwork = NULL;
irregular = NULL;
}
/* ---------------------------------------------------------------------- */
Many2Many::~Many2Many()
{
delete memory;
delete error;
deallocate();
}
/* ----------------------------------------------------------------------
create a many2many pattern, deallocating any previous pattern
each proc will contribute nsrc items with IDs listed in id_src
each proc will receive ndest items with IDs listed in id_dest
only sets up communication via rendezvous algorithm and Irregular class
if id_src does not match id_dest on all procs
------------------------------------------------------------------------- */
void Many2Many::setup(int nsrc, int *id_src, int ndest, int *id_dest)
{
int i,j,isrc,idest,nsend,nrecv;
int *proclist,*work;
std::map<int,int> hash;
std::map<int,int>::iterator loc;
// free any previous many2many info
deallocate();
// allocate on-proc and off-proc index lists
src_own =
(int *) memory->smalloc(nsrc*sizeof(int),"many2many:src_own");
dest_own =
(int *) memory->smalloc(ndest*sizeof(int),"many2many:dest_own");
src_off =
(int *) memory->smalloc(nsrc*sizeof(int),"many2many:src_off");
dest_off =
(int *) memory->smalloc(ndest*sizeof(int),"many2many:dest_off");
// store destination IDs in hash
for (int i = 0; i < ndest; i++)
hash.insert(std::pair<int,int> (id_dest[i],i));
// src_own, dest_own = list of IDs in both src and dest
// nsrc_off = list of IDs in src owned by other procs
nown = nsrc_off = 0;
nsrc_off = 0;
for (i = 0; i < nsrc; i++) {
loc = hash.find(id_src[i]);
if (loc != hash.end()) {
src_own[nown] = i;
dest_own[nown] = loc->second;
nown++;
} else src_off[nsrc_off++] = i;
}
// all done if all procs have one-to-one mapping of src and dest IDs
// else figure out irregular comm needed
int flag = 0;
if (nown == nsrc && nown == ndest) flag = 1;
int flagall;
MPI_Allreduce(&flag,&flagall,1,MPI_INT,MPI_MIN,comm);
if (flagall) return;
// ndest_off = list of IDs in dest owned by other procs
work = (int *) memory->smalloc(ndest*sizeof(int),"many2many:work");
for (i = 0; i < ndest; i++) work[i] = 0;
for (i = 0; i < nown; i++) work[dest_own[i]] = 1;
ndest_off = 0;
for (i = 0; i < ndest; i++)
if (work[i] == 0) dest_off[ndest_off++] = i;
memory->sfree(work);
// realloc off-proc arrays to smaller size
src_off = (int *)
memory->srealloc(src_off,nsrc_off*sizeof(int),"many2many:src_off");
dest_off = (int *)
memory->srealloc(dest_off,ndest_off*sizeof(int),"many2many:dest_off");
// send off-proc src and dest Datums to 3rd-party proc via irregular comm
// proc = ID % nprocs
nsend = nsrc_off + ndest_off;
proclist = new int[nsend];
Datum1 *send1 = new Datum1[nsend];
for (i = 0; i < nsrc_off; i++) {
proclist[i] = id_src[src_off[i]] % nprocs;
send1[i].id = id_src[src_off[i]];
send1[i].proc = me;
send1[i].index = src_off[i];
}
for (i = 0, j = nsrc_off; i < ndest_off; i++, j++) {
proclist[j] = id_dest[dest_off[i]] % nprocs;
send1[j].id = -id_dest[dest_off[i]];
send1[j].proc = me;
send1[j].index = dest_off[i];
}
irregular = new Irregular(comm);
irregular->pattern(nsend,proclist);
nrecv = irregular->size(sizeof(Datum1)) / sizeof(Datum1);
Datum1 *recv1 = new Datum1[nrecv];
irregular->exchange((char *) send1, (char *) recv1);
delete irregular;
delete [] proclist;
// as 3rd-party proc, now have matching pairs of off-proc IDs
// store src IDs (which are positive) in hash
// loop over dest IDs (which are negative) to find matches
// send match info back to src procs via a 2nd irregular comm
nsend = nrecv/2;
proclist = new int[nsend];
Datum2 *send2 = new Datum2[nsend];
nsend = 0;
hash.clear();
for (isrc = 0; isrc < nrecv; isrc++)
if (recv1[isrc].id > 0)
hash.insert(std::pair<int,int> (recv1[isrc].id,isrc));
for (idest = 0; idest < nrecv; idest++)
if (recv1[idest].id < 0) {
loc = hash.find(-recv1[idest].id);
if (loc != hash.end()) {
isrc = loc->second;
proclist[nsend] = recv1[isrc].proc;
send2[nsend].slocal = recv1[isrc].index;
send2[nsend].dlocal = recv1[idest].index;
send2[nsend].dproc = recv1[idest].proc;
nsend++;
} else error->one("Did not receive matching src/dest ID");
}
irregular = new Irregular(comm);
irregular->pattern(nsend,proclist);
nrecv = irregular->size(sizeof(Datum2)) / sizeof(Datum2);
Datum2 *recv2 = new Datum2[nrecv];
irregular->exchange((char *) send2, (char *) recv2);
delete irregular;
delete [] proclist;
// use list of received src->dest Datums to build final irregular commm
// irregular comm will communicate off-proc info from src to dest directly
// work = local indices of dest IDs to send initially
nsend = nrecv;
proclist = new int[nsend];
work = new int[nsend];
for (i = 0; i < nrecv; i++) {
src_off[i] = recv2[i].slocal;
work[i] = recv2[i].dlocal;
proclist[i] = recv2[i].dproc;
}
irregular = new Irregular(comm);
irregular->pattern(nsend,proclist);
// send receiver's local indices
// receiver stores them as indirection list in dest_off
nrecv = irregular->size(sizeof(int)) / sizeof(int);
irregular->exchange((char *) work, (char *) dest_off);
delete [] proclist;
delete [] work;
// create work arrays for data exchange of int/double data
src_iwork =
(int *) memory->smalloc(nsrc_off*sizeof(int),"many2many:src_iwork");
dest_iwork =
(int *) memory->smalloc(ndest_off*sizeof(int),"many2many:dest_iwork");
src_dwork =
(double *) memory->smalloc(nsrc_off*sizeof(double),"many2many:src_dwork");
dest_dwork =
(double *) memory->smalloc(ndest_off*sizeof(double),
"many2many:dest_dwork");
// clean up
delete [] send1;
delete [] recv1;
delete [] send2;
delete [] recv2;
// debug checks for full coverage of srd/dest - can delete eventually
work = new int[MAX(nsrc,ndest)];
for (i = 0; i < nsrc; i++) work[i] = 0;
for (i = 0; i < nown; i++) work[src_own[i]]++;
for (i = 0; i < nsrc_off; i++) work[src_off[i]]++;
for (i = 0; i < nsrc; i++)
if (work[i] != 1) {
char str[128];
sprintf(str,"BAD SRC %d: %d %d\n",me,i,work[i]);
error->one(str);
}
for (i = 0; i < ndest; i++) work[i] = 0;
for (i = 0; i < nown; i++) work[dest_own[i]]++;
for (i = 0; i < ndest_off; i++) work[dest_off[i]]++;
for (i = 0; i < ndest; i++)
if (work[i] != 1) {
char str[128];
sprintf(str,"BAD DEST %d: %d %d\n",me,i,work[i]);
error->one(str);
}
delete [] work;
}
/* ----------------------------------------------------------------------
transfer one src entity to dest entity, matched by IDs in create()
operates on an int vector
------------------------------------------------------------------------- */
void Many2Many::exchange(int *src, int *dest)
{
int i;
// copy on-proc info
for (i = 0; i < nown; i++)
dest[dest_own[i]] = src[src_own[i]];
// communicate off-proc info
// user src_off and dest_off to pack/unpack data
if (irregular) {
int nrecv = irregular->size(sizeof(int)) / sizeof(int);
for (i = 0; i < nsrc_off; i++) src_iwork[i] = src[src_off[i]];
irregular->exchange((char *) src_iwork, (char *) dest_iwork);
for (i = 0; i < ndest_off; i++) dest[dest_off[i]] = dest_iwork[i];
}
}
/* ----------------------------------------------------------------------
transfer one src entity to dest entity, matched by IDs in create()
operates on a double vector
------------------------------------------------------------------------- */
void Many2Many::exchange(double *src, double *dest)
{
int i;
// copy on-proc info
for (int i = 0; i < nown; i++)
dest[dest_own[i]] = src[src_own[i]];
// communicate off-proc info
// user src_off and dest_off to pack/unpack data
if (irregular) {
int nrecv = irregular->size(sizeof(double)) / sizeof(double);
for (i = 0; i < nsrc_off; i++) src_dwork[i] = src[src_off[i]];
irregular->exchange((char *) src_dwork, (char *) dest_dwork);
for (i = 0; i < ndest_off; i++) dest[dest_off[i]] = dest_dwork[i];
}
}
/* ---------------------------------------------------------------------- */
void Many2Many::deallocate()
{
memory->sfree(src_own);
memory->sfree(dest_own);
memory->sfree(src_off);
memory->sfree(dest_off);
memory->sfree(src_iwork);
memory->sfree(dest_iwork);
memory->sfree(src_dwork);
memory->sfree(dest_dwork);
delete irregular;
}

View File

@ -0,0 +1,78 @@
many2many.o: many2many.cpp /usr/local/include/mpi.h \
/usr/local/include/mpio.h /usr/local/include/mpi.h \
/usr/local/include/mpicxx.h many2many.h irregular.h memory.h error.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/map \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_tree.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_algobase.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++config.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/os_defines.h \
/usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstring \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstddef \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/string.h /usr/include/xlocale.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/climits \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/limits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/syslimits.h \
/usr/include/limits.h /usr/include/bits/posix1_lim.h \
/usr/include/bits/local_lim.h /usr/include/linux/limits.h \
/usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \
/usr/include/bits/stdio_lim.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstdlib \
/usr/include/stdlib.h /usr/include/bits/waitflags.h \
/usr/include/bits/waitstatus.h /usr/include/endian.h \
/usr/include/bits/endian.h /usr/include/sys/types.h \
/usr/include/bits/types.h /usr/include/bits/wordsize.h \
/usr/include/bits/typesizes.h /usr/include/time.h \
/usr/include/sys/select.h /usr/include/bits/select.h \
/usr/include/bits/sigset.h /usr/include/bits/time.h \
/usr/include/sys/sysmacros.h /usr/include/bits/pthreadtypes.h \
/usr/include/bits/sched.h /usr/include/alloca.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/new \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/exception \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/iosfwd \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++locale.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstdio \
/usr/include/stdio.h /usr/include/libio.h /usr/include/_G_config.h \
/usr/include/wchar.h /usr/include/bits/wchar.h /usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/clocale \
/usr/include/locale.h /usr/include/bits/locale.h \
/usr/include/langinfo.h /usr/include/nl_types.h /usr/include/iconv.h \
/usr/include/libintl.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++io.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/gthr.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/gthr-default.h \
/usr/include/pthread.h /usr/include/sched.h /usr/include/signal.h \
/usr/include/bits/initspin.h /usr/include/bits/sigthread.h \
/usr/include/unistd.h /usr/include/bits/posix_opt.h \
/usr/include/bits/environments.h /usr/include/bits/confname.h \
/usr/include/getopt.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cctype \
/usr/include/ctype.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stringfwd.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/postypes.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cwchar \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/ctime \
/usr/include/stdint.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/functexcept.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/exception_defines.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_pair.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/type_traits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator_base_types.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator_base_funcs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/concept_check.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/debug/debug.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cassert \
/usr/include/assert.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/ext/new_allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_construct.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_function.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/cpp_type_traits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_map.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_multimap.h

View File

@ -0,0 +1,47 @@
#ifndef MANY2MANY_H
#define MANY2MANY_H
#include "mpi.h"
class Many2Many {
public:
Many2Many(MPI_Comm);
~Many2Many();
void setup(int, int *, int, int *);
void exchange(int *, int *);
void exchange(double *, double *);
protected:
int me,nprocs;
MPI_Comm comm;
class Memory *memory;
class Error *error;
int nown; // # of IDs common to src and dest
int nsrc_off,ndest_off; // # of off-processor IDs
int *src_own,*dest_own; // indices of the owned IDs
int *src_off,*dest_off; // indices of the off-proc IDs
int *src_iwork,*dest_iwork; // work arrays for comm of ints
double *src_dwork,*dest_dwork; // work arrays for comm of doubles
class Irregular *irregular; // irregular comm from src->dest
struct Datum1 {
int id; // src or dest global ID
int proc; // owning proc
int index; // local index on owning proc
};
struct Datum2 {
int slocal; // local index of src ID on sending proc
int dlocal; // local index of dest ID on receiving proc
int dproc; // receiving proc
};
void deallocate();
};
#endif

View File

@ -0,0 +1,103 @@
#include "mpi.h"
#include "stdio.h"
#include "stdlib.h"
#include "many2one.h"
#include "memory.h"
/* ---------------------------------------------------------------------- */
Many2One::Many2One(MPI_Comm caller_comm)
{
comm = caller_comm;
MPI_Comm_rank(comm,&me);
MPI_Comm_size(comm,&nprocs);
memory = new Memory(comm);
if (me == 0) {
counts = new int[nprocs];
multicounts = new int[nprocs];
displs = new int[nprocs];
multidispls = new int[nprocs];
} else counts = multicounts = displs = multidispls = NULL;
idall = NULL;
}
/* ---------------------------------------------------------------------- */
Many2One::~Many2One()
{
delete memory;
delete [] counts;
delete [] multicounts;
delete [] displs;
delete [] multidispls;
memory->sfree(idall);
}
/* ---------------------------------------------------------------------- */
void Many2One::setup(int nsrc_in, int *id, int ndest)
{
nsrc = nsrc_in;
MPI_Allreduce(&nsrc,&nall,1,MPI_INT,MPI_SUM,comm);
MPI_Gather(&nsrc,1,MPI_INT,counts,1,MPI_INT,0,comm);
if (me == 0) {
displs[0] = 0;
for (int i = 1; i < nprocs; i++)
displs[i] = displs[i-1] + counts[i-1];
}
// gather IDs into idall
idall = NULL;
if (me == 0)
idall = (int *) memory->smalloc(nall*sizeof(int),"many2one:idall");
MPI_Gatherv(id,nsrc,MPI_INT,idall,counts,displs,MPI_INT,0,comm);
}
/* ---------------------------------------------------------------------- */
void Many2One::gather(double *src, int n, double *dest)
{
int i,j,ii,jj,m;
if (me == 0)
for (int i = 0; i < nprocs; i++) {
multicounts[i] = n*counts[i];
multidispls[i] = n*displs[i];
}
// allgather src into desttmp
double *desttmp = NULL;
if (me == 0)
desttmp = (double *) memory->smalloc(n*nall*sizeof(double),
"many2one:idsttmp");
MPI_Gatherv(src,n*nsrc,MPI_DOUBLE,desttmp,multicounts,multidispls,
MPI_DOUBLE,0,comm);
// use idall to move datums from desttmp to dest
if (me == 0) {
if (n == 1)
for (i = 0; i < nall; i++) {
j = idall[i] - 1;
dest[j] = desttmp[i];
}
else
for (i = 0; i < nall; i++) {
j = idall[i] - 1;
ii = n*i;
jj = n*j;
for (m = 0; m < n; m++)
dest[jj++] = desttmp[ii++];
}
}
memory->sfree(desttmp);
}

View File

@ -0,0 +1,20 @@
many2one.o: many2one.cpp /usr/local/include/mpi.h \
/usr/local/include/mpio.h /usr/local/include/mpi.h \
/usr/local/include/mpicxx.h /usr/include/stdio.h \
/usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/types.h /usr/include/bits/wordsize.h \
/usr/include/bits/typesizes.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h /usr/include/stdlib.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h many2one.h memory.h

View File

@ -0,0 +1,25 @@
#ifndef MANY2ONE_H
#define MANY2ONE_H
#include "mpi.h"
class Many2One {
public:
Many2One(MPI_Comm);
~Many2One();
void setup(int, int *, int);
void gather(double *, int, double *);
protected:
int me,nprocs;
MPI_Comm comm;
class Memory *memory;
int nsrc,nall;
int *counts,*multicounts;
int *displs,*multidispls;
int *idall;
};
#endif

View File

@ -0,0 +1,120 @@
#include "mpi.h"
#include "stdlib.h"
#include "stdio.h"
#include "memory.h"
#include "error.h"
/* ---------------------------------------------------------------------- */
Memory::Memory(MPI_Comm comm)
{
error = new Error(comm);
}
/* ---------------------------------------------------------------------- */
Memory::~Memory()
{
delete error;
}
/* ----------------------------------------------------------------------
safe malloc
------------------------------------------------------------------------- */
void *Memory::smalloc(int n, const char *name)
{
if (n == 0) return NULL;
void *ptr = malloc(n);
if (ptr == NULL) {
char str[128];
sprintf(str,"Failed to allocate %d bytes for array %s",n,name);
error->one(str);
}
return ptr;
}
/* ----------------------------------------------------------------------
safe free
------------------------------------------------------------------------- */
void Memory::sfree(void *ptr)
{
if (ptr == NULL) return;
free(ptr);
}
/* ----------------------------------------------------------------------
safe realloc
------------------------------------------------------------------------- */
void *Memory::srealloc(void *ptr, int n, const char *name)
{
if (n == 0) {
sfree(ptr);
return NULL;
}
ptr = realloc(ptr,n);
if (ptr == NULL) {
char str[128];
sprintf(str,"Failed to reallocate %d bytes for array %s",n,name);
error->one(str);
}
return ptr;
}
/* ----------------------------------------------------------------------
create a 2d double array
------------------------------------------------------------------------- */
double **Memory::create_2d_double_array(int n1, int n2, const char *name)
{
double *data = (double *) smalloc(n1*n2*sizeof(double),name);
double **array = (double **) smalloc(n1*sizeof(double *),name);
int n = 0;
for (int i = 0; i < n1; i++) {
array[i] = &data[n];
n += n2;
}
return array;
}
/* ----------------------------------------------------------------------
grow or shrink 1st dim of a 2d double array
last dim must stay the same
if either dim is 0, return NULL
------------------------------------------------------------------------- */
double **Memory::grow_2d_double_array(double **array,
int n1, int n2, const char *name)
{
if (array == NULL) return create_2d_double_array(n1,n2,name);
double *data = (double *) srealloc(array[0],n1*n2*sizeof(double),name);
array = (double **) srealloc(array,n1*sizeof(double *),name);
int n = 0;
for (int i = 0; i < n1; i++) {
array[i] = &data[n];
n += n2;
}
return array;
}
/* ----------------------------------------------------------------------
free a 2d double array
------------------------------------------------------------------------- */
void Memory::destroy_2d_double_array(double **array)
{
if (array == NULL) return;
sfree(array[0]);
sfree(array);
}

View File

@ -0,0 +1,19 @@
memory.o: memory.cpp /usr/local/include/mpi.h /usr/local/include/mpio.h \
/usr/local/include/mpi.h /usr/local/include/mpicxx.h \
/usr/include/stdlib.h /usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/bits/types.h \
/usr/include/bits/wordsize.h /usr/include/bits/typesizes.h \
/usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h /usr/include/stdio.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h memory.h error.h

View File

@ -0,0 +1,23 @@
#ifndef MEMORY_H
#define MEMORY_H
#include "mpi.h"
class Memory {
public:
Memory(MPI_Comm);
~Memory();
void *smalloc(int n, const char *);
void sfree(void *);
void *srealloc(void *, int n, const char *name);
double **create_2d_double_array(int, int, const char *);
double **grow_2d_double_array(double **, int, int, const char *);
void destroy_2d_double_array(double **);
private:
class Error *error;
};
#endif

View File

@ -0,0 +1,76 @@
#include "mpi.h"
#include "one2many.h"
#include "memory.h"
#include <map>
/* ---------------------------------------------------------------------- */
One2Many::One2Many(MPI_Comm caller_comm)
{
comm = caller_comm;
MPI_Comm_rank(comm,&me);
MPI_Comm_size(comm,&nprocs);
memory = new Memory(comm);
hash = new std::map<int,int>();
}
/* ---------------------------------------------------------------------- */
One2Many::~One2Many()
{
delete memory;
delete hash;
}
/* ---------------------------------------------------------------------- */
void One2Many::setup(int nsrc_in, int ndest, int *id)
{
nsrc = nsrc_in;
// store my local IDs in hash
hash->clear();
for (int i = 0; i < ndest; i++)
hash->insert(std::pair<int,int> (id[i],i));
}
/* ---------------------------------------------------------------------- */
void One2Many::scatter(double *src, int n, double *dest)
{
int i,j,k,m;
// allocate src on procs that don't have it
int flag = 0;
if (src == NULL) {
src = (double *) memory->smalloc(n*nsrc*sizeof(double),"one2many:src");
flag = 1;
}
// broadcast src from 0 to other procs
MPI_Bcast(src,n*nsrc,MPI_DOUBLE,0,comm);
// each proc loops over entire src
// if I own the global ID, copy src values into dest
std::map<int,int>::iterator loc;
for (m = 1; m <= nsrc; m++) {
loc = hash->find(m);
if (loc == hash->end()) continue;
i = n*loc->second;
j = 3*(m-1);
if (n == 1) dest[i] = src[j];
else
for (k = 0; k < n; k++)
dest[i++] = src[j++];
}
// free locally allocated src
if (flag) memory->sfree(src);
}

View File

@ -0,0 +1,79 @@
one2many.o: one2many.cpp /usr/local/include/mpi.h \
/usr/local/include/mpio.h /usr/local/include/mpi.h \
/usr/local/include/mpicxx.h one2many.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/map \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_tree.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_algobase.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++config.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/os_defines.h \
/usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstring \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstddef \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/string.h /usr/include/xlocale.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/climits \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/limits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/syslimits.h \
/usr/include/limits.h /usr/include/bits/posix1_lim.h \
/usr/include/bits/local_lim.h /usr/include/linux/limits.h \
/usr/include/bits/posix2_lim.h /usr/include/bits/xopen_lim.h \
/usr/include/bits/stdio_lim.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstdlib \
/usr/include/stdlib.h /usr/include/bits/waitflags.h \
/usr/include/bits/waitstatus.h /usr/include/endian.h \
/usr/include/bits/endian.h /usr/include/sys/types.h \
/usr/include/bits/types.h /usr/include/bits/wordsize.h \
/usr/include/bits/typesizes.h /usr/include/time.h \
/usr/include/sys/select.h /usr/include/bits/select.h \
/usr/include/bits/sigset.h /usr/include/bits/time.h \
/usr/include/sys/sysmacros.h /usr/include/bits/pthreadtypes.h \
/usr/include/bits/sched.h /usr/include/alloca.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/new \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/exception \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/iosfwd \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++locale.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cstdio \
/usr/include/stdio.h /usr/include/libio.h /usr/include/_G_config.h \
/usr/include/wchar.h /usr/include/bits/wchar.h /usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/clocale \
/usr/include/locale.h /usr/include/bits/locale.h \
/usr/include/langinfo.h /usr/include/nl_types.h /usr/include/iconv.h \
/usr/include/libintl.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++io.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/gthr.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/gthr-default.h \
/usr/include/pthread.h /usr/include/sched.h /usr/include/signal.h \
/usr/include/bits/initspin.h /usr/include/bits/sigthread.h \
/usr/include/unistd.h /usr/include/bits/posix_opt.h \
/usr/include/bits/environments.h /usr/include/bits/confname.h \
/usr/include/getopt.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cctype \
/usr/include/ctype.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stringfwd.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/postypes.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cwchar \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/ctime \
/usr/include/stdint.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/functexcept.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/exception_defines.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_pair.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/type_traits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator_base_types.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator_base_funcs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/concept_check.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_iterator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/debug/debug.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/cassert \
/usr/include/assert.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/x86_64-redhat-linux/bits/c++allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/ext/new_allocator.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_construct.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_function.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/cpp_type_traits.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_map.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/../../../../include/c++/3.4.6/bits/stl_multimap.h \
memory.h

View File

@ -0,0 +1,24 @@
#ifndef ONE2MANY_H
#define ONE2MANY_H
#include "mpi.h"
#include <map>
class One2Many {
public:
One2Many(MPI_Comm);
~One2Many();
void setup(int, int, int *);
void scatter(double *, int, double *);
protected:
int me,nprocs;
MPI_Comm comm;
class Memory *memory;
std::map<int,int> *hash;
int nsrc;
};
#endif

View File

@ -0,0 +1,83 @@
#include "mpi.h"
#include "stdlib.h"
#include "stdio.h"
#include "send2one.h"
#include "memory.h"
#include "error.h"
/* ---------------------------------------------------------------------- */
Send2One::Send2One(MPI_Comm caller_comm)
{
comm = caller_comm;
MPI_Comm_rank(comm,&me);
MPI_Comm_size(comm,&nprocs);
memory = new Memory(comm);
error = new Error(comm);
buf = NULL;
maxbuf = 0;
}
/* ---------------------------------------------------------------------- */
Send2One::~Send2One()
{
delete memory;
delete error;
memory->sfree(buf);
}
/* ---------------------------------------------------------------------- */
void Send2One::execute()
{
int nme,nmax,nsize,ping;
MPI_Status status;
MPI_Request request;
// pre-processing before ping loop
pre();
// nme = size of data I contribute, in bytes
// nmax = max size of data on any proc, in bytes
// reallocate buf if necessary
nme = size();
MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,comm);
if (nmax > maxbuf) {
maxbuf = nmax;
memory->sfree(buf);
buf = (char *) memory->smalloc(maxbuf,"foo:buf");
}
// pack my data into buf
pack(buf);
// proc 0 pings each proc, receives its data
// all other procs wait for ping, send their data to proc 0
// invoke process() to work with data
if (me == 0) {
for (int iproc = 0; iproc < nprocs; iproc++) {
if (iproc) {
MPI_Irecv(buf,maxbuf,MPI_CHAR,iproc,0,comm,&request);
MPI_Send(&ping,0,MPI_INT,iproc,0,comm);
MPI_Wait(&request,&status);
MPI_Get_count(&status,MPI_CHAR,&nsize);
} else nsize = nme;
process(nsize,buf);
}
} else {
MPI_Recv(&ping,0,MPI_INT,0,0,comm,&status);
MPI_Rsend(buf,nme,MPI_CHAR,0,0,comm);
}
post();
}

View File

@ -0,0 +1,20 @@
send2one.o: send2one.cpp /usr/local/include/mpi.h \
/usr/local/include/mpio.h /usr/local/include/mpi.h \
/usr/local/include/mpicxx.h /usr/include/stdlib.h \
/usr/include/features.h /usr/include/sys/cdefs.h \
/usr/include/gnu/stubs.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stddef.h \
/usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \
/usr/include/endian.h /usr/include/bits/endian.h /usr/include/xlocale.h \
/usr/include/sys/types.h /usr/include/bits/types.h \
/usr/include/bits/wordsize.h /usr/include/bits/typesizes.h \
/usr/include/time.h /usr/include/sys/select.h \
/usr/include/bits/select.h /usr/include/bits/sigset.h \
/usr/include/bits/time.h /usr/include/sys/sysmacros.h \
/usr/include/bits/pthreadtypes.h /usr/include/bits/sched.h \
/usr/include/alloca.h /usr/include/stdio.h /usr/include/libio.h \
/usr/include/_G_config.h /usr/include/wchar.h /usr/include/bits/wchar.h \
/usr/include/gconv.h \
/usr/lib/gcc/x86_64-redhat-linux/3.4.6/include/stdarg.h \
/usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \
/usr/include/bits/stdio.h send2one.h memory.h error.h

View File

@ -0,0 +1,29 @@
#ifndef SEND2ONE_H
#define SEND2ONE_H
#include "mpi.h"
class Send2One {
public:
Send2One(MPI_Comm);
virtual ~Send2One();
void execute();
protected:
int me,nprocs;
MPI_Comm comm;
class Memory *memory;
class Error *error;
int maxbuf;
char *buf;
virtual void pre() = 0;
virtual int size() = 0;
virtual void pack(char *) = 0;
virtual void process(int, char *) = 0;
virtual void post() = 0;
};
#endif