diff --git a/README b/README deleted file mode 100644 index a677e0a8..00000000 --- a/README +++ /dev/null @@ -1,82 +0,0 @@ -/*---------------------------------------------------------------------------*\ - CFDEMcoupling - Open Source CFD-DEM coupling - - CFDEMcoupling is part of the CFDEMproject - www.cfdem.com - Christoph Goniva, christoph.goniva@cfdem.com - Copyright 2009-2012 JKU Linz - Copyright 2012- DCS Computing GmbH, Linz -------------------------------------------------------------------------------- -License - This file is part of CFDEMcoupling. - - CFDEMcoupling is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation; either version 3 of the License, or (at your - option) any later version. - - CFDEMcoupling is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with CFDEMcoupling; if not, write to the Free Software Foundation, - Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -Description - This code is designed to realize coupled CFD-DEM simulations using LIGGGHTS - and OpenFOAM. Note: this code is not part of OpenFOAM (see DISCLAIMER). -\*---------------------------------------------------------------------------*/ - - -CFDEM coupling provides an open source parallel coupled CFD-DEM framework -combining the strengths of LIGGGHTS DEM code and the Open Source -CFD package OpenFOAM(R)(*). The CFDEMcoupling toolbox allows to expand -standard CFD solvers of OpenFOAM(R)(*) to include a coupling to the DEM -code LIGGGHTS. In this toolbox the particle representation within the -CFD solver is organized by "cloud" classes. Key functionalities are organised -in sub-models (e.g. force models, data exchange models, etc.) which can easily -be selected and combined by dictionary settings. - -The coupled solvers run fully parallel on distributed-memory clusters. - -Features are: - -- its modular approach allows users to easily implement new models -- its MPI parallelization enables to use it for large scale problems -- the "forum"_lws on CFD-DEM gives the possibility to exchange with other - users / developers -- the use of GIT allows to easily update to the latest version -- basic documentation is provided - -The file structure: - -- "src" directory including the source files of the coupling toolbox and models -- "applications" directory including the solver files for coupled CFD-DEM simulations -- "doc" directory including the documentation of CFDEMcoupling -- "tutorials" directory including basic tutorial cases showing the functionality - - - -Details on installation are given on the "www.cfdem.com" - -The functionality of this CFD-DEM framwork is described via "tutorial cases" showing -how to use different solvers and models. - -CFDEMcoupling stands for Computational Fluid Dynamics (CFD) - -Discrete Element Method (DEM) coupling. - -CFDEMcoupling is an open-source code, distributed freely under the terms of the -GNU Public License (GPL). - -Core development of CFDEMcoupling is done by -Christoph Goniva and Christoph Kloss, both at DCS Computing GmbH, 2012 - - -\*---------------------------------------------------------------------------*/ -(*) "OpenFOAM(R)"_of is a registered trade mark of Silicon Graphics -International Corp. This offering is not affiliated, approved or endorsed by -Silicon Graphics International Corp., the producer of the OpenFOAM(R) software -and owner of the OpenFOAM(R) trademark. -\*---------------------------------------------------------------------------*/ diff --git a/doc/CFDEMcoupling_Manual.pdf b/doc/CFDEMcoupling_Manual.pdf index 22152968..98f37ddc 100644 Binary files a/doc/CFDEMcoupling_Manual.pdf and b/doc/CFDEMcoupling_Manual.pdf differ diff --git a/doc/githubAccess_public.pdf b/doc/githubAccess_public.pdf index 1adacf4d..7a22d690 100644 Binary files a/doc/githubAccess_public.pdf and b/doc/githubAccess_public.pdf differ diff --git a/src/lagrangian/cfdemParticle/Make/files b/src/lagrangian/cfdemParticle/Make/files index b26fe6cd..e2411291 100644 --- a/src/lagrangian/cfdemParticle/Make/files +++ b/src/lagrangian/cfdemParticle/Make/files @@ -20,18 +20,33 @@ $(forceModels)/forceModel/forceModel.C $(forceModels)/forceModel/newForceModel.C $(forceModels)/noDrag/noDrag.C $(forceModels)/DiFeliceDrag/DiFeliceDrag.C +$(forceModels)/DiFeliceDragNLift/DiFeliceDragNLift.C $(forceModels)/GidaspowDrag/GidaspowDrag.C $(forceModels)/SchillerNaumannDrag/SchillerNaumannDrag.C $(forceModels)/Archimedes/Archimedes.C $(forceModels)/ArchimedesIB/ArchimedesIB.C $(forceModels)/interface/interface.C $(forceModels)/ShirgaonkarIB/ShirgaonkarIB.C +$(forceModels)/fieldTimeAverage/fieldTimeAverage.C +$(forceModels)/fieldBound/fieldBound.C +$(forceModels)/volWeightedAverage/volWeightedAverage.C +$(forceModels)/totalMomentumExchange/totalMomentumExchange.C $(forceModels)/KochHillDrag/KochHillDrag.C +$(forceModels)/BeetstraDrag/multiphaseFlowBasic/multiphaseFlowBasic.C +$(forceModels)/BeetstraDrag/BeetstraDrag.C +$(forceModels)/LaEuScalarLiquid/LaEuScalarLiquid.C $(forceModels)/LaEuScalarTemp/LaEuScalarTemp.C +$(forceModels)/LaEuScalarDust/LaEuScalarDust.C $(forceModels)/virtualMassForce/virtualMassForce.C $(forceModels)/gradPForce/gradPForce.C +$(forceModels)/gradULiftForce/gradULiftForce.C $(forceModels)/viscForce/viscForce.C $(forceModels)/MeiLift/MeiLift.C +$(forceModels)/KochHillDragNLift/KochHillDragNLift.C +$(forceModels)/solidsPressureForce/solidsPressureForce.C +$(forceModels)/periodicPressure/periodicPressure.C +$(forceModels)/periodicPressureControl/periodicPressureControl.C +$(forceModels)/averageSlipVel/averageSlipVel.C $(forceModelsMS)/forceModelMS/forceModelMS.C $(forceModelsMS)/forceModelMS/newForceModelMS.C @@ -42,6 +57,7 @@ $(IOModels)/IOModel/newIOModel.C $(IOModels)/noIO/noIO.C $(IOModels)/basicIO/basicIO.C $(IOModels)/trackIO/trackIO.C +$(IOModels)/sophIO/sophIO.C $(voidFractionModels)/voidFractionModel/voidFractionModel.C $(voidFractionModels)/voidFractionModel/newVoidFractionModel.C @@ -57,22 +73,25 @@ $(locateModels)/locateModel/newLocateModel.C $(locateModels)/standardSearch/standardSearch.C $(locateModels)/engineSearch/engineSearch.C $(locateModels)/turboEngineSearch/turboEngineSearch.C +$(locateModels)/turboEngineSearchM2M/turboEngineSearchM2M.C $(locateModels)/engineSearchIB/engineSearchIB.C - $(meshMotionModels)/meshMotionModel/meshMotionModel.C $(meshMotionModels)/meshMotionModel/newMeshMotionModel.C $(meshMotionModels)/noMeshMotion/noMeshMotion.C +$(meshMotionModels)/DEMdrivenMeshMotion/DEMdrivenMeshMotion.C $(momCoupleModels)/momCoupleModel/momCoupleModel.C $(momCoupleModels)/momCoupleModel/newMomCoupleModel.C $(momCoupleModels)/explicitCouple/explicitCouple.C +$(momCoupleModels)/explicitCoupleSource/explicitCoupleSource.C $(momCoupleModels)/implicitCouple/implicitCouple.C $(momCoupleModels)/noCouple/noCouple.C $(regionModels)/regionModel/regionModel.C $(regionModels)/regionModel/newRegionModel.C $(regionModels)/allRegion/allRegion.C +$(regionModels)/differentialRegion/differentialRegion.C $(dataExchangeModels)/dataExchangeModel/dataExchangeModel.C $(dataExchangeModels)/dataExchangeModel/newDataExchangeModel.C @@ -80,6 +99,7 @@ $(dataExchangeModels)/oneWayVTK/oneWayVTK.C $(dataExchangeModels)/twoWayFiles/twoWayFiles.C $(dataExchangeModels)/noDataExchange/noDataExchange.C $(dataExchangeModels)/twoWayMPI/twoWayMPI.C +$(dataExchangeModels)/twoWayM2M/twoWayM2M.C $(averagingModels)/averagingModel/averagingModel.C $(averagingModels)/averagingModel/newAveragingModel.C diff --git a/src/lagrangian/cfdemParticle/Make/options b/src/lagrangian/cfdemParticle/Make/options index e24e4695..eb5d1463 100644 --- a/src/lagrangian/cfdemParticle/Make/options +++ b/src/lagrangian/cfdemParticle/Make/options @@ -13,6 +13,7 @@ EXE_INC = \ -I$(LIB_SRC)/lagrangian/basic/lnInclude \ -I$(LIB_SRC)/OpenFOAM/containers/HashTables/labelHashSet \ -I$(CFDEM_LIGGGHTS_SRC_DIR) \ + -I$(CFDEM_M2MLIB_PATH) \ LIB_LIBS = \ $(PLIBS) \ @@ -24,4 +25,6 @@ LIB_LIBS = \ -llagrangian \ -lmpi_cxx \ -L$(CFDEM_LIGGGHTS_SRC_DIR) \ - -l$(CFDEM_LIGGGHTS_LIB_NAME) + -l$(CFDEM_LIGGGHTS_LIB_NAME) \ + -L$(CFDEM_M2MLIB_PATH) \ + -lcouple diff --git a/src/lagrangian/cfdemParticle/etc/compileM2Mlib.sh b/src/lagrangian/cfdemParticle/etc/compileM2Mlib.sh new file mode 100755 index 00000000..8d8583d1 --- /dev/null +++ b/src/lagrangian/cfdemParticle/etc/compileM2Mlib.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#===================================================================# +# compile routine for LAMMPS m2m library +# Christoph Goniva - Oct. 2012, DCS Computing GmbH +#=================================================================== + +#- include functions +source $CFDEM_SRC_DIR/etc/functions.sh + +NOW="$(date +"%Y-%m-%d-%H:%M")" +logDir="log" + +cd $CFDEM_SRC_DIR/etc +mkdir $logDir + +#--------------------------------------------------------------------------------# +#- define variables +logpath="$(dirname "$(readlink -f ${BASH_SOURCE[0]})")/$logDir" +logfileName="log_compileM2Mlib" +headerText="$logfileName""-$NOW" +makeFileName="Makefile.$CFDEM_LIGGGHTS_MAKEFILE_NAME" +libraryPath="$CFDEM_M2MLIB_PATH" +#--------------------------------------------------------------------------------# + +compileLMPlib $logpath $logfileName $headerText $makeFileName $libraryPath diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.fedora_fpic b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.fedora_fpic new file mode 100644 index 00000000..cbb77dca --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.fedora_fpic @@ -0,0 +1,43 @@ +# Makefile for coupling library + +SHELL = /bin/sh + +# System-specific settings + +CC = mpic++ +CCFLAGS = -g -O2 -fPIC \ + -funroll-loops -fstrict-aliasing -Wall -W -Wno-uninitialized +DEPFLAGS = -M +LINK = mpic++ +LINKFLAGS = -O -fPIC +ARCHIVE = ar +ARFLAGS = -rcsv +SIZE = size + +# Files + +LIB = libcouple.a +SRC = $(wildcard *.cpp) +INC = $(wildcard *.h) +OBJ = $(SRC:.cpp=.o) + +# Targets + +lib: $(OBJ) + $(ARCHIVE) $(ARFLAGS) $(LIB) $(OBJ) + +clean: + rm $(LIB) *.o *.d + +# Compilation rules + +%.o:%.cpp + $(CC) $(CCFLAGS) -c $< + +%.d:%.cpp + $(CC) $(CCFLAGS) $(DEPFLAGS) $< > $@ + +# Individual dependencies + +DEPENDS = $(OBJ:.o=.d) +include $(DEPENDS) diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.g++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.g++ new file mode 100644 index 00000000..07e58076 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/Makefile.g++ @@ -0,0 +1,42 @@ +# Makefile for coupling library + +SHELL = /bin/sh + +# System-specific settings + +CC = g++ +CCFLAGS = -g -O -DMPICH_IGNORE_CXX_SEEK +DEPFLAGS = -M +LINK = g++ +LINKFLAGS = -g -O +ARCHIVE = ar +ARFLAGS = -rc +SIZE = size + +# Files + +LIB = libcouple.a +SRC = $(wildcard *.cpp) +INC = $(wildcard *.h) +OBJ = $(SRC:.cpp=.o) + +# Targets + +lib: $(OBJ) + $(ARCHIVE) $(ARFLAGS) $(LIB) $(OBJ) + +clean: + rm $(LIB) *.o *.d + +# Compilation rules + +%.o:%.cpp + $(CC) $(CCFLAGS) -c $< + +%.d:%.cpp + $(CC) $(CCFLAGS) $(DEPFLAGS) $< > $@ + +# Individual dependencies + +DEPENDS = $(OBJ:.o=.d) +include $(DEPENDS) diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/README b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/README new file mode 100644 index 00000000..6bf0e6e6 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/README @@ -0,0 +1,16 @@ +This directory has a small collection of routines, useful for +exchanging data between 2 codes being run together as a coupled +application. It is used by the LAMMPS <-> Quest and LAMMPS <-> +SPPARKS applications in 2 sister directories. + +The library dir has a Makefile (which you may need to edit for your +box). If you type + +g++ -f Makefile.g++ + +you should create libcouple.a, which the other coupled applications +link to. + +Note that the library uses MPI, so the Makefile you use needs to +include a path to the MPI include file, if it is not someplace +the compiler will find it. diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.cpp new file mode 100644 index 00000000..c70ec26f --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.cpp @@ -0,0 +1,42 @@ +#include "mpi.h" +#include "stdlib.h" +#include "stdio.h" +#include "error.h" + +/* ---------------------------------------------------------------------- */ + +Error::Error(MPI_Comm caller) +{ + comm = caller; + MPI_Comm_rank(comm,&me); +} + +/* ---------------------------------------------------------------------- + called by all procs +------------------------------------------------------------------------- */ + +void Error::all(const char *str) +{ + if (me == 0) printf("ERROR: %s\n",str); + MPI_Finalize(); + exit(1); +} + +/* ---------------------------------------------------------------------- + called by one proc +------------------------------------------------------------------------- */ + +void Error::one(const char *str) +{ + printf("ERROR on proc %d: %s\n",me,str); + MPI_Abort(comm,1); +} + +/* ---------------------------------------------------------------------- + called by one proc +------------------------------------------------------------------------- */ + +void Error::warning(const char *str) +{ + printf("WARNING: %s\n",str); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.d new file mode 100644 index 00000000..67a22b5b --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.d @@ -0,0 +1,112 @@ +error.o: error.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.h new file mode 100644 index 00000000..f62f6205 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.h @@ -0,0 +1,19 @@ +#ifndef ERROR_H +#define ERROR_H + +#include "mpi.h" + +class Error { + public: + Error(MPI_Comm); + + void all(const char *); + void one(const char *); + void warning(const char *); + + private: + MPI_Comm comm; + int me; +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.o new file mode 100644 index 00000000..9c19d2ef Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/error.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.cpp new file mode 100644 index 00000000..83464deb --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.cpp @@ -0,0 +1,48 @@ +#include "stdio.h" +#include "string.h" +#include "files.h" + +#define MAXLINE 256 + +/* ---------------------------------------------------------------------- */ + +void replace(char *file, char *header, int n, char **lines) +{ + FILE *fpr = fopen(file,"r"); + FILE *fpw = fopen("tmp.file","w"); + + char line[MAXLINE]; + while (fgets(line,MAXLINE,fpr)) { + if (strstr(line,header)) { + fprintf(fpw,"%s",line); + for (int i = 0; i < n; i++) { + fgets(line,MAXLINE,fpr); + fprintf(fpw,"%s",lines[i]); + } + } else fprintf(fpw,"%s",line); + } + + fclose(fpr); + fclose(fpw); + rename("tmp.file",file); +} + +/* ---------------------------------------------------------------------- */ + +char **extract(char *file, char *header, int n, char **lines) +{ + FILE *fp = fopen(file,"r"); + + char line[MAXLINE]; + while (fgets(line,MAXLINE,fp)) { + if (strstr(line,header)) { + for (int i = 0; i < n; i++) { + fgets(line,MAXLINE,fp); + sprintf(lines[i],"%s",line); + } + break; + } + } + + fclose(fp); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.d new file mode 100644 index 00000000..b2661b97 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.d @@ -0,0 +1,12 @@ +files.o: files.cpp /usr/include/stdio.h /usr/include/features.h \ + /usr/include/bits/predefs.h /usr/include/sys/cdefs.h \ + /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ + /usr/include/gnu/stubs-64.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/include/bits/types.h /usr/include/bits/typesizes.h \ + /usr/include/libio.h /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/string.h /usr/include/xlocale.h /usr/include/bits/string3.h \ + files.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.h new file mode 100644 index 00000000..75812d29 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.h @@ -0,0 +1,2 @@ +void replace(char *, char *, int, char **); +char **extract(char *, char *, int, char **); diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.o new file mode 100644 index 00000000..83d5c14e Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/files.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.cpp new file mode 100644 index 00000000..aea9b833 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.cpp @@ -0,0 +1,393 @@ +#include "stdio.h" +#include "stdlib.h" +#include "string.h" +#include "irregular.h" +#include "memory.h" +#include "error.h" + +#define MAX(A,B) ((A) > (B)) ? (A) : (B) + +enum{UNSET,SET}; +enum{NONE,SAME,VARYING}; + +/* ---------------------------------------------------------------------- */ + +Irregular::Irregular(MPI_Comm caller) +{ + comm = caller; + MPI_Comm_rank(comm,&me); + MPI_Comm_size(comm,&nprocs); + + memory = new Memory(comm); + error = new Error(comm); + + init(); + + patternflag = UNSET; + sizestyle = NONE; +} + +/* ---------------------------------------------------------------------- */ + +Irregular::~Irregular() +{ + delete memory; + delete error; + deallocate(); +} + +/* ---------------------------------------------------------------------- + n = # of datums contributed by this proc + proclist = which proc each datum is to be sent to +------------------------------------------------------------------------- */ + +void Irregular::pattern(int n, int *proclist) +{ + // free any previous irregular info + + deallocate(); + init(); + + patternflag = SET; + sizestyle = NONE; + + ndatumsend = n; + + // list = 1 for procs I send to, including self + // nrecv = # of messages I receive, not including self + // self = 0 if no data for self, 1 if there is + + int *list = new int[nprocs]; + int *counts = new int[nprocs]; + + for (int i = 0; i < nprocs; i++) { + list[i] = 0; + counts[i] = 1; + } + + for (int i = 0; i < n; i++) list[proclist[i]] = 1; + MPI_Reduce_scatter(list,&nrecv,counts,MPI_INT,MPI_SUM,comm); + + self = 0; + if (list[me]) self = 1; + if (self) nrecv--; + + // storage for recv info, not including self + + recvproc = new int[nrecv]; + recvcount = new int[nrecv]; + recvsize = new int[nrecv]; + request = new MPI_Request[nrecv]; + status = new MPI_Status[nrecv]; + + // list = # of datums to send to each proc, including self + // nsend = # of messages I send, not including self + + for (int i = 0; i < nprocs; i++) list[i] = 0; + for (int i = 0; i < n; i++) list[proclist[i]]++; + + nsend = 0; + for (int i = 0; i < nprocs; i++) if (list[i] > 0) nsend++; + if (self) nsend--; + + // storage for send info, including self + + sendproc = new int[nsend+self]; + sendcount = new int[nsend+self]; + sendsize = new int[nsend+self]; + sendindices = (int *) memory->smalloc(n*sizeof(int),"sendindices"); + + // setup sendprocs and sendcounts, including self + // each proc begins with iproc > me, and continues until iproc = me + // list ends up with pointer to which send that proc is associated with + + int iproc = me; + int isend = 0; + for (int i = 0; i < nprocs; i++) { + iproc++; + if (iproc == nprocs) iproc = 0; + if (list[iproc] > 0) { + sendproc[isend] = iproc; + sendcount[isend] = list[iproc]; + list[iproc] = isend; + isend++; + } + } + + // post all receives for datum counts + + for (int i = 0; i < nrecv; i++) + MPI_Irecv(&recvcount[i],1,MPI_INT,MPI_ANY_SOURCE,0,comm,&request[i]); + + // barrier to insure receives are posted + + MPI_Barrier(comm); + + // send each datum count, packing buf with needed datums + + for (int i = 0; i < nsend; i++) + MPI_Send(&sendcount[i],1,MPI_INT,sendproc[i],0,comm); + + // insure all MPI_ANY_SOURCE messages are received + // set recvproc + + if (nrecv) MPI_Waitall(nrecv,request,status); + for (int i = 0; i < nrecv; i++) recvproc[i] = status[i].MPI_SOURCE; + + // ndatumrecv = total datums received, including self + + ndatumrecv = 0; + for (int i = 0; i < nrecv; i++) + ndatumrecv += recvcount[i]; + if (self) ndatumrecv += sendcount[nsend]; + + // setup sendindices, including self + // counts = offset into sendindices for each proc I send to + // let sc0 = sendcount[0], sc1 = sendcount[1], etc + // sendindices[0:sc0-1] = indices of datums in 1st message + // sendindices[sc0:sc0+sc1-1] = indices of datums in 2nd message, etc + + counts[0] = 0; + for (int i = 1; i < nsend+self; i++) + counts[i] = counts[i-1] + sendcount[i-1]; + + for (int i = 0; i < n; i++) { + isend = list[proclist[i]]; + sendindices[counts[isend]++] = i; + } + + // clean up + + delete [] counts; + delete [] list; +} + +/* ---------------------------------------------------------------------- + n = size of each received datum + return total size in bytes of received data on this proc +------------------------------------------------------------------------- */ + +int Irregular::size(int n) +{ + if (patternflag == UNSET) error->all("Cannot size without pattern"); + sizestyle = SAME; + + nsize = n; + + nsendmax = 0; + for (int i = 0; i < nsend+self; i++) { + sendsize[i] = nsize * sendcount[i]; + if (i < nsend) nsendmax = MAX(nsendmax,sendsize[i]); + } + + for (int i = 0; i < nrecv; i++) recvsize[i] = nsize * recvcount[i]; + nbytesrecv = nsize * ndatumrecv; + + return nbytesrecv; +} + +/* ---------------------------------------------------------------------- + slength,rlength = size of each datum to send and recv + soffset = offset into eventual buffer of send data for each datum + soffset can be NULL, in which case will build sendoffset from slength + return total size in bytes of received data on this proc +------------------------------------------------------------------------- */ + +int Irregular::size(int *slength, int *soffset, int *rlength) +{ + if (patternflag == UNSET) error->all("Cannot size without pattern"); + sizestyle = VARYING; + + // store local copy of pointers to send lengths/offsets + // if soffset not provided, create local copy from slength + + sendsizedatum = slength; + + if (soffset == NULL) { + sendoffsetflag = 1; + sendoffset = (int *) memory->smalloc(ndatumsend*sizeof(int),"sendoffset"); + + if (ndatumsend) sendoffset[0] = 0; + for (int i = 1; i < ndatumsend; i++) + sendoffset[i] = sendoffset[i-1] + sendsizedatum[i-1]; + + } else { + if (sendoffsetflag) memory->sfree(sendoffset); + sendoffsetflag = 0; + sendoffset = soffset; + } + + nsendmax = 0; + int m = 0; + for (int i = 0; i < nsend+self; i++) { + sendsize[i] = 0; + for (int j = 0; j < sendcount[i]; j++) + sendsize[i] += sendsizedatum[sendindices[m++]]; + if (i < nsend) nsendmax = MAX(nsendmax,sendsize[i]); + } + + nbytesrecv = 0; + m = 0; + for (int i = 0; i < nrecv; i++) { + recvsize[i] = 0; + for (int j = 0; j < recvcount[i]; j++) recvsize[i] += rlength[m++]; + nbytesrecv += recvsize[i]; + } + if (self) nbytesrecv += sendsize[nsend]; + + return nbytesrecv; +} + +/* ---------------------------------------------------------------------- + wrapper on 2 versions of exchange +------------------------------------------------------------------------- */ + +void Irregular::exchange(char *sendbuf, char *recvbuf) +{ + if (sizestyle == SAME) exchange_same(sendbuf,recvbuf); + else if (sizestyle == VARYING) exchange_varying(sendbuf,recvbuf); + else error->all("Irregular size was not set"); +} + +/* ---------------------------------------------------------------------- + sendbuf = data to send + recvbuf = buffer to recv all data into + requires nsize,nsendmax,recvsize,sendsize be setup by size(int) +------------------------------------------------------------------------- */ + +void Irregular::exchange_same(char *sendbuf, char *recvbuf) +{ + // post all receives + + int recvoffset = 0; + for (int irecv = 0; irecv < nrecv; irecv++) { + MPI_Irecv(&recvbuf[recvoffset],recvsize[irecv],MPI_BYTE, + recvproc[irecv],0,comm,&request[irecv]); + recvoffset += recvsize[irecv]; + } + + // malloc buf for largest send + + char *buf = (char *) memory->smalloc(nsendmax,"buf"); + + // barrier to insure receives are posted + + MPI_Barrier(comm); + + // send each message, packing buf with needed datums + + int m = 0; + for (int isend = 0; isend < nsend; isend++) { + int bufoffset = 0; + for (int i = 0; i < sendcount[isend]; i++) { + memcpy(&buf[bufoffset],&sendbuf[nsize*sendindices[m++]],nsize); + bufoffset += nsize; + } + MPI_Send(buf,sendsize[isend],MPI_BYTE,sendproc[isend],0,comm); + } + + // copy self data directly from sendbuf to recvbuf + + if (self) + for (int i = 0; i < sendcount[nsend]; i++) { + memcpy(&recvbuf[recvoffset],&sendbuf[nsize*sendindices[m++]],nsize); + recvoffset += nsize; + } + + // free send buffer + + memory->sfree(buf); + + // wait on all incoming messages + + if (nrecv) MPI_Waitall(nrecv,request,status); +} + +/* ---------------------------------------------------------------------- + sendbuf = data to send + recvbuf = buffer to recv all data into + requires nsendmax,recvsize,sendsize,sendoffset,sendsizedatum + be setup by size(int *, int *) +------------------------------------------------------------------------- */ + +void Irregular::exchange_varying(char *sendbuf, char *recvbuf) +{ + // post all receives + + int recvoffset = 0; + for (int irecv = 0; irecv < nrecv; irecv++) { + MPI_Irecv(&recvbuf[recvoffset],recvsize[irecv],MPI_BYTE, + recvproc[irecv],0,comm,&request[irecv]); + recvoffset += recvsize[irecv]; + } + + // malloc buf for largest send + + char *buf = (char *) memory->smalloc(nsendmax,"buf"); + + // barrier to insure receives are posted + + MPI_Barrier(comm); + + // send each message, packing buf with needed datums + + int index; + int m = 0; + for (int isend = 0; isend < nsend; isend++) { + int bufoffset = 0; + for (int i = 0; i < sendcount[isend]; i++) { + index = sendindices[m++]; + memcpy(&buf[bufoffset],&sendbuf[sendoffset[index]],sendsizedatum[index]); + bufoffset += sendsizedatum[index]; + } + MPI_Send(buf,sendsize[isend],MPI_BYTE,sendproc[isend],0,comm); + } + + // copy self data directly from sendbuf to recvbuf + + if (self) + for (int i = 0; i < sendcount[nsend]; i++) { + index = sendindices[m++]; + memcpy(&recvbuf[recvoffset],&sendbuf[sendoffset[index]], + sendsizedatum[index]); + recvoffset += sendsizedatum[index]; + } + + // free send buffer + + memory->sfree(buf); + + // wait on all incoming messages + + if (nrecv) MPI_Waitall(nrecv,request,status); +} + +/* ---------------------------------------------------------------------- */ + +void Irregular::init() +{ + sendoffsetflag = 0; + sendproc = sendcount = sendsize = sendindices = NULL; + sendoffset = NULL; + recvproc = recvcount = recvsize = NULL; + request = NULL; + status = NULL; +} + +/* ---------------------------------------------------------------------- */ + +void Irregular::deallocate() +{ + delete [] sendproc; + delete [] sendcount; + delete [] sendsize; + memory->sfree(sendindices); + if (sendoffsetflag) memory->sfree(sendoffset); + + delete [] recvproc; + delete [] recvcount; + delete [] recvsize; + + delete [] request; + delete [] status; +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.d new file mode 100644 index 00000000..e860103c --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.d @@ -0,0 +1,111 @@ +irregular.o: irregular.cpp /usr/include/stdio.h /usr/include/features.h \ + /usr/include/bits/predefs.h /usr/include/sys/cdefs.h \ + /usr/include/bits/wordsize.h /usr/include/gnu/stubs.h \ + /usr/include/gnu/stubs-64.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/include/bits/types.h /usr/include/bits/typesizes.h \ + /usr/include/libio.h /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/endian.h \ + /usr/include/bits/endian.h /usr/include/bits/byteswap.h \ + /usr/include/xlocale.h /usr/include/sys/types.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/sys/sysmacros.h /usr/include/bits/pthreadtypes.h \ + /usr/include/alloca.h /usr/include/bits/stdlib.h /usr/include/string.h \ + /usr/include/bits/string3.h irregular.h /usr/lib/openmpi/include/mpi.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h /usr/include/c++/4.4/iostream \ + /usr/include/c++/4.4/ostream /usr/include/c++/4.4/ios \ + /usr/include/c++/4.4/iosfwd /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/signal.h /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h memory.h \ + error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.h new file mode 100644 index 00000000..ea649533 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.h @@ -0,0 +1,58 @@ +#ifndef IRREGULAR_H +#define IRREGULAR_H + +#include "mpi.h" + +class Irregular { + public: + Irregular(MPI_Comm); + ~Irregular(); + + void pattern(int, int *); + int size(int); + int size(int *, int *, int *); + void exchange(char *, char *); + + private: + int me,nprocs; + + int patternflag; // UNSET,SET + int sizestyle; // NONE,SAME,VARYING + + int self; // 0 = no data to copy to self, 1 = yes + + int ndatumsend; // # of datums to send w/ self + int ndatumrecv; // # of datums to recv w/ self + int nbytesrecv; // total bytes in received data w/ self + int nsend; // # of messages to send w/out self + int nrecv; // # of messages to recv w/out self + int nsendmax; // # of bytes in largest send message, w/out self + + int *sendproc; // list of procs to send to w/out self + int *sendcount; // # of datums to send to each proc w/ self + int *sendsize; // # of bytes to send to each proc w/ self + int *sendindices; // indices of datums to send to each proc w/ self + + int nsize; // size of every datum in bytes (SAME) + int *sendsizedatum; // bytes in each datum to send w/ self (VARYING) + int *sendoffset; // byte offset to where each datum starts w/ self + int sendoffsetflag; // 1 if allocated sendoffset, 0 if passed in + + int *recvproc; // list of procs to recv from w/out self + int *recvcount; // # of datums to recv from each proc w/out self + int *recvsize; // # of bytes to recv from each proc w/out self + + MPI_Request *request; // MPI requests for posted recvs + MPI_Status *status; // MPI statuses for Waitall + MPI_Comm comm; // MPI communicator for all communication + + class Memory *memory; + class Error *error; + + void exchange_same(char *, char *); + void exchange_varying(char *, char *); + void init(); + void deallocate(); +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.o new file mode 100644 index 00000000..590607a5 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/irregular.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.cpp new file mode 100644 index 00000000..7490537e --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.cpp @@ -0,0 +1,249 @@ +#include "stdlib.h" +#include "string.h" +#include "lammps_data_write.h" +#include "memory.h" +#include "error.h" + +#define DELTA 4; + +enum{INT,DOUBLE,DOUBLE2}; + +/* ---------------------------------------------------------------------- */ + +LAMMPSDataWrite::LAMMPSDataWrite(MPI_Comm caller_comm) : Send2One(caller_comm) +{ + outfile = NULL; + + nheader = maxheader = 0; + format = NULL; + headtype = NULL; + ihead = NULL; + dhead = NULL; + ddhead = NULL; + + nper = maxper = 0; + atomtype = NULL; + ivec = NULL; + dvec = NULL; + stride = NULL; +} + +/* ---------------------------------------------------------------------- */ + +LAMMPSDataWrite::~LAMMPSDataWrite() +{ + delete [] outfile; + + for (int i = 0; i < nheader; i++) delete [] format[i]; + memory->sfree(format); + memory->sfree(headtype); + memory->sfree(ihead); + memory->sfree(dhead); + memory->destroy_2d_double_array(ddhead); + + memory->sfree(atomtype); + memory->sfree(ivec); + memory->sfree(dvec); + memory->sfree(stride); +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::pre() +{ + if (me == 0) { + fp = fopen(outfile,"w"); + if (fp == NULL) + error->one("Could not open data_write output file"); + } + + if (me == 0) { + fprintf(fp,"%s","LAMMPS data file\n\n"); + for (int i = 0; i < nheader; i++) + if (headtype[i] == INT) fprintf(fp,format[i],ihead[i]); + else if (headtype[i] == DOUBLE) fprintf(fp,format[i],dhead[i]); + else if (headtype[i] == DOUBLE2) fprintf(fp,format[i], + ddhead[i][0],ddhead[i][1]); + fprintf(fp,"\nAtoms\n\n"); + } +} + +/* ---------------------------------------------------------------------- */ + +int LAMMPSDataWrite::size() +{ + return nper*nlocal*sizeof(double); +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::pack(char *cbuf) +{ + int i,j; + + double *dbuf = (double *) cbuf; + + int m = 0; + for (i = 0; i < nlocal; i++) + for (j = 0; j < nper; j++) { + if (i == 0) { + if (atomtype[j] == 0) + printf("AT %d %d %p %d\n", + atomtype[j],stride[j],ivec[j],ivec[j][0]); + else + printf("AT %d %d %p %g\n", + atomtype[j],stride[j],dvec[j],dvec[j][0]); + } + if (atomtype[j] == INT) dbuf[m++] = ivec[j][i*stride[j]]; + else if (atomtype[j] == DOUBLE) dbuf[m++] = dvec[j][i*stride[j]]; + } +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::process(int nbuf, char *cbuf) +{ + int i,j; + + double *dbuf = (double *) cbuf; + int n = nbuf/nper/sizeof(double); + + int m = 0; + for (i = 0; i < n; i++) { + for (j = 0; j < nper; j++) { + double dvalue = dbuf[m++]; + if (atomtype[j] == INT) fprintf(fp,"%d ",static_cast (dvalue)); + else if (atomtype[j] == DOUBLE) fprintf(fp,"%g ",dvalue); + } + fprintf(fp,"\n"); + } +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::post() +{ + if (me == 0) fclose(fp); +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::file(char *fname) +{ + int n = strlen(fname) + 1; + outfile = new char[n]; + strcpy(outfile,fname); +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::header(char *str, int ivalue) +{ + if (nheader == maxheader) grow_header(); + int n = strlen(str) + 2; + format[nheader] = new char[n]; + strcpy(format[nheader],str); + format[nheader][n-2] = '\n'; + format[nheader][n-1] = '\0'; + headtype[nheader] = INT; + ihead[nheader] = ivalue; + nheader++; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::header(char *str, double dvalue) +{ + if (nheader == maxheader) grow_header(); + int n = strlen(str) + 2; + format[nheader] = new char[n]; + strcpy(format[nheader],str); + format[nheader][n-2] = '\n'; + format[nheader][n-1] = '\0'; + headtype[nheader] = DOUBLE; + dhead[nheader] = dvalue; + nheader++; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::header(char *str, double dvalue1, double dvalue2) +{ + if (nheader == maxheader) grow_header(); + int n = strlen(str) + 2; + format[nheader] = new char[n]; + strcpy(format[nheader],str); + format[nheader][n-2] = '\n'; + format[nheader][n-1] = '\0'; + headtype[nheader] = DOUBLE2; + ddhead[nheader][0] = dvalue1; + ddhead[nheader][1] = dvalue2; + nheader++; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::atoms(int n) +{ + nlocal = n; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::atoms(int *vec) +{ + if (nper == maxper) grow_peratom(); + atomtype[nper] = INT; + ivec[nper] = vec; + stride[nper] = 1; + nper++; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::atoms(double *vec) +{ + if (nper == maxper) grow_peratom(); + atomtype[nper] = DOUBLE; + dvec[nper] = vec; + stride[nper] = 1; + nper++; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::atoms(int n, double **vec) +{ + if (nper+n >= maxper) grow_peratom(); + for (int i = 0; i < n; i++) { + atomtype[nper] = DOUBLE; + dvec[nper] = &vec[0][i]; + stride[nper] = n; + nper++; + } +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::grow_header() +{ + int n = maxheader + DELTA; + format = (char **) memory->srealloc(format,n*sizeof(char *),"ldw:format"); + headtype = (int *) memory->srealloc(headtype,n*sizeof(int),"ldw:headtype"); + ihead = (int *) memory->srealloc(ihead,n*sizeof(int),"ldw:ihead"); + dhead = (double *) memory->srealloc(dhead,n*sizeof(double),"ldw:dhead"); + ddhead = memory->grow_2d_double_array(ddhead,n,2,"ldw:ddhead"); + maxheader = n; +} + +/* ---------------------------------------------------------------------- */ + +void LAMMPSDataWrite::grow_peratom() +{ + int n = maxper + DELTA; + atomtype = (int *) memory->srealloc(atomtype,n*sizeof(int *),"ldw:atomtype"); + ivec = (int **) memory->srealloc(ivec,n*sizeof(int *),"ldw:ihead"); + dvec = (double **) memory->srealloc(dvec,n*sizeof(double *),"ldw:dhead"); + stride = (int *) memory->srealloc(stride,n*sizeof(int *),"ldw:stride"); + maxper = n; +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.d new file mode 100644 index 00000000..fd03bf8e --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.d @@ -0,0 +1,113 @@ +lammps_data_write.o: lammps_data_write.cpp /usr/include/stdlib.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/include/bits/waitflags.h /usr/include/bits/waitstatus.h \ + /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/xlocale.h \ + /usr/include/sys/types.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/time.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/bits/sigset.h /usr/include/bits/time.h \ + /usr/include/sys/sysmacros.h /usr/include/bits/pthreadtypes.h \ + /usr/include/alloca.h /usr/include/bits/stdlib.h /usr/include/string.h \ + /usr/include/bits/string3.h lammps_data_write.h send2one.h \ + /usr/lib/openmpi/include/mpi.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/libio.h /usr/include/_G_config.h \ + /usr/include/wchar.h /usr/include/bits/stdio_lim.h \ + /usr/include/bits/sys_errlist.h /usr/include/bits/stdio.h \ + /usr/include/bits/stdio2.h /usr/include/c++/4.4/iostream \ + /usr/include/c++/4.4/ostream /usr/include/c++/4.4/ios \ + /usr/include/c++/4.4/iosfwd /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/bits/wchar2.h \ + /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/bits/sched.h \ + /usr/include/signal.h /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h memory.h \ + error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.h new file mode 100644 index 00000000..5eda0f40 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.h @@ -0,0 +1,48 @@ +#ifndef LAMMPS_DATA_WRITE_H +#define LAMMPS_DATA_WRITE_H + +#include "send2one.h" +#include "stdio.h" + +class LAMMPSDataWrite : public Send2One { + public: + LAMMPSDataWrite(MPI_Comm); + ~LAMMPSDataWrite(); + + void pre(); + int size(); + void pack(char *); + void process(int, char *); + void post(); + + void file(char *); + void header(char *, int); + void header(char *, double); + void header(char *, double, double); + void atoms(int); + void atoms(int *); + void atoms(double *); + void atoms(int, double **); + + private: + char *outfile; + int nlocal; + FILE *fp; + + int nheader,maxheader; + char **format; + int *headtype,*ihead; + double *dhead; + double **ddhead; + + int nper,maxper; + int *atomtype; + int **ivec; + double **dvec; + int *stride; + + void grow_header(); + void grow_peratom(); +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.o new file mode 100644 index 00000000..60b76396 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/lammps_data_write.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/libcouple.a b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/libcouple.a new file mode 100644 index 00000000..b68d5eb3 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/libcouple.a differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.cpp new file mode 100644 index 00000000..305d440e --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.cpp @@ -0,0 +1,317 @@ +#include "mpi.h" +#include "stdio.h" +#include "many2many.h" +#include "irregular.h" +#include "memory.h" +#include "error.h" + +#include + +#define MAX(A,B) ((A) > (B)) ? (A) : (B) + +/* ---------------------------------------------------------------------- */ + +Many2Many::Many2Many(MPI_Comm caller) +{ + comm = caller; + MPI_Comm_rank(comm,&me); + MPI_Comm_size(comm,&nprocs); + + memory = new Memory(comm); + error = new Error(comm); + + src_own = dest_own = NULL; + src_off = dest_off = NULL; + src_iwork = dest_iwork = NULL; + src_dwork = dest_dwork = NULL; + irregular = NULL; +} + +/* ---------------------------------------------------------------------- */ + +Many2Many::~Many2Many() +{ + delete memory; + delete error; + deallocate(); +} + +/* ---------------------------------------------------------------------- + create a many2many pattern, deallocating any previous pattern + each proc will contribute nsrc items with IDs listed in id_src + each proc will receive ndest items with IDs listed in id_dest + only sets up communication via rendezvous algorithm and Irregular class + if id_src does not match id_dest on all procs +------------------------------------------------------------------------- */ + +void Many2Many::setup(int nsrc, int *id_src, int ndest, int *id_dest) +{ + int i,j,isrc,idest,nsend,nrecv; + int *proclist,*work; + std::map hash; + std::map::iterator loc; + + // free any previous many2many info + + deallocate(); + + // allocate on-proc and off-proc index lists + + src_own = + (int *) memory->smalloc(nsrc*sizeof(int),"many2many:src_own"); + dest_own = + (int *) memory->smalloc(ndest*sizeof(int),"many2many:dest_own"); + src_off = + (int *) memory->smalloc(nsrc*sizeof(int),"many2many:src_off"); + dest_off = + (int *) memory->smalloc(ndest*sizeof(int),"many2many:dest_off"); + + // store destination IDs in hash + + for (int i = 0; i < ndest; i++) + hash.insert(std::pair (id_dest[i],i)); + + // src_own, dest_own = list of IDs in both src and dest + // nsrc_off = list of IDs in src owned by other procs + + nown = nsrc_off = 0; + nsrc_off = 0; + for (i = 0; i < nsrc; i++) { + loc = hash.find(id_src[i]); + if (loc != hash.end()) { + src_own[nown] = i; + dest_own[nown] = loc->second; + nown++; + } else src_off[nsrc_off++] = i; + } + + // all done if all procs have one-to-one mapping of src and dest IDs + // else figure out irregular comm needed + + int flag = 0; + if (nown == nsrc && nown == ndest) flag = 1; + int flagall; + MPI_Allreduce(&flag,&flagall,1,MPI_INT,MPI_MIN,comm); + if (flagall) return; + + // ndest_off = list of IDs in dest owned by other procs + + work = (int *) memory->smalloc(ndest*sizeof(int),"many2many:work"); + + for (i = 0; i < ndest; i++) work[i] = 0; + for (i = 0; i < nown; i++) work[dest_own[i]] = 1; + + ndest_off = 0; + for (i = 0; i < ndest; i++) + if (work[i] == 0) dest_off[ndest_off++] = i; + + memory->sfree(work); + + // realloc off-proc arrays to smaller size + + src_off = (int *) + memory->srealloc(src_off,nsrc_off*sizeof(int),"many2many:src_off"); + dest_off = (int *) + memory->srealloc(dest_off,ndest_off*sizeof(int),"many2many:dest_off"); + + // send off-proc src and dest Datums to 3rd-party proc via irregular comm + // proc = ID % nprocs + + nsend = nsrc_off + ndest_off; + proclist = new int[nsend]; + Datum1 *send1 = new Datum1[nsend]; + + for (i = 0; i < nsrc_off; i++) { + proclist[i] = id_src[src_off[i]] % nprocs; + send1[i].id = id_src[src_off[i]]; + send1[i].proc = me; + send1[i].index = src_off[i]; + } + for (i = 0, j = nsrc_off; i < ndest_off; i++, j++) { + proclist[j] = id_dest[dest_off[i]] % nprocs; + send1[j].id = -id_dest[dest_off[i]]; + send1[j].proc = me; + send1[j].index = dest_off[i]; + } + + irregular = new Irregular(comm); + irregular->pattern(nsend,proclist); + nrecv = irregular->size(sizeof(Datum1)) / sizeof(Datum1); + Datum1 *recv1 = new Datum1[nrecv]; + irregular->exchange((char *) send1, (char *) recv1); + delete irregular; + delete [] proclist; + + // as 3rd-party proc, now have matching pairs of off-proc IDs + // store src IDs (which are positive) in hash + // loop over dest IDs (which are negative) to find matches + // send match info back to src procs via a 2nd irregular comm + + nsend = nrecv/2; + proclist = new int[nsend]; + Datum2 *send2 = new Datum2[nsend]; + nsend = 0; + + hash.clear(); + for (isrc = 0; isrc < nrecv; isrc++) + if (recv1[isrc].id > 0) + hash.insert(std::pair (recv1[isrc].id,isrc)); + + for (idest = 0; idest < nrecv; idest++) + if (recv1[idest].id < 0) { + loc = hash.find(-recv1[idest].id); + if (loc != hash.end()) { + isrc = loc->second; + proclist[nsend] = recv1[isrc].proc; + send2[nsend].slocal = recv1[isrc].index; + send2[nsend].dlocal = recv1[idest].index; + send2[nsend].dproc = recv1[idest].proc; + nsend++; + } else error->one("Did not receive matching src/dest ID"); + } + + irregular = new Irregular(comm); + irregular->pattern(nsend,proclist); + nrecv = irregular->size(sizeof(Datum2)) / sizeof(Datum2); + Datum2 *recv2 = new Datum2[nrecv]; + irregular->exchange((char *) send2, (char *) recv2); + delete irregular; + delete [] proclist; + + // use list of received src->dest Datums to build final irregular commm + // irregular comm will communicate off-proc info from src to dest directly + // work = local indices of dest IDs to send initially + + nsend = nrecv; + proclist = new int[nsend]; + work = new int[nsend]; + + for (i = 0; i < nrecv; i++) { + src_off[i] = recv2[i].slocal; + work[i] = recv2[i].dlocal; + proclist[i] = recv2[i].dproc; + } + + irregular = new Irregular(comm); + irregular->pattern(nsend,proclist); + + // send receiver's local indices + // receiver stores them as indirection list in dest_off + + nrecv = irregular->size(sizeof(int)) / sizeof(int); + irregular->exchange((char *) work, (char *) dest_off); + + delete [] proclist; + delete [] work; + + // create work arrays for data exchange of int/double data + + src_iwork = + (int *) memory->smalloc(nsrc_off*sizeof(int),"many2many:src_iwork"); + dest_iwork = + (int *) memory->smalloc(ndest_off*sizeof(int),"many2many:dest_iwork"); + src_dwork = + (double *) memory->smalloc(nsrc_off*sizeof(double),"many2many:src_dwork"); + dest_dwork = + (double *) memory->smalloc(ndest_off*sizeof(double), + "many2many:dest_dwork"); + + // clean up + + delete [] send1; + delete [] recv1; + delete [] send2; + delete [] recv2; + + // debug checks for full coverage of srd/dest - can delete eventually + + work = new int[MAX(nsrc,ndest)]; + + for (i = 0; i < nsrc; i++) work[i] = 0; + for (i = 0; i < nown; i++) work[src_own[i]]++; + for (i = 0; i < nsrc_off; i++) work[src_off[i]]++; + for (i = 0; i < nsrc; i++) + if (work[i] != 1) { + char str[128]; + sprintf(str,"BAD SRC %d: %d %d\n",me,i,work[i]); + error->one(str); + } + + for (i = 0; i < ndest; i++) work[i] = 0; + for (i = 0; i < nown; i++) work[dest_own[i]]++; + for (i = 0; i < ndest_off; i++) work[dest_off[i]]++; + for (i = 0; i < ndest; i++) + if (work[i] != 1) { + char str[128]; + sprintf(str,"BAD DEST %d: %d %d\n",me,i,work[i]); + error->one(str); + } + + delete [] work; +} + +/* ---------------------------------------------------------------------- + transfer one src entity to dest entity, matched by IDs in create() + operates on an int vector +------------------------------------------------------------------------- */ + +void Many2Many::exchange(int *src, int *dest) +{ + int i; + + // copy on-proc info + + for (i = 0; i < nown; i++) + dest[dest_own[i]] = src[src_own[i]]; + + // communicate off-proc info + // user src_off and dest_off to pack/unpack data + + if (irregular) { + int nrecv = irregular->size(sizeof(int)) / sizeof(int); + for (i = 0; i < nsrc_off; i++) src_iwork[i] = src[src_off[i]]; + irregular->exchange((char *) src_iwork, (char *) dest_iwork); + for (i = 0; i < ndest_off; i++) dest[dest_off[i]] = dest_iwork[i]; + } +} + +/* ---------------------------------------------------------------------- + transfer one src entity to dest entity, matched by IDs in create() + operates on a double vector +------------------------------------------------------------------------- */ + +void Many2Many::exchange(double *src, double *dest) +{ + int i; + + // copy on-proc info + + for (int i = 0; i < nown; i++) + dest[dest_own[i]] = src[src_own[i]]; + + // communicate off-proc info + // user src_off and dest_off to pack/unpack data + + if (irregular) { + int nrecv = irregular->size(sizeof(double)) / sizeof(double); + for (i = 0; i < nsrc_off; i++) src_dwork[i] = src[src_off[i]]; + irregular->exchange((char *) src_dwork, (char *) dest_dwork); + for (i = 0; i < ndest_off; i++) dest[dest_off[i]] = dest_dwork[i]; + } +} + +/* ---------------------------------------------------------------------- */ + +void Many2Many::deallocate() +{ + memory->sfree(src_own); + memory->sfree(dest_own); + memory->sfree(src_off); + memory->sfree(dest_off); + memory->sfree(src_iwork); + memory->sfree(dest_iwork); + memory->sfree(src_dwork); + memory->sfree(dest_dwork); + + delete irregular; +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.d new file mode 100644 index 00000000..d0eeb9af --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.d @@ -0,0 +1,108 @@ +many2many.o: many2many.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h many2many.h \ + irregular.h memory.h error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.h new file mode 100644 index 00000000..a0c4a2b4 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.h @@ -0,0 +1,47 @@ +#ifndef MANY2MANY_H +#define MANY2MANY_H + +#include "mpi.h" + +class Many2Many { + public: + Many2Many(MPI_Comm); + ~Many2Many(); + + void setup(int, int *, int, int *); + void exchange(int *, int *); + void exchange(double *, double *); + + protected: + int me,nprocs; + MPI_Comm comm; + class Memory *memory; + class Error *error; + + int nown; // # of IDs common to src and dest + int nsrc_off,ndest_off; // # of off-processor IDs + + int *src_own,*dest_own; // indices of the owned IDs + int *src_off,*dest_off; // indices of the off-proc IDs + + int *src_iwork,*dest_iwork; // work arrays for comm of ints + double *src_dwork,*dest_dwork; // work arrays for comm of doubles + + class Irregular *irregular; // irregular comm from src->dest + + struct Datum1 { + int id; // src or dest global ID + int proc; // owning proc + int index; // local index on owning proc + }; + + struct Datum2 { + int slocal; // local index of src ID on sending proc + int dlocal; // local index of dest ID on receiving proc + int dproc; // receiving proc + }; + + void deallocate(); +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.o new file mode 100644 index 00000000..ccfebb73 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2many.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.cpp new file mode 100644 index 00000000..5b5467aa --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.cpp @@ -0,0 +1,103 @@ +#include "mpi.h" +#include "stdio.h" +#include "stdlib.h" +#include "many2one.h" +#include "memory.h" + +/* ---------------------------------------------------------------------- */ + +Many2One::Many2One(MPI_Comm caller_comm) +{ + comm = caller_comm; + MPI_Comm_rank(comm,&me); + MPI_Comm_size(comm,&nprocs); + + memory = new Memory(comm); + + if (me == 0) { + counts = new int[nprocs]; + multicounts = new int[nprocs]; + displs = new int[nprocs]; + multidispls = new int[nprocs]; + } else counts = multicounts = displs = multidispls = NULL; + + idall = NULL; +} + +/* ---------------------------------------------------------------------- */ + +Many2One::~Many2One() +{ + delete memory; + + delete [] counts; + delete [] multicounts; + delete [] displs; + delete [] multidispls; + + memory->sfree(idall); +} + +/* ---------------------------------------------------------------------- */ + +void Many2One::setup(int nsrc_in, int *id, int ndest) +{ + nsrc = nsrc_in; + MPI_Allreduce(&nsrc,&nall,1,MPI_INT,MPI_SUM,comm); + MPI_Gather(&nsrc,1,MPI_INT,counts,1,MPI_INT,0,comm); + + if (me == 0) { + displs[0] = 0; + for (int i = 1; i < nprocs; i++) + displs[i] = displs[i-1] + counts[i-1]; + } + + // gather IDs into idall + + idall = NULL; + if (me == 0) + idall = (int *) memory->smalloc(nall*sizeof(int),"many2one:idall"); + MPI_Gatherv(id,nsrc,MPI_INT,idall,counts,displs,MPI_INT,0,comm); +} + +/* ---------------------------------------------------------------------- */ + +void Many2One::gather(double *src, int n, double *dest) +{ + int i,j,ii,jj,m; + + if (me == 0) + for (int i = 0; i < nprocs; i++) { + multicounts[i] = n*counts[i]; + multidispls[i] = n*displs[i]; + } + + // allgather src into desttmp + + double *desttmp = NULL; + if (me == 0) + desttmp = (double *) memory->smalloc(n*nall*sizeof(double), + "many2one:idsttmp"); + MPI_Gatherv(src,n*nsrc,MPI_DOUBLE,desttmp,multicounts,multidispls, + MPI_DOUBLE,0,comm); + + // use idall to move datums from desttmp to dest + + if (me == 0) { + if (n == 1) + for (i = 0; i < nall; i++) { + j = idall[i] - 1; + dest[j] = desttmp[i]; + } + else + for (i = 0; i < nall; i++) { + j = idall[i] - 1; + ii = n*i; + jj = n*j; + for (m = 0; m < n; m++) + dest[jj++] = desttmp[ii++]; + } + } + + memory->sfree(desttmp); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.d new file mode 100644 index 00000000..90f9b44d --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.d @@ -0,0 +1,112 @@ +many2one.o: many2one.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h many2one.h memory.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.h new file mode 100644 index 00000000..7e487ec4 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.h @@ -0,0 +1,25 @@ +#ifndef MANY2ONE_H +#define MANY2ONE_H + +#include "mpi.h" + +class Many2One { + public: + Many2One(MPI_Comm); + ~Many2One(); + + void setup(int, int *, int); + void gather(double *, int, double *); + + protected: + int me,nprocs; + MPI_Comm comm; + class Memory *memory; + + int nsrc,nall; + int *counts,*multicounts; + int *displs,*multidispls; + int *idall; +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.o new file mode 100644 index 00000000..64563a51 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/many2one.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.cpp new file mode 100644 index 00000000..7377fac7 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.cpp @@ -0,0 +1,120 @@ +#include "mpi.h" +#include "stdlib.h" +#include "stdio.h" +#include "memory.h" +#include "error.h" + +/* ---------------------------------------------------------------------- */ + +Memory::Memory(MPI_Comm comm) +{ + error = new Error(comm); +} + +/* ---------------------------------------------------------------------- */ + +Memory::~Memory() +{ + delete error; +} + +/* ---------------------------------------------------------------------- + safe malloc +------------------------------------------------------------------------- */ + +void *Memory::smalloc(int n, const char *name) +{ + if (n == 0) return NULL; + void *ptr = malloc(n); + if (ptr == NULL) { + char str[128]; + sprintf(str,"Failed to allocate %d bytes for array %s",n,name); + error->one(str); + } + return ptr; +} + +/* ---------------------------------------------------------------------- + safe free +------------------------------------------------------------------------- */ + +void Memory::sfree(void *ptr) +{ + if (ptr == NULL) return; + free(ptr); +} + +/* ---------------------------------------------------------------------- + safe realloc +------------------------------------------------------------------------- */ + +void *Memory::srealloc(void *ptr, int n, const char *name) +{ + if (n == 0) { + sfree(ptr); + return NULL; + } + + ptr = realloc(ptr,n); + if (ptr == NULL) { + char str[128]; + sprintf(str,"Failed to reallocate %d bytes for array %s",n,name); + error->one(str); + } + return ptr; +} + +/* ---------------------------------------------------------------------- + create a 2d double array +------------------------------------------------------------------------- */ + +double **Memory::create_2d_double_array(int n1, int n2, const char *name) + +{ + double *data = (double *) smalloc(n1*n2*sizeof(double),name); + double **array = (double **) smalloc(n1*sizeof(double *),name); + + int n = 0; + for (int i = 0; i < n1; i++) { + array[i] = &data[n]; + n += n2; + } + + return array; +} + +/* ---------------------------------------------------------------------- + grow or shrink 1st dim of a 2d double array + last dim must stay the same + if either dim is 0, return NULL +------------------------------------------------------------------------- */ + +double **Memory::grow_2d_double_array(double **array, + int n1, int n2, const char *name) + +{ + if (array == NULL) return create_2d_double_array(n1,n2,name); + + double *data = (double *) srealloc(array[0],n1*n2*sizeof(double),name); + array = (double **) srealloc(array,n1*sizeof(double *),name); + + int n = 0; + for (int i = 0; i < n1; i++) { + array[i] = &data[n]; + n += n2; + } + + return array; +} + +/* ---------------------------------------------------------------------- + free a 2d double array +------------------------------------------------------------------------- */ + +void Memory::destroy_2d_double_array(double **array) + +{ + if (array == NULL) return; + sfree(array[0]); + sfree(array); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.d new file mode 100644 index 00000000..a1e300f2 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.d @@ -0,0 +1,112 @@ +memory.o: memory.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h memory.h error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.h new file mode 100644 index 00000000..d6efa778 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.h @@ -0,0 +1,23 @@ +#ifndef MEMORY_H +#define MEMORY_H + +#include "mpi.h" + +class Memory { + public: + Memory(MPI_Comm); + ~Memory(); + + void *smalloc(int n, const char *); + void sfree(void *); + void *srealloc(void *, int n, const char *name); + + double **create_2d_double_array(int, int, const char *); + double **grow_2d_double_array(double **, int, int, const char *); + void destroy_2d_double_array(double **); + + private: + class Error *error; +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.o new file mode 100644 index 00000000..481444b4 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/memory.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.cpp new file mode 100644 index 00000000..e26bf708 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.cpp @@ -0,0 +1,76 @@ +#include "mpi.h" +#include "one2many.h" +#include "memory.h" + +#include + +/* ---------------------------------------------------------------------- */ + +One2Many::One2Many(MPI_Comm caller_comm) +{ + comm = caller_comm; + MPI_Comm_rank(comm,&me); + MPI_Comm_size(comm,&nprocs); + + memory = new Memory(comm); + hash = new std::map(); +} + +/* ---------------------------------------------------------------------- */ + +One2Many::~One2Many() +{ + delete memory; + delete hash; +} + +/* ---------------------------------------------------------------------- */ + +void One2Many::setup(int nsrc_in, int ndest, int *id) +{ + nsrc = nsrc_in; + + // store my local IDs in hash + + hash->clear(); + for (int i = 0; i < ndest; i++) + hash->insert(std::pair (id[i],i)); +} + +/* ---------------------------------------------------------------------- */ + +void One2Many::scatter(double *src, int n, double *dest) +{ + int i,j,k,m; + + // allocate src on procs that don't have it + + int flag = 0; + if (src == NULL) { + src = (double *) memory->smalloc(n*nsrc*sizeof(double),"one2many:src"); + flag = 1; + } + + // broadcast src from 0 to other procs + + MPI_Bcast(src,n*nsrc,MPI_DOUBLE,0,comm); + + // each proc loops over entire src + // if I own the global ID, copy src values into dest + + std::map::iterator loc; + for (m = 1; m <= nsrc; m++) { + loc = hash->find(m); + if (loc == hash->end()) continue; + i = n*loc->second; + j = 3*(m-1); + if (n == 1) dest[i] = src[j]; + else + for (k = 0; k < n; k++) + dest[i++] = src[j++]; + } + + // free locally allocated src + + if (flag) memory->sfree(src); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.d new file mode 100644 index 00000000..77550934 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.d @@ -0,0 +1,108 @@ +one2many.o: one2many.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h one2many.h \ + memory.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.h new file mode 100644 index 00000000..97978661 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.h @@ -0,0 +1,24 @@ +#ifndef ONE2MANY_H +#define ONE2MANY_H + +#include "mpi.h" + +#include + +class One2Many { + public: + One2Many(MPI_Comm); + ~One2Many(); + + void setup(int, int, int *); + void scatter(double *, int, double *); + + protected: + int me,nprocs; + MPI_Comm comm; + class Memory *memory; + std::map *hash; + int nsrc; +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.o new file mode 100644 index 00000000..c0bac0ef Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/one2many.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.cpp b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.cpp new file mode 100644 index 00000000..d969b467 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.cpp @@ -0,0 +1,83 @@ +#include "mpi.h" +#include "stdlib.h" +#include "stdio.h" +#include "send2one.h" +#include "memory.h" +#include "error.h" + +/* ---------------------------------------------------------------------- */ + +Send2One::Send2One(MPI_Comm caller_comm) +{ + comm = caller_comm; + MPI_Comm_rank(comm,&me); + MPI_Comm_size(comm,&nprocs); + + memory = new Memory(comm); + error = new Error(comm); + + buf = NULL; + maxbuf = 0; +} + +/* ---------------------------------------------------------------------- */ + +Send2One::~Send2One() +{ + delete memory; + delete error; + memory->sfree(buf); +} + +/* ---------------------------------------------------------------------- */ + +void Send2One::execute() +{ + int nme,nmax,nsize,ping; + MPI_Status status; + MPI_Request request; + + // pre-processing before ping loop + + pre(); + + // nme = size of data I contribute, in bytes + // nmax = max size of data on any proc, in bytes + // reallocate buf if necessary + + nme = size(); + MPI_Allreduce(&nme,&nmax,1,MPI_INT,MPI_MAX,comm); + + if (nmax > maxbuf) { + maxbuf = nmax; + memory->sfree(buf); + buf = (char *) memory->smalloc(maxbuf,"foo:buf"); + } + + // pack my data into buf + + pack(buf); + + // proc 0 pings each proc, receives its data + // all other procs wait for ping, send their data to proc 0 + // invoke process() to work with data + + if (me == 0) { + for (int iproc = 0; iproc < nprocs; iproc++) { + if (iproc) { + MPI_Irecv(buf,maxbuf,MPI_CHAR,iproc,0,comm,&request); + MPI_Send(&ping,0,MPI_INT,iproc,0,comm); + MPI_Wait(&request,&status); + MPI_Get_count(&status,MPI_CHAR,&nsize); + } else nsize = nme; + + process(nsize,buf); + } + + } else { + MPI_Recv(&ping,0,MPI_INT,0,0,comm,&status); + MPI_Rsend(buf,nme,MPI_CHAR,0,0,comm); + } + + post(); +} diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.d b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.d new file mode 100644 index 00000000..9dc1ae89 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.d @@ -0,0 +1,112 @@ +send2one.o: send2one.cpp /usr/lib/openmpi/include/mpi.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stddef.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/mpicxx.h \ + /usr/include/c++/4.4/map /usr/include/c++/4.4/bits/stl_tree.h \ + /usr/include/c++/4.4/bits/stl_algobase.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++config.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/os_defines.h \ + /usr/include/features.h /usr/include/bits/predefs.h \ + /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \ + /usr/include/gnu/stubs.h /usr/include/gnu/stubs-64.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/cpu_defines.h \ + /usr/include/c++/4.4/cstddef /usr/include/c++/4.4/bits/functexcept.h \ + /usr/include/c++/4.4/exception_defines.h \ + /usr/include/c++/4.4/bits/cpp_type_traits.h \ + /usr/include/c++/4.4/ext/type_traits.h \ + /usr/include/c++/4.4/ext/numeric_traits.h \ + /usr/include/c++/4.4/bits/stl_pair.h /usr/include/c++/4.4/bits/move.h \ + /usr/include/c++/4.4/bits/concept_check.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_types.h \ + /usr/include/c++/4.4/bits/stl_iterator_base_funcs.h \ + /usr/include/c++/4.4/bits/stl_iterator.h \ + /usr/include/c++/4.4/debug/debug.h /usr/include/c++/4.4/bits/allocator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++allocator.h \ + /usr/include/c++/4.4/ext/new_allocator.h /usr/include/c++/4.4/new \ + /usr/include/c++/4.4/exception /usr/include/c++/4.4/bits/stl_function.h \ + /usr/include/c++/4.4/backward/binders.h \ + /usr/include/c++/4.4/bits/stl_map.h \ + /usr/include/c++/4.4/initializer_list \ + /usr/include/c++/4.4/bits/stl_multimap.h /usr/include/c++/4.4/utility \ + /usr/include/c++/4.4/bits/stl_relops.h \ + /usr/lib/gcc/x86_64-linux-gnu/4.4.3/include/stdarg.h \ + /usr/include/stdio.h /usr/include/bits/types.h \ + /usr/include/bits/typesizes.h /usr/include/libio.h \ + /usr/include/_G_config.h /usr/include/wchar.h \ + /usr/include/bits/stdio_lim.h /usr/include/bits/sys_errlist.h \ + /usr/include/bits/stdio.h /usr/include/bits/stdio2.h \ + /usr/include/c++/4.4/iostream /usr/include/c++/4.4/ostream \ + /usr/include/c++/4.4/ios /usr/include/c++/4.4/iosfwd \ + /usr/include/c++/4.4/bits/stringfwd.h \ + /usr/include/c++/4.4/bits/postypes.h /usr/include/c++/4.4/cwchar \ + /usr/include/bits/wchar.h /usr/include/xlocale.h \ + /usr/include/bits/wchar2.h /usr/include/c++/4.4/bits/char_traits.h \ + /usr/include/c++/4.4/bits/localefwd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/c++locale.h \ + /usr/include/c++/4.4/clocale /usr/include/locale.h \ + /usr/include/bits/locale.h /usr/include/c++/4.4/cctype \ + /usr/include/ctype.h /usr/include/endian.h /usr/include/bits/endian.h \ + /usr/include/bits/byteswap.h /usr/include/c++/4.4/bits/ios_base.h \ + /usr/include/c++/4.4/ext/atomicity.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/gthr-default.h \ + /usr/include/pthread.h /usr/include/sched.h /usr/include/time.h \ + /usr/include/bits/sched.h /usr/include/bits/time.h /usr/include/signal.h \ + /usr/include/bits/sigset.h /usr/include/bits/pthreadtypes.h \ + /usr/include/bits/setjmp.h /usr/include/unistd.h \ + /usr/include/bits/posix_opt.h /usr/include/bits/environments.h \ + /usr/include/bits/confname.h /usr/include/getopt.h \ + /usr/include/bits/unistd.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/atomic_word.h \ + /usr/include/c++/4.4/bits/locale_classes.h /usr/include/c++/4.4/string \ + /usr/include/c++/4.4/bits/ostream_insert.h \ + /usr/include/c++/4.4/cxxabi-forced.h \ + /usr/include/c++/4.4/bits/basic_string.h \ + /usr/include/c++/4.4/bits/basic_string.tcc \ + /usr/include/c++/4.4/bits/locale_classes.tcc \ + /usr/include/c++/4.4/streambuf /usr/include/c++/4.4/bits/streambuf.tcc \ + /usr/include/c++/4.4/bits/basic_ios.h \ + /usr/include/c++/4.4/bits/locale_facets.h /usr/include/c++/4.4/cwctype \ + /usr/include/wctype.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_base.h \ + /usr/include/c++/4.4/bits/streambuf_iterator.h \ + /usr/include/c++/4.4/x86_64-linux-gnu/bits/ctype_inline.h \ + /usr/include/c++/4.4/bits/locale_facets.tcc \ + /usr/include/c++/4.4/bits/basic_ios.tcc \ + /usr/include/c++/4.4/bits/ostream.tcc /usr/include/c++/4.4/istream \ + /usr/include/c++/4.4/bits/istream.tcc \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/constants.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/exception.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/datatype_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/functions_inln.h \ + /usr/include/string.h /usr/include/bits/string3.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/request_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/comm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intracomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/topology_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/intercomm_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/group_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/op_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/errhandler_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/status_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/info_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/win_inln.h \ + /usr/lib/openmpi/include/openmpi/ompi/mpi/cxx/file_inln.h \ + /usr/include/stdlib.h /usr/include/bits/waitflags.h \ + /usr/include/bits/waitstatus.h /usr/include/sys/types.h \ + /usr/include/sys/select.h /usr/include/bits/select.h \ + /usr/include/sys/sysmacros.h /usr/include/alloca.h \ + /usr/include/bits/stdlib.h send2one.h memory.h error.h diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.h b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.h new file mode 100644 index 00000000..df25b5fb --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.h @@ -0,0 +1,29 @@ +#ifndef SEND2ONE_H +#define SEND2ONE_H + +#include "mpi.h" + +class Send2One { + public: + Send2One(MPI_Comm); + virtual ~Send2One(); + + void execute(); + + protected: + int me,nprocs; + MPI_Comm comm; + class Memory *memory; + class Error *error; + + int maxbuf; + char *buf; + + virtual void pre() = 0; + virtual int size() = 0; + virtual void pack(char *) = 0; + virtual void process(int, char *) = 0; + virtual void post() = 0; +}; + +#endif diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.o b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.o new file mode 100644 index 00000000..1cd72575 Binary files /dev/null and b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/library/send2one.o differ diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/twoWayM2M.C b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/twoWayM2M.C new file mode 100644 index 00000000..6298d243 --- /dev/null +++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayM2M/twoWayM2M.C @@ -0,0 +1,597 @@ +/*---------------------------------------------------------------------------*\ + CFDEMcoupling - Open Source CFD-DEM coupling + + CFDEMcoupling is part of the CFDEMproject + www.cfdem.com + Christoph Goniva, christoph.goniva@cfdem.com + Copyright 2009-2012 JKU Linz + Copyright 2012- DCS Computing GmbH, Linz +------------------------------------------------------------------------------- +License + This file is part of CFDEMcoupling. + + CFDEMcoupling is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation; either version 3 of the License, or (at your + option) any later version. + + CFDEMcoupling is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with CFDEMcoupling; if not, write to the Free Software Foundation, + Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Description + This code is designed to realize coupled CFD-DEM simulations using LIGGGHTS + and OpenFOAM(R). Note: this code is not part of OpenFOAM(R) (see DISCLAIMER). +\*---------------------------------------------------------------------------*/ + +#include "error.H" +#include "twoWayM2M.H" +#include "addToRunTimeSelectionTable.H" +#include "clockModel.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +defineTypeNameAndDebug(twoWayM2M, 0); + +addToRunTimeSelectionTable +( + dataExchangeModel, + twoWayM2M, + dictionary +); + + +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +// Construct from components +twoWayM2M::twoWayM2M +( + const dictionary& dict, + cfdemCloud& sm +) +: + dataExchangeModel(dict,sm), + propsDict_(dict.subDict(typeName + "Props")) +{ + Info<<"Starting up LIGGGHTS for first time execution"<input->one(line); + } + + // get DEM time step size + DEMts_ = lmp->update->dt; + checkTSsize(); + + // m2m stuff + lmp2foam_ = new Many2Many(MPI_COMM_WORLD); + lmp2foam_vec_ = new Many2Many(MPI_COMM_WORLD); + foam2lmp_vec_ = new Many2Many(MPI_COMM_WORLD); + nlocal_lammps_ = -1; + id_lammps_ = NULL; + id_lammpsComm_ = NULL; + id_lammps_vec_ = NULL; + nlocal_foam_ = -1; + id_foam_ = NULL; + id_foam_vec_ = NULL; + nlocal_foam_lost_ = -1; + id_foam_lost_ = NULL; + id_foam_lost_all = NULL; + id_foam_nowhere_all = NULL; + lost_pos_ = NULL; + lost_pos_all = NULL; + cellID_foam_ = NULL; + pos_foam_ = NULL; + //Foam::dataExchangeModel::allocateArray(idHashTable_,0,100000);//liggghts_get_maxtag(lmp)); // define idHashTable +} + + +// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // + +twoWayM2M::~twoWayM2M() +{ + free(id_lammps_); + delete[] id_lammpsComm_; + delete[] id_lammps_vec_; + delete[] id_foam_vec_; + delete[] id_foam_; + delete[] id_foam_lost_; + delete[] lost_pos_; + delete[] cellID_foam_; + delete[] pos_foam_; + //delete[] lmp2foam_; + //delete lmp; +} + + +// * * * * * * * * * * * * * * * private Member Functions * * * * * * * * * * * * * // +char* twoWayM2M::wordToChar(word& inWord) const +{ + string HH = string(inWord); + return const_cast(HH.c_str()); +} + + +// * * * * * * * * * * * * * * * public Member Functions * * * * * * * * * * * * * // +void twoWayM2M::getData +( + word name, + word type, + double ** const& field, + label step +) const +{ + char* charName = wordToChar(name); + if ( type == "vector-atom" && name != "x") + { + //if (nlocal_lammps_>0){ + double **tata_ = (double **) lammps_extract_atom(lmp,charName); + lmp2foam_vec_->exchange(&(tata_[0][0]), &(field[0][0])); + /*}else{ + // is this else necessary? + double *tata_ = (double *) lammps_extract_atom(lmp,charName); + lmp2foam_vec_->exchange(tata_, &(field[0][0])); + }*/ + //for (int i = 0; i < nlocal_foam_; i++) + // Pout << name <<"=" << tata_[i][0]<<","<exchange(tmp_, &(field[0][0])); + //for (int i = 0; i < nlocal_foam_; i++) + // Pout << name <<"[0][i]=" << field[0][i] <modify->find_fix_property(charName,"property/atom","vector",0,0,"cfd coupling",false); + if(fix) + tata_ = (double **) static_cast(fix)->array_atom; + else + Warning << "coupling fix not found!"<exchange(&(field[0][0]),&(tata_[0][0])); + + //================== + //for(int index = 0;index < nlocal_lammps_; ++index){ + // vector forceField(field[index][0],field[index][1],field[index][2]); + // vector tataField(tata_[index][0],tata_[index][1],tata_[index][2]); + // Pout << "particle=" << index << " ,forceField=" << forceField<< " ,tataField=" << tataField << endl; + //} + //================== + }else{ + Warning << "not implemented!"<input->one(command); + } + } + particleCloud_.clockM().stop("LIGGGHTS"); + Info<<"LIGGGHTS finished"<setup(nlocal_lammps_,id_lammpsComm_,nlocal_foam_,id_foam_); + lmp2foam_vec_->setup(nlocal_lammps_*3,id_lammps_vec_,nlocal_foam_*3,id_foam_vec_); + foam2lmp_vec_->setup(nlocal_foam_*3,id_foam_vec_,nlocal_lammps_*3,id_lammps_vec_); + particleCloud_.clockM().stop("setup_Comm"); +} + +void Foam::twoWayM2M::locateParticle() const +{ + //?======= + // if I do not creat new communicators it fails at liggghts porcessor transfer! + // this should probably not be done that often!!! + lmp2foam_ = new Many2Many(MPI_COMM_WORLD); + lmp2foam_vec_ = new Many2Many(MPI_COMM_WORLD); + foam2lmp_vec_ = new Many2Many(MPI_COMM_WORLD); + //?======= + + int nop = particleCloud_.numberOfParticles(); + + // realloc array of lost particles + if (particleCloud_.numberOfParticlesChanged()) + { + // these arrays will be to long, but we do not know their length a priori + delete[] id_foam_lost_; + Foam::dataExchangeModel::allocateArray(id_foam_lost_,0,nop); + + delete [] lost_pos_; + Foam::dataExchangeModel::allocateArray(lost_pos_,0.,nop*3); + + delete [] id_foam_; + Foam::dataExchangeModel::allocateArray(id_foam_,0,nop); + + delete [] id_foam_vec_; + Foam::dataExchangeModel::allocateArray(id_foam_vec_,0,nop*3); + + delete [] id_lammps_vec_; + Foam::dataExchangeModel::allocateArray(id_lammps_vec_,0,nop*3); + + delete [] cellID_foam_; + Foam::dataExchangeModel::allocateArray(cellID_foam_,0,nop); + delete [] pos_foam_; + Foam::dataExchangeModel::allocateArray(pos_foam_,0,nop*3); + } + else + { + // reset array to zero + for (int i=0; i= 0) + { + // IDs for scalars + id_foam_[nlocal_foam_] = id_lammps_[i]; + + // IDs for vectors + for (int j=0;j<3;j++) + { + id_foam_vec_[nlocal_foam_*3+j] = id_lammps_[i]*3+j; + pos_foam_[nlocal_foam_*3+j] = pos[j]; + } + cellID_foam_[nlocal_foam_] = cellID; + + // cell id hash table + //idHashTable_[id_lammps_[i]]=cellID; + + nlocal_foam_ += 1; + //Pout << "found particle at pos=" << pos << endl; + } + else + { + id_foam_lost_[nlocal_foam_lost_] = id_lammps_[i]; + + for (int j=0; j<3; j++) + lost_pos_[nlocal_foam_lost_*3+j] = pos[j]; + + nlocal_foam_lost_ += 1; + //Pout << "cellID="<< cellID << " lost particle id="<< id_lammps_[i] <<" at pos=" << pos << endl; + } + } + particleCloud_.clockM().stop("locate_Stage1"); + + // using allgather to allreduce lost particles + particleCloud_.clockM().start(8,"locate_Stage2"); + int nlocal_foam_lost_all = LAMMPS_NS::MPI_Allgather_Vector(lost_pos_, nlocal_foam_lost_*3, lost_pos_all, MPI_COMM_WORLD)/3; + LAMMPS_NS::MPI_Allgather_Vector(id_foam_lost_, nlocal_foam_lost_, id_foam_lost_all, MPI_COMM_WORLD); + + // locate lost particles + for (int i = 0; i < nlocal_foam_lost_all; i++) + { + pos = vector(lost_pos_all[i*3+0],lost_pos_all[i*3+1],lost_pos_all[i*3+2]); + searchCellID = -1;//idHashTable_[id_foam_lost_all[i]]; + particleCloud_.clockM().start(9,"findSingleCell"); + cellID = particleCloud_.locateM().findSingleCell(pos,searchCellID); + particleCloud_.clockM().stop("findSingleCell"); + + // found particle on cfd proc + if (cellID >= 0) + { + // IDs for scalars + id_foam_[nlocal_foam_] = id_foam_lost_all[i]; + + // IDs for vectors + for (int j=0;j<3;j++) + { + id_foam_vec_[nlocal_foam_*3+j] = id_foam_lost_all[i]*3+j; + pos_foam_[nlocal_foam_*3+j] = pos[j]; + } + cellID_foam_[nlocal_foam_] = cellID; + + // mark that ID was finally found + id_foam_lost_all[i]=-1; + + // cell id hash table + //idHashTable_[id_foam_lost_all[i]]=cellID; + + nlocal_foam_ += 1; + } + } + particleCloud_.clockM().stop("locate_Stage2"); + + /* // check if really all particles were found + particleCloud_.clockM().start(10,"locate_Stage3"); + Foam::dataExchangeModel::allocateArray(id_foam_nowhere_all,1,nlocal_foam_lost_all); + MPI_Allreduce(id_foam_lost_all, id_foam_nowhere_all, nlocal_foam_lost_all, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + + int i=0; + while (i < nlocal_foam_lost_all) + { + // these particles where found nowhere + if (id_foam_nowhere_all[i] > 0) + { + for (int j=0;j +#include +#include +#include "mpi.h" +#include // these are LAMMPS include files +#include +#include +#include +#include +#include +#include +// m2m stuff +#include +#include +#include "locateModel.H" + +#include +#include +#include +#include "meshSearch.H" +//=================================// + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +/*---------------------------------------------------------------------------*\ + Class noDrag Declaration +\*---------------------------------------------------------------------------*/ + +class twoWayM2M +: + public dataExchangeModel +{ +private: + + // private data + dictionary propsDict_; + + // new vars + mutable Many2Many *lmp2foam_; + mutable Many2Many *lmp2foam_vec_; + mutable Many2Many *foam2lmp_vec_; + mutable int nlocal_lammps_; + mutable int *id_lammps_; + mutable int *id_lammpsComm_; + mutable int *id_lammps_vec_; + mutable int nlocal_foam_; + mutable int *id_foam_; + mutable int *id_foam_vec_; + mutable double *tmp_; + mutable double **pos_lammps_; + mutable int nlocal_foam_lost_; + mutable int *id_foam_lost_; + mutable int *id_foam_lost_all; + mutable int *id_foam_nowhere_all; + mutable double *lost_pos_; + mutable double *lost_pos_all; + mutable int *cellID_foam_; + mutable double *pos_foam_; + + // optimization + //mutable int *idHashTable_; + + // variables + int me; + + int nprocs; + + int liggghts; + + MPI_Comm comm_liggghts; + + LAMMPS_NS::LAMMPS *lmp; + + // private member functions + + char* wordToChar(word&) const; + +public: + + //- Runtime type information + TypeName("twoWayM2M"); + + + // Constructors + + //- Construct from components + twoWayM2M + ( + const dictionary& dict, + cfdemCloud& sm + ); + + // Destructor + + ~twoWayM2M(); + + + // Member Functions + void getData + ( + word name, + word type, + double ** const& field, + label step + ) const; + + void getData + ( + word name, + word type, + int ** const& field, + label step + ) const; + + void giveData + ( + word name, + word type, + double ** const& field, + const char* datatype + ) const; + + //============ + // double ** + void allocateArray(double**&, double, int, int) const; + + void allocateArray(double**&, double, int,const char* ="nparticles") const; + + //============ + // int ** + void allocateArray(int**&, int, int, int) const; + + void allocateArray(int**&, int, int,const char* ="nparticles") const; + //============== + + bool couple() const; + + int getNumberOfParticles() const; + int getNumberOfClumps() const; + + void syncIDs() const; + void locateParticle() const; + void exchange(double*,double*) const; +}; + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/CFD/system/decomposeParDict b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/CFD/system/decomposeParDict index 38d4bd34..4de14078 100644 --- a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/CFD/system/decomposeParDict +++ b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/CFD/system/decomposeParDict @@ -17,7 +17,7 @@ FoamFile // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // -numberOfSubdomains 8; +numberOfSubdomains 4; //- Keep owner and neighbour on same processor for faces in zones: // preserveFaceZones (heater solid1 solid3); @@ -30,7 +30,7 @@ numberOfSubdomains 8; simpleCoeffs { - n (2 2 2); + n (2 2 1); delta 0.001; } diff --git a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/DEM/in.liggghts_init b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/DEM/in.liggghts_init index cf38ff8c..6c74a9cf 100755 --- a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/DEM/in.liggghts_init +++ b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/DEM/in.liggghts_init @@ -13,7 +13,7 @@ boundary f f f newton off units si -processors 2 2 2 +processors 2 2 1 region reg block 0 1 0 1 0 4 units box create_box 1 reg diff --git a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/parCFDDEMrun.sh b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/parCFDDEMrun.sh index fa4eab31..2b6b4d25 100644 --- a/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/parCFDDEMrun.sh +++ b/tutorials/cfdemSolverIB/twoSpheresGlowinskiMPI/parCFDDEMrun.sh @@ -19,7 +19,7 @@ logpath=$casePath headerText="run_parallel_cfdemSolverPiso_ErgunTestMPI_CFDDEM" logfileName="log_$headerText" solverName="cfdemSolverIB" -nrProcs="8" +nrProcs="4" machineFileName="none" # yourMachinefileName | none debugMode="off" # on | off | prof testHarnessPath="$CFDEM_TEST_HARNESS_PATH" diff --git a/tutorials/cfdemSolverPiso/ErgunTestMPI/CFD/octave/totalPressureDrop.m b/tutorials/cfdemSolverPiso/ErgunTestMPI/CFD/octave/totalPressureDrop.m index eb2cee05..7863459d 100644 --- a/tutorials/cfdemSolverPiso/ErgunTestMPI/CFD/octave/totalPressureDrop.m +++ b/tutorials/cfdemSolverPiso/ErgunTestMPI/CFD/octave/totalPressureDrop.m @@ -67,14 +67,12 @@ elseif (ReMF>=1000) Umf = sqrt(dp*(rhoP-rhoG)*g/(1.75*rhoG)*epsilon^3*phip); ReMF = Umf*dp*rhoG/muG; end -Umf -ReMF dpUmf= L * ( 150*((1-epsilon)^2/epsilon^3)*((muG.*Umf)/(phip*dp)^2) +1.75*((1-epsilon)/epsilon^3)*((rhoG.*Umf.^2)/(phip*dp)) )/10000/rhoG; - +%dpUmf2=(L*(1-epsilon)*(rhoP-rhoG)*g+pHydr)/10000 %====================================% % plot data %====================================% diff --git a/tutorials/cfdemSolverPiso/ErgunTestMPI_restart/CFD/octave/totalPressureDrop.m b/tutorials/cfdemSolverPiso/ErgunTestMPI_restart/CFD/octave/totalPressureDrop.m index b42321a0..60b9b505 100644 --- a/tutorials/cfdemSolverPiso/ErgunTestMPI_restart/CFD/octave/totalPressureDrop.m +++ b/tutorials/cfdemSolverPiso/ErgunTestMPI_restart/CFD/octave/totalPressureDrop.m @@ -71,9 +71,17 @@ fprintf('so the result does not depend on density\n') %================================== rhoP = 2000 % particle density in kg/m3 g = 9.81 % gravity m/s2 -Umf = dp^2*(rhoP-rhoG)*g/(150*muG)*(epsilon^3*phip^2)/(1-epsilon) -ReMF = Umf*dp*rhoG/muG % must be <20 !!! -%Umf = sqrt(phip*dp^2/1.75*(rhoP-rhoG)/rhoG*g*epsilon^3) % Re>1000 +Umf = dp^2*(rhoP-rhoG)*g/(150*muG)*(epsilon^3*phip^2)/(1-epsilon); +ReMF = Umf*dp*rhoG/muG; +if(ReMF<20) + fprintf('applying eqn1 for Umf.\n') +elseif(ReMF>20 && ReMF<1000) + fprintf('applying eqn1 for Umf.\n') +elseif (ReMF>=1000) + fprintf('applying eqn2 for Umf.\n') + Umf = sqrt(dp*(rhoP-rhoG)*g/(1.75*rhoG)*epsilon^3*phip); + ReMF = Umf*dp*rhoG/muG; +end dpUmf= L * ( 150*((1-epsilon)^2/epsilon^3)*((muG.*Umf)/(phip*dp)^2)