Committer: ckloss <ckloss@fluid38.(none)>

On branch master
 Initial commit for lpp, version 2011-10-11

src/lpp.py

@@ -0,0 +1,183 @@
+oneline = "writing pp-data in vtk format automatically, saving memory"
+
+docstr = """this is the docstr of LIGGGHTSPostProcessing"""
+
+from dump import dump
+from math import floor
+from math import ceil
+from vtk import vtk
+import glob
+import multiprocessing
+import sys
+import time
+import os
+import exceptions
+# changes py p.s. - include a few command line options
+import getopt
+
+class lpp:
+  
+  #=============================================================================
+  # creates a filelist, seperates it to sublists
+  # creates multiple processes
+  # calls lppWorker for each sublist
+  #   lppWorker
+  #   calls dump, vtk and manyGran for the given list of files
+  #   returns 0
+  #=============================================================================
+  
+  def __init__(self, *list, **kwargs):
+    # do argument parsing, raise errors if non-integers were given
+    # this can be changed if one wants less overhead but use more memory:
+    # make this figure higher but if possible a multiple of 8
+    self.cpunum      = multiprocessing.cpu_count()
+    self.chunksize   = 8
+
+    if "--chunksize" in kwargs: 
+      try:
+        if int(kwargs["--chunksize"]) > 0:
+          self.chunksize = int(kwargs["--chunksize"])
+        else: raise ValueError
+      except ValueError:
+        raise ValueError, "Invalid or no argument given for chunksize"
+
+    if "--cpunum" in kwargs: 
+      try:
+        if int(kwargs["--cpunum"]) > 0 and int(kwargs["--cpunum"]) <= self.cpunum:
+          self.cpunum = int(kwargs["--cpunum"])
+        else: raise ValueError
+      except ValueError:
+        raise ValueError, "Invalid or no argument given for cpunum"
+
+    
+    # suppress output with 'False'
+    if "--debug" in kwargs: self.debugMode = True
+    else: self.debugMode = False
+    
+    if "--quiet" in kwargs:
+      self.output = False
+      self.debugMode = False
+    else: self.output = True
+
+    if self.output:
+      print "starting LIGGGHTS memory optimized parallel post processing"
+      print "chunksize:", self.chunksize, "-->",self.chunksize,\
+        "files are processed per chunk. If you run out of memory reduce chunksize."
+    starttime = time.time()    
+
+    if self.debugMode: print "number of process:", os.getpid()
+    
+    # check whether file-list is nonempty
+    self.flist = list[0]
+    if len(self.flist) == 0 and len(list) == 1:
+      raise StandardError, "no dump file specified"
+    
+    if self.output:
+      print "Working with", self.cpunum, "processes..."
+    
+    # seperate list in pieces of 96+rest
+    listlen = len(self.flist)
+    self.slices = []
+    
+    residualPresent = int(bool(listlen-floor(listlen/self.chunksize)*self.chunksize))
+    
+    for i in xrange(int(floor(listlen/self.chunksize))+residualPresent):
+      slice = self.flist[i*self.chunksize:(i+1)*self.chunksize]
+      self.slices.append(slice)
+    self.flist = []
+
+    output = ""
+    if "-o" in kwargs: output = kwargs["-o"]
+    # generate input for lppWorker
+    dumpInput = [{"filelist":self.slices[i],\
+      "debugMode":self.debugMode,"output":output} for i in xrange(len(self.slices))]
+    
+    numberOfRuns = len(dumpInput)
+    i = 0
+    while i < len(dumpInput):
+
+      if self.output:    
+        print "calculating chunks",i+1,"-",min(i+self.cpunum,numberOfRuns),"of",numberOfRuns
+      
+      if self.debugMode: print "input of this \"map\": ",dumpInput[i:i+self.cpunum]
+      
+      # create job_server
+      job_server = multiprocessing.Pool(processes = self.cpunum)
+      
+      # map lppWorker on all inputs via job_server (parallelly)
+      job_server.map(lppWorker,dumpInput[i:i+self.cpunum])
+      
+      # close jobserver
+      job_server.close()
+      i += self.cpunum
+    
+    endtime = time.time()
+    if self.output:    
+      print "wrote", listlen,"granular snapshots in VTK format"
+      print "time needed:",endtime-starttime,"sec"
+
+def lppWorker(input):
+  flist = input["filelist"]
+  debugMode = input["debugMode"]
+  outfileName = input["output"]
+  #print "number of subprocess:", os.getpid()
+  #print flist,"\n\n"
+  
+  flistlen = len(flist)
+  splitfname = flist[0].rsplit(".")
+  if outfileName == "":
+    # generate name of manyGran
+    granName = splitfname[len(splitfname)-1]
+  elif outfileName.endswith("/"):
+    granName = outfileName + splitfname[len(splitfname)-1]
+  else:
+    granName = outfileName
+
+  # call dump, vtk, manyGran
+  try:
+    d = dump({"filelist":flist, "debugMode":debugMode})
+    v = vtk(d)
+    if debugMode: print "\nfileNums: ",d.fileNums,"\n"
+    v.manyGran(granName,fileNos=d.fileNums,output=debugMode)
+  except KeyboardInterrupt:
+    raise
+  
+  return 0
+
+def printHelp():
+  print "usage: pizza [options] dump.example\n where dump.example is a filename",\
+    "or regular expression passing all relevant dump files to pizza."
+  print "Important command line options:"
+  print "-o fname    : define output file name (default is liggghts + timestep number)"
+  print "--chunksize : sets the chunksize, default: 8"
+  print "--cpunum    : sets the number of processes to start, default (and maximum)",\
+    "is the amout of cpu cores avaliable at your system"
+  print "--help      : writes this help message and exits"
+  print "For details, read README_GRANULAR.txt"
+
+if __name__ == "__main__":
+  if len(sys.argv) > 1:
+    # parse options
+    optlist, args = getopt.gnu_getopt(sys.argv[1:],'o:',['chunksize=','cpunum=','debug','help','quiet'])
+    optdict = dict(optlist)
+    if "--help" in optdict:
+      printHelp()
+    else:
+      try:
+        lpp(args,**optdict)
+      except KeyboardInterrupt:
+        print "aborted by user"
+      except BaseException, e:
+        print "aborting due to errors:", e
+
+    #===========================================================================
+    # except:
+    #  if sys.exc_type == exceptions.SystemExit:
+    #    pass
+    #  else: 
+    #    print sys.exc_info()
+    #===========================================================================
+  else:
+    printHelp()
+
+