From 52ad71871cbe665e2aa772a8a88d53e035232a71 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 8 May 2018 14:58:50 +0200 Subject: [PATCH] CONFIG: add patches for ParaView Catalyst - resolve problem with working directory: https://develop.openfoam.com/Community/catalyst/issues/4 https://gitlab.kitware.com/paraview/paraview/merge_requests/2436 --- etc/patches/paraview-5.5.0 | 480 +++++++++++++++++++++++++++++++++++++ makeParaView.example | 1 + makeVTK.example | 20 +- 3 files changed, 498 insertions(+), 3 deletions(-) diff --git a/etc/patches/paraview-5.5.0 b/etc/patches/paraview-5.5.0 index 8c8947d..32104aa 100644 --- a/etc/patches/paraview-5.5.0 +++ b/etc/patches/paraview-5.5.0 @@ -18,3 +18,483 @@ version_matches "${version_string}") if(CMAKE_MATCH_0) set(full ${CMAKE_MATCH_0}) +--- ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPProcessor.cxx.orig 2018-04-06 22:03:33.000000000 +0200 ++++ ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPProcessor.cxx 2018-05-11 12:02:26.894772713 +0200 +@@ -38,6 +38,7 @@ + #include "vtkStringArray.h" + + #include ++#include + + struct vtkCPProcessorInternals + { +@@ -47,12 +48,13 @@ + }; + + vtkStandardNewMacro(vtkCPProcessor); +-vtkMultiProcessController* vtkCPProcessor::Controller = NULL; ++vtkMultiProcessController* vtkCPProcessor::Controller = nullptr; + //---------------------------------------------------------------------------- + vtkCPProcessor::vtkCPProcessor() + { + this->Internal = new vtkCPProcessorInternals; +- this->InitializationHelper = NULL; ++ this->InitializationHelper = nullptr; ++ this->WorkingDirectory = nullptr; + } + + //---------------------------------------------------------------------------- +@@ -61,14 +63,15 @@ + if (this->Internal) + { + delete this->Internal; +- this->Internal = NULL; ++ this->Internal = nullptr; + } + + if (this->InitializationHelper) + { + this->InitializationHelper->Delete(); +- this->InitializationHelper = NULL; ++ this->InitializationHelper = nullptr; + } ++ this->SetWorkingDirectory(nullptr); + } + + //---------------------------------------------------------------------------- +@@ -95,7 +98,7 @@ + { + if (which < 0 || which >= this->GetNumberOfPipelines()) + { +- return NULL; ++ return nullptr; + } + int counter = 0; + vtkCPProcessorInternals::PipelineListIterator iter = this->Internal->Pipelines.begin(); +@@ -108,7 +111,7 @@ + counter++; + iter++; + } +- return NULL; ++ return nullptr; + } + + //---------------------------------------------------------------------------- +@@ -130,17 +133,41 @@ + } + + //---------------------------------------------------------------------------- +-int vtkCPProcessor::Initialize() ++int vtkCPProcessor::Initialize(const char* workingDirectory) + { +- if (this->InitializationHelper == NULL) ++ if (this->InitializationHelper == nullptr) + { + this->InitializationHelper = this->NewInitializationHelper(); + } ++ // make sure the directory exists here so that we only do it once ++ if (workingDirectory) ++ { ++ vtkMultiProcessController* controller = vtkMultiProcessController::GetGlobalController(); ++ int success = 1; ++ if (controller == nullptr || controller->GetLocalProcessId() == 0) ++ { ++ success = vtksys::SystemTools::MakeDirectory(workingDirectory) == true ? 1 : 0; ++ if (success == 0) ++ { ++ vtkWarningMacro("Could not make " ++ << workingDirectory << " directory. " ++ << "Results will be generated in current working directory instead."); ++ } ++ } ++ if (controller) ++ { ++ controller->Broadcast(&success, 1, 0); ++ } ++ if (success) ++ { ++ this->SetWorkingDirectory(workingDirectory); ++ } ++ } + return 1; + } + + //---------------------------------------------------------------------------- +-int vtkCPProcessor::Initialize(vtkMPICommunicatorOpaqueComm& comm) ++int vtkCPProcessor::Initialize(vtkMPICommunicatorOpaqueComm& comm, const char* workingDirectory) + { + #ifdef PARAVIEW_USE_MPI + if (vtkCPProcessor::Controller) +@@ -148,7 +175,7 @@ + vtkErrorMacro("Can only initialize with a communicator once per process."); + return 0; + } +- if (this->InitializationHelper == NULL) ++ if (this->InitializationHelper == nullptr) + { + vtkMPICommunicator* communicator = vtkMPICommunicator::New(); + communicator->InitializeExternal(&comm); +@@ -157,12 +184,12 @@ + this->Controller = controller; + this->Controller->SetGlobalController(controller); + communicator->Delete(); +- return this->Initialize(); ++ return this->Initialize(workingDirectory); + } + return 1; + #else + static_cast(&comm); // get rid of variable not used warning +- return this->Initialize(); ++ return this->Initialize(workingDirectory); + #endif + } + +@@ -225,6 +252,13 @@ + input->GetFieldData()->AddArray(catalystChannel); + } + } ++ ++ std::string originalWorkingDirectory; ++ if (this->WorkingDirectory) ++ { ++ originalWorkingDirectory = vtksys::SystemTools::GetCurrentWorkingDirectory(); ++ vtksys::SystemTools::ChangeDirectory(this->WorkingDirectory); ++ } + for (vtkCPProcessorInternals::PipelineListIterator iter = this->Internal->Pipelines.begin(); + iter != this->Internal->Pipelines.end(); iter++) + { +@@ -248,6 +282,10 @@ + } + } + } ++ if (originalWorkingDirectory.empty() == false) ++ { ++ vtksys::SystemTools::ChangeDirectory(originalWorkingDirectory); ++ } + // we want to reset everything here to make sure that new information + // is properly passed in the next time. + dataDescription->ResetAll(); +@@ -259,7 +297,7 @@ + { + if (this->Controller) + { +- this->Controller->SetGlobalController(NULL); ++ this->Controller->SetGlobalController(nullptr); + this->Controller->Finalize(1); + this->Controller->Delete(); + } +--- ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPProcessor.h.orig 2018-04-06 22:03:33.000000000 +0200 ++++ ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPProcessor.h 2018-05-11 12:02:26.894772713 +0200 +@@ -76,14 +76,16 @@ + virtual void RemoveAllPipelines(); + + /// Initialize the co-processor. Returns 1 if successful and 0 +- /// otherwise. + /// otherwise. If Catalyst is built with MPI then Initialize() + /// can also be called with a specific MPI communicator if + /// MPI_COMM_WORLD isn't the proper one. Catalyst is initialized +- /// to use MPI_COMM_WORLD by default. +- virtual int Initialize(); ++ /// to use MPI_COMM_WORLD by default. Both methods have an optional ++ /// workingDirectory argument which will set *WorkingDirectory* so ++ /// that files will be put relative to this directory. ++ virtual int Initialize(const char* workingDirectory = nullptr); + #ifndef __WRAP__ +- virtual int Initialize(vtkMPICommunicatorOpaqueComm& comm); ++ virtual int Initialize( ++ vtkMPICommunicatorOpaqueComm& comm, const char* workingDirectory = nullptr); + #endif + + /// The Catalyst input field data string array name. This array will +@@ -111,6 +113,13 @@ + /// implementation an opportunity to clean up, before it is destroyed. + virtual int Finalize(); + ++ /// Get the current working directory for outputting Catalyst files. ++ /// If not set then Catalyst output files will be relative to the ++ /// current working directory. This will not affect where Catalyst ++ /// looks for Python scripts. *WorkingDirectory* gets set through ++ /// the *Initialize()* methods. ++ vtkGetStringMacro(WorkingDirectory); ++ + protected: + vtkCPProcessor(); + virtual ~vtkCPProcessor(); +@@ -118,6 +127,11 @@ + /// Create a new instance of the InitializationHelper. + virtual vtkObject* NewInitializationHelper(); + ++ /// Set the current working directory for outputting Catalyst files. ++ /// This is a protected method since simulation code adaptors should ++ /// set this through the *Initialize()* methods. ++ vtkSetStringMacro(WorkingDirectory); ++ + private: + vtkCPProcessor(const vtkCPProcessor&) = delete; + void operator=(const vtkCPProcessor&) = delete; +@@ -125,6 +139,7 @@ + vtkCPProcessorInternals* Internal; + vtkObject* InitializationHelper; + static vtkMultiProcessController* Controller; ++ char* WorkingDirectory; + }; + + #endif +--- ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPXMLPWriterPipeline.cxx.orig 2018-04-06 22:03:33.000000000 +0200 ++++ ParaView-v5.5.0/CoProcessing/Catalyst/vtkCPXMLPWriterPipeline.cxx 2018-05-11 12:02:26.894772713 +0200 +@@ -31,6 +31,7 @@ + #include + #include + ++#include + #include + #include + +@@ -174,7 +175,7 @@ + + for (unsigned int i = 0; i < dataDescription->GetNumberOfInputDescriptions(); i++) + { +- const char* inputName = dataDescription->GetInputDescriptionName(i); ++ std::string inputName = dataDescription->GetInputDescriptionName(i); + vtkCPInputDataDescription* idd = dataDescription->GetInputDescription(i); + vtkDataObject* grid = idd->GetGrid(); + if (grid == nullptr) +@@ -206,6 +207,8 @@ + vtkSMStringVectorProperty* fileName = + vtkSMStringVectorProperty::SafeDownCast(writer->GetProperty("FileName")); + ++ // If we have a / in the channel name we take it out of the filename we're going to write to ++ inputName.erase(std::remove(inputName.begin(), inputName.end(), '/'), inputName.end()); + std::ostringstream o; + if (this->Path.empty() == false) + { +--- ParaView-v5.5.0/Wrapping/Python/paraview/coprocessing.py.orig 2018-04-06 22:03:33.000000000 +0200 ++++ ParaView-v5.5.0/Wrapping/Python/paraview/coprocessing.py 2018-05-11 12:02:27.038772408 +0200 +@@ -11,22 +11,12 @@ + from paraview.vtk.vtkPVVTKExtensionsCore import * + import math + +-# ----------------------------------------------------------------------------- +-def IsInModulo(timestep, frequencyArray): +- """ +- Return True if the given timestep is in one of the provided frequency. +- This can be interpreted as follow:: +- +- isFM = IsInModulo(timestep, [2,3,7]) +- +- is similar to:: ++# If the user created a filename in a location that doesn't exist by default we'll ++# make the directory for them. This can be changed though by setting createDirectoriesIfNeeded ++# to False. ++createDirectoriesIfNeeded = True + +- isFM = (timestep % 2 == 0) or (timestep % 3 == 0) or (timestep % 7 == 0) +- """ +- for frequency in frequencyArray: +- if frequency > 0 and (timestep % frequency == 0): +- return True +- return False ++# ----------------------------------------------------------------------------- + + class CoProcessor(object): + """Base class for co-processing Pipelines. +@@ -68,6 +58,9 @@ + self.__CinemaTracks = {} + self.__InitialFrequencies = {} + self.__PrintEnsightFormatString = False ++ self.__TimeStepToStartOutputAt=0 ++ self.__ForceOutputAtFirstCall=False ++ self.__FirstTimeStepIndex = None + + def SetPrintEnsightFormatString(self, enable): + """If outputting ExodusII files with the purpose of reading them into +@@ -87,6 +80,17 @@ + "Incorrect argument type: %s, must be a dict" % type(frequencies)) + self.__InitialFrequencies = frequencies + ++ def SetInitialOutputOptions(self, timeStepToStartOutputAt, forceOutputAtFirstCall): ++ """Set the frequencies at which the pipeline needs to be updated. ++ Typically, this is called by the subclass once it has determined what ++ timesteps co-processing will be needed to be done. ++ frequencies is a map, with key->string name of for the simulation ++ input, and value is a list of frequencies. ++ """ ++ ++ self.__TimeStepToStartOutputAt=timeStepToStartOutputAt ++ self.__ForceOutputAtFirstCall=forceOutputAtFirstCall ++ + def EnableLiveVisualization(self, enable, frequency = 1): + """Call this method to enable live-visualization. When enabled, + DoLiveVisualization() will communicate with ParaView server if possible +@@ -115,7 +119,7 @@ + # if this is a time step to do live then all of the inputs + # must be made available. note that we want the pipeline built + # before we do the actual first live connection. +- if self.__EnableLiveVisualization and timestep % self.__LiveVisualizationFrequency == 0 \ ++ if self.__EnableLiveVisualization and self.NeedToOutput(timestep, self.__LiveVisualizationFrequency) \ + and self.__LiveVisualizationLink: + if self.__LiveVisualizationLink.Initialize(servermanager.ActiveConnection.Session.GetSessionProxyManager()): + num_inputs = datadescription.GetNumberOfInputDescriptions() +@@ -132,13 +136,13 @@ + # hasn't been set up yet). If we don't have live enabled + # we know that the output frequencies aren't changed and can + # just use the initial frequencies. +- if self.__InitialFrequencies or not self.__EnableLiveVisualization: ++ if self.__ForceOutputAtFirstCall or self.__InitialFrequencies or not self.__EnableLiveVisualization: + num_inputs = datadescription.GetNumberOfInputDescriptions() + for cc in range(num_inputs): + input_name = datadescription.GetInputDescriptionName(cc) + + freqs = self.__InitialFrequencies.get(input_name, []) +- if self.__EnableLiveVisualization or ( self and IsInModulo(timestep, freqs) ): ++ if self.__EnableLiveVisualization or ( self and self.IsInModulo(timestep, freqs) ): + datadescription.GetInputDescription(cc).AllFieldsOn() + datadescription.GetInputDescription(cc).GenerateMeshOn() + else: +@@ -149,15 +153,14 @@ + for writer in self.__WritersList: + frequency = writer.parameters.GetProperty( + "WriteFrequency").GetElement(0) +- if (timestep % frequency) == 0 or \ +- datadescription.GetForceOutput() == True: ++ if self.NeedToOutput(timestep, frequency) or datadescription.GetForceOutput() == True: + writerinputs = cpstate.locate_simulation_inputs(writer) + for writerinput in writerinputs: + datadescription.GetInputDescriptionByName(writerinput).AllFieldsOn() + datadescription.GetInputDescriptionByName(writerinput).GenerateMeshOn() + + for view in self.__ViewsList: +- if (view.cpFrequency and timestep % view.cpFrequency == 0) or \ ++ if (view.cpFrequency and self.NeedToOutput(timestep, view.cpFrequency)) or \ + datadescription.GetForceOutput() == True: + viewinputs = cpstate.locate_simulation_inputs_for_view(view) + for viewinput in viewinputs: +@@ -192,8 +195,7 @@ + for writer in self.__WritersList: + frequency = writer.parameters.GetProperty( + "WriteFrequency").GetElement(0) +- if (timestep % frequency) == 0 or \ +- datadescription.GetForceOutput() == True: ++ if self.NeedToOutput(timestep, frequency) or datadescription.GetForceOutput() == True: + fileName = writer.parameters.GetProperty("FileName").GetElement(0) + paddingamount = writer.parameters.GetProperty("PaddingAmount").GetElement(0) + helperName = writer.GetXMLName() +@@ -203,6 +205,23 @@ + else: + ts = str(timestep).rjust(paddingamount, '0') + writer.FileName = fileName.replace("%t", ts) ++ if '/' in writer.FileName and createDirectoriesIfNeeded: ++ oktowrite = [1.] ++ import vtk ++ comm = vtk.vtkMultiProcessController.GetGlobalController() ++ if comm.GetLocalProcessId() == 0: ++ import os ++ newDir = writer.FileName[0:writer.FileName.rfind('/')] ++ try: ++ os.makedirs(newDir) ++ except OSError: ++ if not os.path.isdir(newDir): ++ print ("ERROR: Cannot make directory for", writer.FileName, ". No data will be written.") ++ oktowrite[0] = 0. ++ comm.Broadcast(oktowrite, 1, 0) ++ if oktowrite[0] == 0: ++ # we can't make the directory so no reason to update the pipeline ++ return + writer.UpdatePipeline(datadescription.GetTime()) + + def WriteImages(self, datadescription, rescale_lookuptable=False, +@@ -240,7 +259,7 @@ + + cinema_dirs = [] + for view in self.__ViewsList: +- if (view.cpFrequency and timestep % view.cpFrequency == 0) or \ ++ if (view.cpFrequency and self.NeedToOutput(timestep, view.cpFrequency)) or \ + datadescription.GetForceOutput() == True: + fname = view.cpFileName + ts = str(timestep).rjust(padding_amount, '0') +@@ -267,6 +286,24 @@ + if dirname: + cinema_dirs.append(dirname) + else: ++ if '/' in fname and createDirectoriesIfNeeded: ++ oktowrite = [1.] ++ import vtk ++ comm = vtk.vtkMultiProcessController.GetGlobalController() ++ if comm.GetLocalProcessId() == 0: ++ import os ++ newDir = fname[0:fname.rfind('/')] ++ try: ++ os.makedirs(newDir) ++ except OSError: ++ if not os.path.isdir(newDir): ++ print ("ERROR: Cannot make directory for", fname, ". No image will be output.") ++ oktowrite[0] = 0. ++ comm.Broadcast(oktowrite, 1, 0) ++ if oktowrite[0] == 0: ++ # we can't make the directory so no reason to update the pipeline ++ return ++ + if image_quality is None and fname.endswith('png'): + # for png quality = 0 means no compression. compression can be a potentially + # very costly serial operation on process 0 +@@ -307,7 +344,7 @@ + + + timeStep = datadescription.GetTimeStep() +- if self.__EnableLiveVisualization and timeStep % self.__LiveVisualizationFrequency == 0: ++ if self.__EnableLiveVisualization and self.NeedToOutput(timeStep, self.__LiveVisualizationFrequency): + if not self.__LiveVisualizationLink.Initialize(servermanager.ActiveConnection.Session.GetSessionProxyManager()): + return + +@@ -412,7 +449,7 @@ + """ + controller = servermanager.ParaViewPipelineController() + # assume that a client only proxy with the same name as a writer +- # is available in "insitu_writer_paramters" ++ # is available in "insitu_writer_parameters" + + # Since coprocessor sometimes pass writer as a custom object and not + # a proxy, we need to handle that. Just creating any arbitrary writer +@@ -666,3 +703,42 @@ + #restore what we showed + pv_introspect.restore_visibility(pxystate) + return os.path.basename(vfname) ++ ++ def IsInModulo(self, timestep, frequencies): ++ """ ++ Return True if the given timestep is in one of the provided frequency. ++ This can be interpreted as follow:: ++ ++ isFM = IsInModulo(timestep-timeStepToStartOutputAt, [2,3,7]) ++ ++ is similar to:: ++ ++ isFM = (timestep-timeStepToStartOutputAt % 2 == 0) or (timestep-timeStepToStartOutputAt % 3 == 0) or (timestep-timeStepToStartOutputAt % 7 == 0) ++ ++ The timeStepToStartOutputAt is the first timestep that will potentially be output. ++ """ ++ if timestep < self.__TimeStepToStartOutputAt and not self.__ForceOutputAtFirstCall: ++ return False ++ for frequency in frequencies: ++ if frequency > 0 and self.NeedToOutput(timestep, frequency): ++ return True ++ ++ return False ++ ++ ++ def NeedToOutput(self, timestep, frequency): ++ """ ++ Return True if we need to output based on the input timestep and frequency. Checks based ++ __FirstTimeStepIndex, __FirstTimeStepIndex, __ForceOutputAtFirstCall and __TimeStepToStartOutputAt ++ member variables. ++ """ ++ if self.__FirstTimeStepIndex == None: ++ self.__FirstTimeStepIndex = timestep ++ ++ if self.__ForceOutputAtFirstCall and self.__FirstTimeStepIndex == timestep: ++ return True ++ ++ if self.__TimeStepToStartOutputAt <= timestep and (timestep-self.__TimeStepToStartOutputAt) % frequency == 0: ++ return True ++ ++ return False diff --git a/makeParaView.example b/makeParaView.example index 9caf442..fd7dcee 100755 --- a/makeParaView.example +++ b/makeParaView.example @@ -7,6 +7,7 @@ # mesa=mesa-13.0.3 mesa=mesa-17.1.1 +set -x ./makeParaView \ -mpi=0 \ -mesa-prefix $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mesa \ diff --git a/makeVTK.example b/makeVTK.example index 46d6e4c..e7b5181 100755 --- a/makeVTK.example +++ b/makeVTK.example @@ -2,15 +2,29 @@ # An example for building particular combinations of VTK with # - mesa (off-screen only) -vtk=VTK-8.1.0 +vtk=VTK-9.0.0 # mesa=mesa-11.2.2 # mesa=mesa-13.0.3 mesa=mesa-17.1.1 +# Request building MPI modules +# VTK_Group_MPI=ON +# +# Request building vtkAcceleratorsVTKm +# Module_vtkAcceleratorsVTKm=OFF +# +# Request building vtkParallelMPI +# Module_vtkParallelMPI=ON + +set -x ./makeVTK \ - $vtk \ + -mpi=0 \ -osmesa \ -mesa-prefix $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mesa \ - "$@" + $vtk "$@" \ + VTK_Group_MPI=ON \ + Module_vtkAcceleratorsVTKm=ON \ + Module_vtkParallelMPI=ON + #------------------------------------------------------------------------------