Allowed to skip a list of input scripts as specified in the config file

tools/regression-tests/config.yaml

@@ -28,6 +28,11 @@
       Press:
         abs: 1e-2
         rel: 1e-4
+  skip:
+    [ in.rigid.poems3,
+      in.rigid.poems4
+    ]
+
   nugget: 1.0
   epsilon: 1e-16
 

tools/regression-tests/config_gpu.yaml

@@ -20,5 +20,14 @@
     E_vdwl:
       abs: 1e-3
       rel: 1e-7
+  overrides:
+    in.rigid.tnr:
+      Temp:
+        abs: 1e-3
+        rel: 1e-5
+      Press:
+        abs: 1e-2
+        rel: 1e-4
+
   nugget: 1.0
   epsilon: 1e-16

tools/regression-tests/run_tests.py

@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
 '''
-UPDATE: Feb 8, 2024:
+pip install numpy pyyaml junit_xml
+
+UPDATE: July 5, 2024:
   Launching the LAMMPS binary under testing using a configuration defined in a yaml file (e.g. config.yaml).
   Comparing the output thermo with that in the existing log file (with the same nprocs)
     + data in the log files are extracted and converted into yaml data structure
@@ -8,8 +11,7 @@ UPDATE: Feb 8, 2024:
     + launch tests with mpirun with multiple procs
     + specify what LAMMPS binary version to test (e.g., testing separate builds)
     + simplify the build configuration (no need to build the Python module)
-  NOTE: Need to allow to tolerances specified for invidual input scripts,
+    + specify tolerances for individual quantities for any input script to override the global values
-        or each config.yaml is for a set of example folders
 
   Example usage:
     1) Simple use (using the provided tools/regression-tests/config.yaml and the examples/ folder at the top level)
@@ -28,7 +30,8 @@ import fnmatch
 import subprocess
 from argparse import ArgumentParser
 
-# need "pip install pyyaml numpy"
+import logging
+# need "pip install numpy pyyaml"
 import yaml
 import numpy as np
 
@@ -145,7 +148,7 @@ def extract_data_to_yaml(inputFileName):
     return thermo
 
 '''
-  return the list of installed packages
+    return a tuple of the list of installed packages, OS, GitInfo and compile_flags
 '''
 def get_lammps_build_configuration(lmp_binary):
     cmd_str = lmp_binary + " -h"
@@ -236,10 +239,10 @@ def has_markers(input):
     return False
 
 '''
-  Iterate over a list of input files using the testing configuration
+  Iterate over a list of input files using the given lmp_binary, the testing configuration
-  return total number of tests, and the number of tests with failures
+  return test results, as a list of TestResult instances
 '''
-def iterate(input_list, config, results, removeAnnotatedInput=False):
+def iterate(lmp_binary, input_list, config, results, removeAnnotatedInput=False):
   EPSILON = np.float64(config['epsilon'])
   nugget = float(config['nugget'])
 
@@ -253,6 +256,11 @@ def iterate(input_list, config, results, removeAnnotatedInput=False):
   # iterate over the input scripts
   for input in input_list:
 
+    # skip the input file if listed
+    if 'skip' in config:
+      if input in config['skip']:
+        continue
+
     str_t = "\nRunning " + input + f" ({test_id+1}/{num_tests})"
 
     result = TestResult(name=input, output="", time="", status="passed")
@@ -335,8 +343,10 @@ def iterate(input_list, config, results, removeAnnotatedInput=False):
 
     num_runs = len(thermo)
     if num_runs == 0:
-      print(f"ERROR: Failed with the running with {input_test}. The run terminated with the following output:\n")
+      print(f"ERROR: Failed with {input_test}. Check the log file for the run output.\n")
-      print(f"{output}")
+      #print(f"{output}")
+      logger.info(f"The run terminated with the following output:\n")
+      logger.info(f"\n{output}")
       result.status = "error"
       results.append(result)
       continue 
@@ -351,7 +361,8 @@ def iterate(input_list, config, results, removeAnnotatedInput=False):
     # comparing output vs reference values
     width = 20
     if verbose == True:
-      print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) + "Abs Diff Check".center(width) +  "Rel Diff Check".center(width))
+      print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) +
+            "Abs Diff Check".center(width) +  "Rel Diff Check".center(width))
     
     # check if overrides for this input scipt is specified
     overrides = {}
@@ -420,16 +431,19 @@ def iterate(input_list, config, results, removeAnnotatedInput=False):
               rel_diff_check = "N/A"          
 
           if verbose == True and abs_diff_check != "N/A"  and rel_diff_check != "N/A":
-          print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} {abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}")
+              print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} "
+                  "{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}")
 
     if num_abs_failed > 0:
         print(f"{num_abs_failed} absolute diff checks failed with the specified tolerances.")
         result.status = "failed"
+        if verbose == True:
             for i in failed_abs_output:
               print(f"- {i}")
     if num_rel_failed > 0:
         print(f"{num_rel_failed} relative diff checks failed with the specified tolerances.")
         result.status = "failed"
+        if verbose == True:
           for i in failed_rel_output:
             print(f"- {i}")
     if num_abs_failed == 0 and num_rel_failed == 0:
@@ -455,6 +469,9 @@ def iterate(input_list, config, results, removeAnnotatedInput=False):
 '''
 if __name__ == "__main__":
 
+    logger = logging.getLogger(__name__)
+    logging.basicConfig(filename='run.log', level=logging.INFO)
+
     # default values
     lmp_binary = ""
     configFileName = "config.yaml"
@@ -564,7 +581,6 @@ if __name__ == "__main__":
         if 'DIPOLE' in packages:
             example_subfolders.append('../../examples/dipole')
 
-
         if 'DPD-BASIC' in packages:
             example_subfolders.append('../../examples/PACKAGES/dpd-basic/dpd')
             example_subfolders.append('../../examples/PACKAGES/dpd-basic/dpdext')
@@ -613,7 +629,7 @@ if __name__ == "__main__":
 
           # iterate through the input scripts
           results = []
-      num_passed = iterate(input_list, config, results)
+          num_passed = iterate(lmp_binary, input_list, config, results)
           passed_tests += num_passed
 
           all_results.extend(results)