diff --git a/tools/regression-tests/run_tests.py b/tools/regression-tests/run_tests.py index 65667f042a..3278117c60 100644 --- a/tools/regression-tests/run_tests.py +++ b/tools/regression-tests/run_tests.py @@ -227,17 +227,19 @@ def has_markers(input): ''' Iterate over a list of input files using the testing configuration + return total number of tests, and the number of tests with failures ''' def iterate(input_list, config, removeAnnotatedInput=False): EPSILON = np.float64(config['epsilon']) nugget = float(config['nugget']) - nprocs = int(config['nprocs']) - num_tests = len(input_list) + num_passed = 0 test_id = 0 + + # using REG-commented input scripts using_markers = False - + # iterate over the input scripts for input in input_list: @@ -268,6 +270,41 @@ def iterate(input_list, config, removeAnnotatedInput=False): print(str_t) print(f"-"*len(str_t)) + # check if a log file exists in the current folder: log.[date].basename.[nprocs] + basename = input_test.replace('in.','') + logfile_exist = False + + # if there are multiple log files for different number of procs, pick the maximum number + pattern = f'log.*.{basename}.*' + max_np = 1 + for file in os.listdir('.'): + if fnmatch.fnmatch(file, pattern): + p = file.rsplit('.', 1) + if max_np < int(p[1]): + max_np = int(p[1]) + logfile_exist = True + thermo_ref_file = file + + # if the maximum number of procs is different from the value in the configuration file + # then override the setting for this input script + saved_nprocs = config['nprocs'] + if max_np != int(config['nprocs']): + config['nprocs'] = str(max_np) + + if logfile_exist: + thermo_ref = extract_data_to_yaml(thermo_ref_file) + num_runs_ref = len(thermo_ref) + else: + print(f"Cannot find reference log file with {pattern}.") + # try to read in the thermo yaml output from the working directory + thermo_ref_file = 'thermo.' + input + '.yaml' + file_exist = os.path.isfile(thermo_ref_file) + if file_exist == True: + thermo_ref = extract_thermo(thermo_ref_file) + else: + print(f"SKIPPED: {thermo_ref_file} does not exist in the working directory.") + continue + # using the LAMMPS python module (for single-proc runs) # lmp = lammps() # lmp.file(input_test) @@ -275,49 +312,28 @@ def iterate(input_list, config, removeAnnotatedInput=False): # or more customizable with config.yaml execute(lmp_binary, config, input_test) + # restore the nprocs value in the configuration + config['nprocs'] = saved_nprocs + # process thermo output - #thermo = extract_thermo("log.lammps") thermo = extract_data_to_yaml("log.lammps") num_runs = len(thermo) if num_runs == 0: - print(f"Failed with {input_test}. Check if the run with this input script completed normally.\n") + print(f"Failed with the unning with {input_test}. Check if the run with this input script completed normally.\n") continue - # check if a log file exists in the current folder: log.[date].basename.[nprocs] - basename = input_test.replace('in.','') - logfile_exist = False - pattern = f'log.*.{basename}.*.{nprocs}' - for file in os.listdir('.'): - if fnmatch.fnmatch(file, pattern): - logfile_exist = True - break - - if logfile_exist: - thermo_ref = extract_data_to_yaml(file) - num_runs_ref = len(thermo_ref) - else: - print(f"Cannot find reference log file with {pattern}") - # read in the thermo yaml output from the working directory - thermo_ref_file = 'thermo.' + input + '.yaml' - file_exist = os.path.isfile(thermo_ref_file) - if file_exist == True: - thermo_ref = extract_thermo(thermo_ref_file) - else: - print(f"SKIPPED: {thermo_ref_file} does not exist") - continue - - print(f"Comparing thermo output from log.lammps with the reference log {file}") + print(f"Comparing thermo output from log.lammps with the reference log {thermo_ref_file}") if num_runs != num_runs_ref: print(f"ERROR: Number of runs in log.lammps ({num_runs}) is not the same as that in the reference log ({num_runs_ref})") continue - + # comparing output vs reference values width = 20 print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) + "Abs Diff Check".center(width) + "Rel Diff Check".center(width)) irun = 0 num_fields = len(thermo[irun]['keywords']) - + # get the total number of the thermo output lines nthermo_steps = len(thermo[irun]['data']) # get the output at the last timestep @@ -331,12 +347,11 @@ def iterate(input_list, config, removeAnnotatedInput=False): val = thermo[irun]['data'][thermo_step][i] ref = thermo_ref[irun]['data'][thermo_step][i] abs_diff = abs(float(val) - float(ref)) - + if abs(float(ref)) > EPSILON: rel_diff = abs(float(val) - float(ref))/abs(float(ref)) else: rel_diff = abs(float(val) - float(ref))/abs(float(ref)+nugget) - abs_diff_check = "PASSED" rel_diff_check = "PASSED" @@ -348,18 +363,20 @@ def iterate(input_list, config, removeAnnotatedInput=False): if rel_diff > float(config['tolerance'][quantity]['rel']): rel_diff_check = "FAILED" num_rel_failed = num_rel_failed + 1 + else: abs_diff_check = "N/A" rel_diff_check = "N/A" print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} {abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}") + if num_abs_failed > 0: - print(f"{num_abs_failed} abs checks failed") + print(f"{num_abs_failed} absolute diff checks failed with the specified tolerances.") elif num_rel_failed > 0: - print(f"{num_rel_failed} rel checks failed") + print(f"{num_rel_failed} relative diff checks failed with the specified tolerances.") else: print("All checks passed. (N/A means tolerance not defined in the config file.)") - + num_passed = num_passed + 1 print("-"*(5*width+4)) test_id = test_id + 1 @@ -369,10 +386,11 @@ def iterate(input_list, config, removeAnnotatedInput=False): cmd_str = "rm " + input_test os.system(cmd_str) + return num_passed ''' TODO: - - automate annotating the example input scripts of the installed packages + - automate annotating the example input scripts if thermo style is multi (e.g. examples/peptide) ''' if __name__ == "__main__": @@ -426,10 +444,21 @@ if __name__ == "__main__": # append the example subfolders depending on the installed packages if 'MOLECULE' in packages: example_subfolders.append('../../examples/micelle') + # peptide thermo_style as multi + #example_subfolders.append('../../examples/peptide') #if 'ASPHERE' in packages: # example_subfolders.append('../../examples/ASPHERE/ellipsoid') + #if 'AMOEBA' in packages: + # example_subfolders.append('../../examples/amoeba') + + if 'BODY' in packages: + example_subfolders.append('../../examples/body') + + if 'DIELECTRIC' in packages: + example_subfolders.append('../../examples/PACKAGES/dielectric') + if 'COLLOID' in packages: example_subfolders.append('../../examples/colloid') @@ -438,48 +467,64 @@ if __name__ == "__main__": if 'MANYBODY' in packages: example_subfolders.append('../../examples/tersoff') + example_subfolders.append('../../examples/vashishta') + + if 'RIGID' in packages: + example_subfolders.append('../../examples/rigid') if inplace_input == True: # save current working dir p = subprocess.run("pwd", shell=True, text=True, capture_output=True) pwd = p.stdout.split('\n')[0] + pwd = os.path.abspath(pwd) print("Working directory: " + pwd) - # change dir to a folder under examples/ + # change dir to a folder under examples/, need to use os.chdir() # TODO: loop through the subfolders under examples/, depending on the installed packages + total_tests = 0 + passed_tests = 0 + for directory in example_subfolders: + p = subprocess.run("pwd", shell=True, text=True, capture_output=True) print("\nEntering " + directory) os.chdir(directory) # create a symbolic link to the lammps binary at the present directory - cmd_str = "ln -s " + lmp_binary + " lmp" - os.system(cmd_str) + if os.path.isfile("lmp") == False: + cmd_str = "ln -s " + lmp_binary + " lmp" + os.system(cmd_str) cmd_str = "ls in.*" p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) input_list = p.stdout.split('\n') input_list.remove('') - print("List of input scripts:") - print(input_list) + print(f"List of input scripts: {input_list}") + total_tests += len(input_list) # iterate through the input scripts - iterate(input_list, config) + num_passed = iterate(input_list, config) + passed_tests += num_passed # unlink the symbolic link cmd_str = "unlink lmp" os.system(cmd_str) + # get back to the working dir - cmd_str = "cd " + pwd - os.system(cmd_str) + os.chdir(pwd) + else: # or using the input scripts in the working directory input_list=['in.lj', 'in.rhodo', 'in.eam'] - iterate(input_list, config) + total_tests = len(input_list) + passed_tests = iterate(input_list, config) + + print("Summary:") + print(f" - {passed_tests} passed / {total_tests} tests")