ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handled a few cases with mismatched columns in thermo output

Browse Source
This commit is contained in:
Trung Nguyen
2024-07-26 09:51:19 -05:00
parent 1a344853e0
commit 59ef492982
1 changed files with 104 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
tools/regression-tests/run_tests.py
View File

@ -51,10 +51,10 @@ Example usage:
4) Test a LAMMPS binary with the whole top-level /examples folder in a LAMMPS source tree 4) Test a LAMMPS binary with the whole top-level /examples folder in a LAMMPS source tree
python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples
5) Analyze (dry run) the LAMMPS binary annd whole top-level /examples folder in a LAMMPS source tree 5) Analyze the LAMMPS binary annd whole top-level /examples folder in a LAMMPS source tree
and generate separate input lists for 8 workers: and generate separate input lists for 8 workers:
python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples \ python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples \
--dry-run --num-workers=8 --analyze --num-workers=8
This is used for splitting the subfolders into separate input lists and launching different instances This is used for splitting the subfolders into separate input lists and launching different instances
of run_tests.py simultaneously. of run_tests.py simultaneously.
@ -363,24 +363,25 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
# skip the input file if listed # skip the input file if listed
if 'skip' in config: if 'skip' in config:
if input in config['skip']: if input in config['skip']:
msg = f"SKIPPED: {input} as specified in the configuration file {configFileName}" msg = " + " + input + f" ({test_id+1}/{num_tests}): skipped as specified in {configFileName}"
print(msg) print(msg)
logger.info(msg) logger.info(msg)
progress.write(f"{input}: {{ folder: {input_folder}, status: skipped }}\n") progress.write(f"{input}: {{ folder: {input_folder}, status: skipped }}\n")
progress.close() progress.close()
test_id = test_id + 1 #test_id = test_id + 1
continue continue
# also skip if the test already completed # also skip if the test already completed
if input in last_progress: if input in last_progress:
status = last_progress[input]['status'] status = last_progress[input]['status']
if status == 'completed': if status == 'completed':
msg = f"COMPLETED: {input} marked as completed in the progress file {progress_file}" #msg = f" COMPLETED: {input} marked as completed in the progress file {progress_file}"
msg = " + " + input + f" ({test_id+1}/{num_tests}): marked as completed in the progress file {progress_file}"
logger.info(msg) logger.info(msg)
print(msg) print(msg)
progress.write(msg) progress.write(msg)
progress.close() progress.close()
test_id = test_id + 1 #test_id = test_id + 1
continue continue
str_t = " + " + input + f" ({test_id+1}/{num_tests})" str_t = " + " + input + f" ({test_id+1}/{num_tests})"
@ -467,7 +468,7 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
num_runs = len(thermo) num_runs = len(thermo)
if num_runs == 0: if num_runs == 0:
logger.info(f"The run terminated with {input_test} gives the following output:\n") logger.info(f"The run terminated with {input_test} gives the following output:")
logger.info(f"\n{output}") logger.info(f"\n{output}")
if "Unrecognized" in output: if "Unrecognized" in output:
result.status = "error, unrecognized command, package not installed" result.status = "error, unrecognized command, package not installed"
@ -515,9 +516,12 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
continue continue
logger.info(f" Comparing thermo output from log.lammps against the reference log file {thermo_ref_file}") logger.info(f" Comparing thermo output from log.lammps against the reference log file {thermo_ref_file}")
# check if the number of runs matches with that in the reference log file # check if the number of runs matches with that in the reference log file
if num_runs != num_runs_ref: if num_runs != num_runs_ref:
logger.info(f"ERROR: Number of runs in log.lammps ({num_runs}) is not the same as that in the reference log ({num_runs_ref})") logger.info(f" ERROR: Number of runs in log.lammps ({num_runs}) is "
"different from that in the reference log ({num_runs_ref})."
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, incomplete runs" result.status = "error, incomplete runs"
results.append(result) results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n") progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
@ -526,6 +530,21 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
test_id = test_id + 1 test_id = test_id + 1
continue continue
# check if the number of fields match with that in the reference log file in the first run for early exit
num_fields = len(thermo[0]['keywords'])
num_fields_ref = len(thermo_ref[0]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo colums in log.lammps ({num_fields}) is "
"different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, mismatched columns in the log files"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# comparing output vs reference values # comparing output vs reference values
width = 20 width = 20
if verbose == True: if verbose == True:
@ -538,16 +557,24 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
if input_test in config['overrides']: if input_test in config['overrides']:
overrides = config['overrides'][input_test] overrides = config['overrides'][input_test]
# iterate through all num_runs # iterate through num_runs
num_abs_failed = 0 num_abs_failed = 0
num_rel_failed = 0 num_rel_failed = 0
failed_abs_output = [] failed_abs_output = []
failed_rel_output = [] failed_rel_output = []
num_checks = 0 num_checks = 0
mismatched_columns = False
for irun in range(num_runs): for irun in range(num_runs):
num_fields = len(thermo[irun]['keywords']) num_fields = len(thermo[irun]['keywords'])
num_fields_ref = len(thermo_ref[irun]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo columns in log.lammps ({num_fields}) is "
"different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the example folder, possibly due to the mpirun command.")
mismatched_columns = True
continue
# get the total number of the thermo output lines # get the total number of the thermo output lines
nthermo_steps = len(thermo[irun]['data']) nthermo_steps = len(thermo[irun]['data'])
@ -602,6 +629,13 @@ def iterate(lmp_binary, input_folder, input_list, config, results, progress_file
print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} " print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} "
"{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}") "{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}")
# after all runs completed, or are interrupted in one of the runs (mismatched_columns = True)
if mismatched_columns == True:
msg = f" mismatched log file."
print(msg)
logger.info(msg)
result.status = "failed"
if num_abs_failed > 0: if num_abs_failed > 0:
msg = f" {num_abs_failed} abs diff checks failed." msg = f" {num_abs_failed} abs diff checks failed."
print(msg) print(msg)
@ -662,7 +696,7 @@ if __name__ == "__main__":
progress_file = "progress.yaml" progress_file = "progress.yaml"
log_file = "run.log" log_file = "run.log"
list_input = "" list_input = ""
dry_run = False analyze = False
# distribute the total number of input scripts over the workers # distribute the total number of input scripts over the workers
num_workers = 1 num_workers = 1
@ -685,8 +719,8 @@ if __name__ == "__main__":
parser.add_argument("--output-file",dest="output", default=output_file, help="Output file") parser.add_argument("--output-file",dest="output", default=output_file, help="Output file")
parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file") parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file")
parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file") parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file")
parser.add_argument("--dry-run",dest="dry_run", action='store_true', default=False, parser.add_argument("--analyze",dest="analyze", action='store_true', default=False,
help="Only report statistics, not running the tests") help="Analyze and report statistics, not running the tests")
args = parser.parse_args() args = parser.parse_args()
@ -706,7 +740,7 @@ if __name__ == "__main__":
genref = args.genref genref = args.genref
verbose = args.verbose verbose = args.verbose
log_file = args.logfile log_file = args.logfile
dry_run = args.dry_run analyze = args.analyze
resume = args.resume resume = args.resume
progress_file = args.progress_file progress_file = args.progress_file
@ -814,8 +848,8 @@ if __name__ == "__main__":
else: else:
inplace_input = False inplace_input = False
# if only statistics, not running any test # if analyze the example folders (and split into separate lists for top-level examples), not running any test
if dry_run == True: if analyze == True:
quit() quit()
all_results = [] all_results = []
@ -904,17 +938,19 @@ if __name__ == "__main__":
# print out summary # print out summary
msg = "\nSummary:\n" msg = "\nSummary:\n"
msg += f" Total {total_tests} runs\n" msg += f" Total tests: {total_tests}\n"
msg += f" - {completed_tests} runs completed\n" msg += f" - Failed : {error_tests}\n"
msg += f" - {error_tests} runs failed with error\n" msg += f" - Completed: {completed_tests}\n"
msg += f" - {memleak_tests} runs with memory leak detected\n" if memleak_tests < completed_tests:
msg += f" - {passed_tests} numerical tests passed\n\n" msg += f" - memory leak detected : {memleak_tests}\n"
msg = "\nOutput:\n" if passed_tests < completed_tests:
msg += f" - Test results in JUnit XML format: {output_file}\n" msg += f" - numerical tests passed: {passed_tests}\n"
msg += "\nOutput:\n"
msg += f" - Running log with screen output: {log_file}\n"
msg += f" - Progress with the input list : {progress_file}\n" msg += f" - Progress with the input list : {progress_file}\n"
msg += f" - Running log : {log_file}" msg += f" - Regression test results : {output_file}\n"
print(msg)
print(msg)
# optional: need to check if junit_xml packaged is already installed in the env # optional: need to check if junit_xml packaged is already installed in the env
# generate a JUnit XML file # generate a JUnit XML file