ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handled more cases with ERROR in log.lammps

Browse Source
This commit is contained in:
Trung Nguyen
2024-07-26 14:11:43 -05:00
parent 90d98d9a92
commit e0857ad558
1 changed files with 389 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

780
tools/regression-tests/run_tests.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
''' '''
UPDATE: July 21, 2024: UPDATE: July 26, 2024:
Launching the LAMMPS binary under testing using a configuration defined in a yaml file (e.g. config.yaml). Launching the LAMMPS binary under testing using a configuration defined in a yaml file (e.g. config.yaml).
Comparing the output thermo with that in the existing log file (with the same nprocs) Comparing the output thermo with that in the existing log file (with the same nprocs)
+ data in the log files are extracted and converted into yaml data structure + data in the log files are extracted and converted into yaml data structure
@ -90,7 +90,391 @@ class TestResult:
self.checks = 0 self.checks = 0
self.status = status self.status = status
'''
Iterate over a list of input folders and scripts using the given lmp_binary and the testing configuration
lmp_binary : full path to the LAMMPS binary
input_folder : the absolute path to the input files
input_list : list of the input scripts under the input_folder
config : the dict that contains the test configuration
output_buf: placeholder for storing the output of a given worker
return
results : a list of TestResult objects
stat : a dictionary that lists the number of passed, skipped, failed tests
progress_file: yaml file that stores the tested input script and status
last_progress: the dictionary that shows the status of the last tests
NOTE:
To map a function to individual workers:
def func(input1, input2, output_buf):
# do smth
return result
# args is a list of num_workers tuples, each tuple contains the arguments passed to the function executed by a worker
args = []
for i in range(num_workers):
args.append((input1, input2, output_buf))
with Pool(num_workers) as pool:
results = pool.starmap(func, args)
'''
def iterate(lmp_binary, input_folder, input_list, config, results, progress_file, last_progress=None, output_buf=None):
EPSILON = np.float64(config['epsilon'])
nugget = float(config['nugget'])
num_tests = len(input_list)
num_completed = 0
num_passed = 0
num_skipped = 0
num_error = 0
num_memleak = 0
test_id = 0
# using REG-commented input scripts, now turned off (False)
using_markers = False
# iterate over the input scripts
for input in input_list:
# check if the progress file exists to append or create a new one
if os.path.isfile(progress_file) == True:
progress = open(progress_file, "a")
else:
progress = open(progress_file, "w")
# skip the input file if listed in the config file
if 'skip' in config:
if input in config['skip']:
msg = " + " + input + f" ({test_id+1}/{num_tests}): skipped as specified in {configFileName}"
print(msg)
logger.info(msg)
progress.write(f"{input}: {{ folder: {input_folder}, status: skipped }}\n")
progress.close()
num_skipped = num_skipped + 1
test_id = test_id + 1
continue
# also skip if the test already completed as marked in the progress file
if input in last_progress:
status = last_progress[input]['status']
if status == 'completed':
msg = " + " + input + f" ({test_id+1}/{num_tests}): marked as completed in the progress file {progress_file}"
logger.info(msg)
print(msg)
progress.write(msg)
progress.close()
num_skipped = num_skipped + 1
test_id = test_id + 1
continue
# if annotating input scripts with REG markers is True
if using_markers == True:
input_test = 'test.' + input
if os.path.isfile(input) == True:
if has_markers(input):
process_markers(input, input_test)
else:
print(f"WARNING: {input} does not have REG markers")
input_markers = input + '.markers'
# if the input file with the REG markers does not exist
# attempt to plug in the REG markers before each run command
if os.path.isfile(input_markers) == False:
cmd_str = "cp " + input + " " + input_markers
os.system(cmd_str)
generate_markers(input, input_markers)
process_markers(input_markers, input_test)
else:
# else the same file name for testing
input_test = input
str_t = " + " + input_test + f" ({test_id+1}/{num_tests})"
logger.info(str_t)
print(str_t)
# check if a log file exists in the current folder: log.DDMMMYY.basename.[nprocs]
basename = input_test.replace('in.','')
logfile_exist = False
# if there are multiple log files for different number of procs, pick the maximum number
cmd_str = "ls log.*"
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
logfile_list = p.stdout.split('\n')
logfile_list.remove('')
max_np = 1
for file in logfile_list:
# looks for pattern log.{date}.{basename}.g++.{nprocs}
# get the date from the log files
date = file.split('.',2)[1]
pattern = f'log.{date}.{basename}.*'
if fnmatch.fnmatch(file, pattern):
p = file.rsplit('.', 1)
if p[1].isnumeric():
if max_np < int(p[1]):
max_np = int(p[1])
logfile_exist = True
thermo_ref_file = file
# if the maximum number of procs is different from the value in the configuration file
# then override the setting for this input script
saved_nprocs = config['nprocs']
if max_np != int(config['nprocs']):
config['nprocs'] = str(max_np)
result = TestResult(name=input, output="", time="", status="passed")
# run the LAMMPS binary with the input script
cmd_str, output, error, returncode = execute(lmp_binary, config, input_test)
# restore the nprocs value in the configuration
config['nprocs'] = saved_nprocs
# check if a log.lammps file exists in the current folder
if os.path.isfile("log.lammps") == False:
logger.info(f" ERROR: No log.lammps generated with {input_test} with return code {returncode}. Check the {log_file} for the run output.\n")
logger.info(f"\n{input_test}:")
logger.info(f"\n{error}")
progress.write(f"{input}: {{ folder: {input_folder}, status: error, no log.lammps }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# process thermo output from the run
thermo = extract_data_to_yaml("log.lammps")
num_runs = len(thermo)
if "ERROR" in output or num_runs == 0:
cmd_str = "grep ERROR log.lammps"
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
error_line = p.stdout.split('\n')[0]
logger.info(f" The run terminated with {input_test} gives the following output:")
logger.info(f" {error_line}")
if "Unrecognized" in output:
result.status = "error, unrecognized command, package not installed"
elif "Unknown" in output:
result.status = "error, unknown command, package not installed"
else:
result.status = f"error, due to {error_line}."
logger.info(f" Failed with {input_test}.\n")
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# At this point, the run completed without trivial errors
# check if there is a reference log file for this input
if logfile_exist:
thermo_ref = extract_data_to_yaml(thermo_ref_file)
if thermo_ref:
num_runs_ref = len(thermo_ref)
else:
logger.info(f" ERROR: Error parsing {thermo_ref_file}.")
result.status = "skipped numerical checks due to parsing the log file"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: numerical checks skipped, unsupported log file format}}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
else:
msg = f" Cannot find the reference log file for {input_test} with the expected format log.[date].{basename}.*.[nprocs]"
logger.info(msg)
print(msg)
# try to read in the thermo yaml output from the working directory
thermo_ref_file = 'thermo.' + input + '.yaml'
file_exist = os.path.isfile(thermo_ref_file)
if file_exist == True:
thermo_ref = extract_thermo(thermo_ref_file)
num_runs_ref = len(thermo_ref)
else:
logger.info(f" {thermo_ref_file} also does not exist in the working directory.")
result.status = "skipped due to missing the reference log file"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: numerical checks skipped, missing the reference log file }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
logger.info(f" Comparing thermo output from log.lammps against the reference log file {thermo_ref_file}")
# check if the number of runs matches with that in the reference log file
if num_runs != num_runs_ref:
logger.info(f" ERROR: Number of runs in log.lammps ({num_runs}) is different from that in the reference log ({num_runs_ref})."
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, incomplete runs"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# check if the number of fields match with that in the reference log file in the first run for early exit
num_fields = len(thermo[0]['keywords'])
num_fields_ref = len(thermo_ref[0]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo colums in log.lammps ({num_fields}) is different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, mismatched columns in the log files"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# comparing output vs reference values
width = 20
if verbose == True:
print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) +
"Abs Diff Check".center(width) + "Rel Diff Check".center(width))
# check if overrides for this input scipt is specified
overrides = {}
if 'overrides' in config:
if input_test in config['overrides']:
overrides = config['overrides'][input_test]
# iterate through num_runs
num_abs_failed = 0
num_rel_failed = 0
failed_abs_output = []
failed_rel_output = []
num_checks = 0
mismatched_columns = False
for irun in range(num_runs):
num_fields = len(thermo[irun]['keywords'])
num_fields_ref = len(thermo_ref[irun]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo columns in log.lammps ({num_fields}) is "
"different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the example folder, possibly due to the mpirun command.")
mismatched_columns = True
continue
# get the total number of the thermo output lines
nthermo_steps = len(thermo[irun]['data'])
# get the output at the last timestep
thermo_step = nthermo_steps - 1
# iterate over the fields
for i in range(num_fields):
quantity = thermo[irun]['keywords'][i]
val = thermo[irun]['data'][thermo_step][i]
ref = thermo_ref[irun]['data'][thermo_step][i]
abs_diff = abs(float(val) - float(ref))
if abs(float(ref)) > EPSILON:
rel_diff = abs(float(val) - float(ref))/abs(float(ref))
else:
rel_diff = abs(float(val) - float(ref))/abs(float(ref)+nugget)
abs_diff_check = "PASSED"
rel_diff_check = "PASSED"
if quantity in config['tolerance'] or quantity in overrides:
if quantity in config['tolerance']:
abs_tol = float(config['tolerance'][quantity]['abs'])
rel_tol = float(config['tolerance'][quantity]['rel'])
# overrides the global tolerance values if specified
if quantity in overrides:
abs_tol = float(overrides[quantity]['abs'])
rel_tol = float(overrides[quantity]['rel'])
num_checks = num_checks + 2
if abs_diff > abs_tol:
abs_diff_check = "FAILED"
reason = f"Run {irun}: {quantity}: actual ({abs_diff:0.2e}) > expected ({abs_tol:0.2e})"
failed_abs_output.append(f"{reason}")
num_abs_failed = num_abs_failed + 1
if rel_diff > rel_tol:
rel_diff_check = "FAILED"
reason = f"Run {irun}: {quantity}: actual ({rel_diff:0.2e}) > expected ({rel_tol:0.2e})"
failed_rel_output.append(f"{reason}")
num_rel_failed = num_rel_failed + 1
else:
# N/A means that tolerances are not defined in the config file
abs_diff_check = "N/A"
rel_diff_check = "N/A"
if verbose == True and abs_diff_check != "N/A" and rel_diff_check != "N/A":
print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} "
"{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}")
# after all runs completed, or are interrupted in one of the runs (mismatched_columns = True)
if mismatched_columns == True:
msg = f" mismatched log file."
print(msg)
logger.info(msg)
result.status = "failed"
if num_abs_failed > 0:
msg = f" {num_abs_failed} abs diff thermo checks failed."
print(msg)
logger.info(msg)
result.status = "failed"
if verbose == True:
for i in failed_abs_output:
print(f"- {i}")
if num_rel_failed > 0:
msg = f" {num_rel_failed} rel diff thermo checks failed."
print(msg)
logger.info(msg)
result.status = "failed"
if verbose == True:
for i in failed_rel_output:
print(f"- {i}")
if num_abs_failed == 0 and num_rel_failed == 0:
msg = f" all {num_checks} thermo checks passed."
print(msg)
logger.info(msg)
result.status = "passed"
num_passed = num_passed + 1
results.append(result)
# check if memleak detects from valgrind run (need to replace "mpirun" -> valgrind --leak-check=yes mpirun")
msg = "completed"
if 'valgrind' in config['mpiexec']:
if "All heap blocks were free" in error:
msg += ", no memory leak"
else:
msg += ", memory leaks detected"
num_memleak = num_memleak + 1
progress.write(f"{input}: {{ folder: {input_folder}, status: {msg} }}\n")
progress.close()
# count the number of completed runs
num_completed = num_completed + 1
test_id = test_id + 1
stat = { 'num_completed': num_completed,
'num_passed': num_passed,
'num_skipped': num_skipped,
'num_error': num_error,
'num_memleak': num_memleak,
}
return stat
# HELPER FUNCTIONS
''' '''
get the thermo output from a log file with thermo style yaml get the thermo output from a log file with thermo style yaml
@ -310,396 +694,10 @@ def has_markers(inputFileName):
return True return True
return False return False
'''
Iterate over a list of input files using the given lmp_binary, the testing configuration
lmp_binary : full path to the LAMMPS binary
input_folder : the absolute path to the input files
input_list : list of the input scripts under the input_folder
config : the dict that contains the test configuration
removeAnnotatedInput: True if the annotated input script will be removed
output_buf: placeholder for storing the output of a given worker
return
results : a list of TestResult objects
stat : a dictionary that lists the number of passed, skipped, failed tests
progress_file: yaml file that stores the tested input script and status
last_progress: the dictionary that shows the status of the last tests
NOTE:
To map a function to individual workers:
def func(input1, input2, output_buf):
# do smth
return result
# args is a list of num_workers tuples, each tuple contains the arguments passed to the function executed by a worker
args = []
for i in range(num_workers):
args.append((input1, input2, output_buf))
with Pool(num_workers) as pool:
results = pool.starmap(func, args)
''' '''
def iterate(lmp_binary, input_folder, input_list, config, results, progress_file, last_progress=None, output_buf=None, removeAnnotatedInput=False): Main entry
EPSILON = np.float64(config['epsilon']) '''
nugget = float(config['nugget'])
num_tests = len(input_list)
num_completed = 0
num_passed = 0
num_skipped = 0
num_error = 0
num_memleak = 0
test_id = 0
# using REG-commented input scripts, now turned off (False)
using_markers = False
# iterate over the input scripts
for input in input_list:
if os.path.isfile(progress_file) == True:
progress = open(progress_file, "a")
else:
progress = open(progress_file, "w")
# skip the input file if listed
if 'skip' in config:
if input in config['skip']:
msg = " + " + input + f" ({test_id+1}/{num_tests}): skipped as specified in {configFileName}"
print(msg)
logger.info(msg)
progress.write(f"{input}: {{ folder: {input_folder}, status: skipped }}\n")
progress.close()
num_skipped = num_skipped + 1
test_id = test_id + 1
continue
# also skip if the test already completed
if input in last_progress:
status = last_progress[input]['status']
if status == 'completed':
msg = " + " + input + f" ({test_id+1}/{num_tests}): marked as completed in the progress file {progress_file}"
logger.info(msg)
print(msg)
progress.write(msg)
progress.close()
num_skipped = num_skipped + 1
test_id = test_id + 1
continue
result = TestResult(name=input, output="", time="", status="passed")
# if annotating input scripts with REG markers is True
if using_markers == True:
input_test = 'test.' + input
if os.path.isfile(input) == True:
if has_markers(input):
process_markers(input, input_test)
else:
print(f"WARNING: {input} does not have REG markers")
input_markers = input + '.markers'
# if the input file with the REG markers does not exist
# attempt to plug in the REG markers before each run command
if os.path.isfile(input_markers) == False:
cmd_str = "cp " + input + " " + input_markers
os.system(cmd_str)
generate_markers(input, input_markers)
process_markers(input_markers, input_test)
else:
# else the same file name for testing
input_test = input
str_t = " + " + input_test + f" ({test_id+1}/{num_tests})"
#logger.info(f"-"*len(str_t))
#print(f"-"*len(str_t))
logger.info(str_t)
print(str_t)
# check if a log file exists in the current folder: log.DDMMMYY.basename.[nprocs]
basename = input_test.replace('in.','')
logfile_exist = False
# if there are multiple log files for different number of procs, pick the maximum number
cmd_str = "ls log.*"
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
logfile_list = p.stdout.split('\n')
logfile_list.remove('')
max_np = 1
for file in logfile_list:
# looks for pattern log.{date}.{basename}.g++.{nprocs}
# get the date from the log files
date = file.split('.',2)[1]
pattern = f'log.{date}.{basename}.*'
if fnmatch.fnmatch(file, pattern):
p = file.rsplit('.', 1)
if p[1].isnumeric():
if max_np < int(p[1]):
max_np = int(p[1])
logfile_exist = True
thermo_ref_file = file
# if the maximum number of procs is different from the value in the configuration file
# then override the setting for this input script
saved_nprocs = config['nprocs']
if max_np != int(config['nprocs']):
config['nprocs'] = str(max_np)
# or more customizable with config.yaml
cmd_str, output, error, returncode = execute(lmp_binary, config, input_test)
# restore the nprocs value in the configuration
config['nprocs'] = saved_nprocs
# check if a log.lammps file exists in the current folder
if os.path.isfile("log.lammps") == False:
logger.info(f" ERROR: No log.lammps generated with {input_test} with return code {returncode}. Check the {log_file} for the run output.\n")
logger.info(f"\n{input_test}:")
logger.info(f"\n{error}")
progress.write(f"{input}: {{ folder: {input_folder}, status: error, no log.lammps }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# process thermo output from the run
thermo = extract_data_to_yaml("log.lammps")
num_runs = len(thermo)
if num_runs == 0:
logger.info(f"The run terminated with {input_test} gives the following output:")
logger.info(f"\n{output}")
if "Unrecognized" in output:
result.status = "error, unrecognized command, package not installed"
elif "Unknown" in output:
result.status = "error, unknown command, package not installed"
else:
result.status = "error, due to other reason. Check the {logfile} for more details."
logger.info(f"ERROR: Failed with {input_test} due to {result.status}.\n")
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# At this point, the run completed without trivial errors
# check if there is a reference log file for this input
if logfile_exist:
thermo_ref = extract_data_to_yaml(thermo_ref_file)
if thermo_ref:
num_runs_ref = len(thermo_ref)
else:
logger.info(f" ERROR: Error parsing {thermo_ref_file}.")
result.status = "skipped numerical checks due to parsing the log file"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: numerical checks skipped, unsupported log file format}}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
else:
msg = f" Cannot find the reference log file for {input_test} with the expected format log.[date].{basename}.*.[nprocs]"
logger.info(msg)
print(msg)
# try to read in the thermo yaml output from the working directory
thermo_ref_file = 'thermo.' + input + '.yaml'
file_exist = os.path.isfile(thermo_ref_file)
if file_exist == True:
thermo_ref = extract_thermo(thermo_ref_file)
num_runs_ref = len(thermo_ref)
else:
logger.info(f" {thermo_ref_file} also does not exist in the working directory.")
result.status = "skipped due to missing the reference log file"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: numerical checks skipped, missing the reference log file }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
logger.info(f" Comparing thermo output from log.lammps against the reference log file {thermo_ref_file}")
# check if the number of runs matches with that in the reference log file
if num_runs != num_runs_ref:
logger.info(f" ERROR: Number of runs in log.lammps ({num_runs}) is "
"different from that in the reference log ({num_runs_ref})."
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, incomplete runs"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# check if the number of fields match with that in the reference log file in the first run for early exit
num_fields = len(thermo[0]['keywords'])
num_fields_ref = len(thermo_ref[0]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo colums in log.lammps ({num_fields}) is "
"different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the folder, possibly due to the mpirun command.")
result.status = "error, mismatched columns in the log files"
results.append(result)
progress.write(f"{input}: {{ folder: {input_folder}, status: {result.status} }}\n")
progress.close()
num_error = num_error + 1
test_id = test_id + 1
continue
# comparing output vs reference values
width = 20
if verbose == True:
print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) +
"Abs Diff Check".center(width) + "Rel Diff Check".center(width))
# check if overrides for this input scipt is specified
overrides = {}
if 'overrides' in config:
if input_test in config['overrides']:
overrides = config['overrides'][input_test]
# iterate through num_runs
num_abs_failed = 0
num_rel_failed = 0
failed_abs_output = []
failed_rel_output = []
num_checks = 0
mismatched_columns = False
for irun in range(num_runs):
num_fields = len(thermo[irun]['keywords'])
num_fields_ref = len(thermo_ref[irun]['keywords'])
if num_fields != num_fields_ref:
logger.info(f" ERROR: Number of thermo columns in log.lammps ({num_fields}) is "
"different from that in the reference log ({num_fields_ref}) in run {irun}. "
"Check README in the example folder, possibly due to the mpirun command.")
mismatched_columns = True
continue
# get the total number of the thermo output lines
nthermo_steps = len(thermo[irun]['data'])
# get the output at the last timestep
thermo_step = nthermo_steps - 1
# iterate over the fields
for i in range(num_fields):
quantity = thermo[irun]['keywords'][i]
val = thermo[irun]['data'][thermo_step][i]
ref = thermo_ref[irun]['data'][thermo_step][i]
abs_diff = abs(float(val) - float(ref))
if abs(float(ref)) > EPSILON:
rel_diff = abs(float(val) - float(ref))/abs(float(ref))
else:
rel_diff = abs(float(val) - float(ref))/abs(float(ref)+nugget)
abs_diff_check = "PASSED"
rel_diff_check = "PASSED"
if quantity in config['tolerance'] or quantity in overrides:
if quantity in config['tolerance']:
abs_tol = float(config['tolerance'][quantity]['abs'])
rel_tol = float(config['tolerance'][quantity]['rel'])
# overrides the global tolerance values if specified
if quantity in overrides:
abs_tol = float(overrides[quantity]['abs'])
rel_tol = float(overrides[quantity]['rel'])
num_checks = num_checks + 2
if abs_diff > abs_tol:
abs_diff_check = "FAILED"
reason = f"Run {irun}: {quantity}: actual ({abs_diff:0.2e}) > expected ({abs_tol:0.2e})"
failed_abs_output.append(f"{reason}")
num_abs_failed = num_abs_failed + 1
if rel_diff > rel_tol:
rel_diff_check = "FAILED"
reason = f"Run {irun}: {quantity}: actual ({rel_diff:0.2e}) > expected ({rel_tol:0.2e})"
failed_rel_output.append(f"{reason}")
num_rel_failed = num_rel_failed + 1
else:
# N/A means that tolerances are not defined in the config file
abs_diff_check = "N/A"
rel_diff_check = "N/A"
if verbose == True and abs_diff_check != "N/A" and rel_diff_check != "N/A":
print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} "
"{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}")
# after all runs completed, or are interrupted in one of the runs (mismatched_columns = True)
if mismatched_columns == True:
msg = f" mismatched log file."
print(msg)
logger.info(msg)
result.status = "failed"
if num_abs_failed > 0:
msg = f" {num_abs_failed} abs diff thermo checks failed."
print(msg)
logger.info(msg)
result.status = "failed"
if verbose == True:
for i in failed_abs_output:
print(f"- {i}")
if num_rel_failed > 0:
msg = f" {num_rel_failed} rel diff thermo checks failed."
print(msg)
logger.info(msg)
result.status = "failed"
if verbose == True:
for i in failed_rel_output:
print(f"- {i}")
if num_abs_failed == 0 and num_rel_failed == 0:
msg = f" all {num_checks} thermo checks passed."
print(msg)
logger.info(msg)
result.status = "passed"
num_passed = num_passed + 1
results.append(result)
# check if memleak detects from valgrind run (need to replace "mpirun" -> valgrind --leak-check=yes mpirun")
msg = "completed"
if 'valgrind' in config['mpiexec']:
if "All heap blocks were free" in error:
msg += ", no memory leak"
else:
msg += ", memory leaks detected"
num_memleak = num_memleak + 1
progress.write(f"{input}: {{ folder: {input_folder}, status: {msg} }}\n")
progress.close()
# count the number of completed runs
num_completed = num_completed + 1
test_id = test_id + 1
# remove the annotated input script
if removeAnnotatedInput == True and inplace_input == False:
cmd_str = "rm " + input_test
os.system(cmd_str)
stat = { 'num_completed': num_completed,
'num_passed': num_passed,
'num_skipped': num_skipped,
'num_error': num_error,
'num_memleak': num_memleak
}
return stat
if __name__ == "__main__": if __name__ == "__main__":
# default values # default values
@ -737,7 +735,7 @@ if __name__ == "__main__":
parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file") parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file")
parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file") parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file")
parser.add_argument("--analyze",dest="analyze", action='store_true', default=False, parser.add_argument("--analyze",dest="analyze", action='store_true', default=False,
help="Analyze and report statistics, not running the tests") help="Analyze the testing folders and report statistics, not running the tests")
args = parser.parse_args() args = parser.parse_args()
@ -901,7 +899,7 @@ if __name__ == "__main__":
# default setting is to use inplace_input # default setting is to use inplace_input
if inplace_input == True: if inplace_input == True:
# change dir to a folder under examples/, need to use os.chdir() # change dir to a folder under examples/
# TODO: loop through the subfolders under examples/, depending on the installed packages # TODO: loop through the subfolders under examples/, depending on the installed packages
''' '''