ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

integrate quick regression support into regression tester

Browse Source
This commit is contained in:
Axel Kohlmeyer
2024-08-31 23:16:02 -04:00
parent 62bfd7dc74
commit 6fb50cbdc1
5 changed files with 220 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/full-regression.yml vendored
View File

@ -11,14 +11,14 @@ on:
jobs:
build:
name: Build
name: Build LAMMPS
# restrict to official LAMMPS repository
if: ${{ github.repository == 'lammps/lammps' }}
runs-on: ubuntu-latest
env:
CCACHE_DIR: ${{ github.workspace }}/.ccache
strategy:
max-parallel: 2
max-parallel: 4
matrix:
idx: [ 0, 1, 2, 3 ]
@ -71,7 +71,7 @@ jobs:
cmake --build build
ccache -s
- name: Full regression tests, splitting the top-level example input into 4 lists
- name: Run Full Regression Tests
shell: bash
run: |
source linuxenv/bin/activate

36
.github/workflows/quick-regression.yml vendored
View File

@ -14,11 +14,16 @@ on:
jobs:
build:
name: Quick Regression Test
name: Build LAMMPS
# restrict to official LAMMPS repository
if: ${{ github.repository == 'lammps/lammps' }}
runs-on: ubuntu-latest
env:
CCACHE_DIR: ${{ github.workspace }}/.ccache
strategy:
max-parallel: 4
matrix:
idx: [ 0, 1, 2, 3 ]
steps:
- name: Checkout repository
@ -70,21 +75,38 @@ jobs:
cmake --build build
ccache -s
- name: Run Selected Regression Tests
- name: Run Regression Tests for Modified Styles
shell: bash
run: |
source linuxenv/bin/activate
python3 tools/regression-tests/get-quick-list.py
python3 tools/regression-tests/run_tests.py \
--lmp-bin=build/lmp \
--config-file=tools/regression-tests/config_serial.yaml \
--list-input=input_list.txt
tar -cvf quick-regression-test.tar run.log progress.yaml
--examples-top-level=examples --quick --quick-branch=origin/develop --num-workers=4
python3 tools/regression-tests/run_tests.py \
--lmp-bin=build/lmp \
--config-file=tools/regression-tests/config_serial.yaml \
--list-input=input-list-${{ matrix.idx }}.txt \
--output-file=output-${{ matrix.idx }}.xml \
--progress-file=progress-${{ matrix.idx }}.yaml \
--log-file=run-${{ matrix.idx }}.log
tar -cvf quick-regression-test-${{ matrix.idx }}.tar run-${{ matrix.idx }}.log progress-${{ matrix.idx }}.yaml output-${{ matrix.idx }}.xml
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: quick-regression-test-artifact
path: quick-regression-test.tar
name: quick-regression-test-artifact-${{ matrix.idx }}
path: quick-regression-test-${{ matrix.idx }}.tar
merge:
runs-on: ubuntu-latest
needs: build
steps:
- name: Merge Artifacts
uses: actions/upload-artifact/merge@v4
with:
name: merged-quick-regresssion-artifact
pattern: quick-regression-test-artifact-*

2
tools/regression-tests/config_serial.yaml
View File

@ -36,7 +36,7 @@
in.bucky-plus-cnt*,
]
timeout: 10
timeout: 30
nugget: 1.0
epsilon: 1e-16

14
tools/regression-tests/get-quick-list.py → tools/regression-tests/get_quick_list.py
View File

@ -244,7 +244,7 @@ def get_examples_using_styles(regex, examples='examples'):
with open(filename) as f:
for line in f:
if commands.match(line):
inputs.append(filename)
inputs.append(str(filename))
break
return inputs
@ -258,14 +258,8 @@ if __name__ == "__main__":
regex = make_regex(styles)
if regex:
inputs = get_examples_using_styles(regex, os.path.join(LAMMPS_DIR,'examples'))
print("Suggested inputs for testing:")
# input_list.txt is used for the regression tester tool
with open('input_list.txt', 'w') as f:
for inp in inputs:
print(inp)
f.write(str(inp) + '\n')
else:
inputs = []
print("Found changes to the following styles:")
print("Commands: ", styles['command'])
print("Atom styles: ", styles['atom'])
@ -282,3 +276,5 @@ if __name__ == "__main__":
print("Region styles: ", styles['region'])
print("Integrate styles: ", styles['integrate'])
print("Minimize styles: ", styles['minimize'])
print("Example input files affected: ", len(inputs))

244
tools/regression-tests/run_tests.py
View File

@ -58,7 +58,7 @@ Example usage:
5) Test a LAMMPS binary with the whole top-level /examples folder in a LAMMPS source tree
python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples
6) Analyze the LAMMPS binary annd whole top-level /examples folder in a LAMMPS source tree
6) Analyze the LAMMPS binary and whole top-level /examples folder in a LAMMPS source tree
and generate separate input lists for 8 workers:
python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples \
--analyze --num-workers=8
@ -76,6 +76,7 @@ import logging
import os
import re
import subprocess
import sys
#from multiprocessing import Pool
# need "pip install numpy pyyaml"
@ -90,6 +91,13 @@ try:
except ImportError:
from yaml import SafeLoader as Loader
# infer top level LAMMPS dir from filename
LAMMPS_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))
# import git interface module
sys.path.append(os.path.realpath(os.path.join(LAMMPS_DIR, 'tools', 'regression-tests')))
import get_quick_list
'''
data structure to store the test result
'''
@ -881,6 +889,8 @@ if __name__ == "__main__":
list_input = ""
list_subfolders = ""
analyze = False
quick = False
quick_branch = "origin/develop"
# distribute the total number of input scripts over the workers
num_workers = 1
@ -888,9 +898,9 @@ if __name__ == "__main__":
# parse the arguments
parser = ArgumentParser()
parser.add_argument("--lmp-bin", dest="lmp_binary", default="", help="LAMMPS binary")
parser.add_argument("--config-file", dest="config_file", default=configFileName,
help="Configuration YAML file")
parser.add_argument("--examples-top-level", dest="example_toplevel", default="", help="Examples top-level")
parser.add_argument("--config-file", dest="config_file", default=configFileName, help="Configuration YAML file")
parser.add_argument("--examples-top-level", dest="example_toplevel", default=os.path.join(LAMMPS_DIR, 'examples'),
help="Examples top-level")
parser.add_argument("--example-folders", dest="example_folders", default="", help="Example subfolders")
parser.add_argument("--list-input", dest="list_input", default="", help="File that lists the input scripts")
parser.add_argument("--list-subfolders", dest="list_subfolders", default="", help="File that lists the subfolders")
@ -904,8 +914,13 @@ if __name__ == "__main__":
parser.add_argument("--output-file",dest="output", default=output_file, help="Output file")
parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file")
parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file")
parser.add_argument("--analyze",dest="analyze", action='store_true', default=False,
analyze = parser.add_mutually_exclusive_group()
analyze.add_argument("--analyze",dest="analyze", action='store_true', default=False,
help="Analyze the testing folders and report statistics, not running the tests")
analyze.add_argument("--quick", dest="quick", action='store_true', default=False,
help="Determine which test inputs have commands changed between a branch and the head")
parser.add_argument("--quick-branch", dest="quick_branch", default=quick_branch,
help="Branch to which compare the current head to for changed styles")
parser.add_argument("--skip-numerical-check",dest="skip_numerical_check", action='store_true', default=False,
help="Generating reference data")
@ -929,6 +944,8 @@ if __name__ == "__main__":
verbose = args.verbose
log_file = args.logfile
analyze = args.analyze
quick = args.quick
quick_branch = args.quick_branch
skip_numerical_check = args.skip_numerical_check
resume = args.resume
progress_file = args.progress_file
@ -948,51 +965,20 @@ if __name__ == "__main__":
inplace_input = True
test_cases = []
# if the example folders are not specified from the command-line argument --example-folders
# then use the path from --example-top-folder, or from the input-list read from a text file
if len(example_subfolders) == 0:
# if the top level is specified
if len(example_toplevel) != 0:
# getting the list of all the input files because there are subfolders (e.g. PACKAGES) under the top level
cmd_str = f"find {example_toplevel} -name \"in.*\" "
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
input_list = p.stdout.split('\n')
input_list.remove("")
msg = f"\nThere are {len(input_list)} input scripts in total under the {example_toplevel} folder."
# generate list of input scripts with commands that have been changed
if quick:
headers = get_quick_list.changed_files_from_git(quick_branch)
print("headers ", headers)
styles = get_quick_list.get_command_from_header(headers, LAMMPS_DIR)
print("styles ", styles)
regex = get_quick_list.make_regex(styles)
print("regex ", regex)
if regex:
input_list = get_quick_list.get_examples_using_styles(regex, example_toplevel)
msg = f"\nThere are {len(input_list)} input scripts with changed styles relative to branch {quick_branch}."
print(msg)
logger.info(msg)
# get the input file list
# TODO: generate a list of tuples, each tuple contains a folder list for a worker,
# then use multiprocessing.Pool starmap()
folder_list = []
for input in input_list:
folder = input.rsplit('/', 1)[0]
# unique folders in the list
if folder not in folder_list:
folder_list.append(folder)
# divide the list of folders into num_workers chunks
sublists = divide_into_N(folder_list, num_workers)
# write each chunk to a file
idx = 0
for list_input in sublists:
filename = f"folder-list-{idx}.txt"
with open(filename, "w") as f:
for folder in list_input:
# count the number of input scripts in each folder
cmd_str = f"ls {folder}/in.* | wc -l"
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
num_input = p.stdout.split('\n')[0]
f.write(folder + ' ' + num_input + '\n')
f.close()
idx = idx + 1
# working on all the folders for now
example_subfolders = folder_list
# divide the list of input scripts into num_workers chunks
sublists = divide_into_N(input_list, num_workers)
@ -1005,53 +991,125 @@ if __name__ == "__main__":
f.write(inp + '\n')
f.close()
idx = idx + 1
# if a list of subfolders is provided from a text file (list_subfolders from the command-line argument)
elif len(list_subfolders) != 0:
num_inputscripts = 0
with open(list_subfolders, "r") as f:
all_subfolders = f.read().splitlines()
f.close()
for line in all_subfolders:
if len(line) > 0:
# skip subfolders
if line[0] == '#':
continue
folder = line.split()[0]
example_subfolders.append(folder)
num_inputscripts += int(line.split()[1])
msg = f"\nThere are {len(example_subfolders)} folders with {num_inputscripts} input scripts in total listed in {list_input}."
print(msg)
logger.info(msg)
# if a list of input scripts is provided from a text file (list_input from the command-line argument)
elif len(list_input) != 0:
num_inputscripts = 0
folder_list = []
with open(list_input, "r") as f:
all_inputs = f.read().splitlines()
f.close()
for line in all_inputs:
if len(line) > 0:
# skip input scripts
if line[0] == '#':
continue
input = line.split()[0]
folder = input.rsplit('/', 1)[0]
# unique folders in the list
if folder not in folder_list:
folder_list.append(folder)
example_inputs.append(input)
num_inputscripts += 1
example_subfolders = folder_list
msg = f"\nThere are {num_inputscripts} input scripts listed in {list_input}."
print(msg)
logger.info(msg)
else:
inplace_input = False
msg = f"\nThere are no input scripts with changed styles relative to branch {quick_branch}."
print(msg)
logger.info(msg)
for idx in range(0, num_workers):
try:
os.remove(f"folder-list-{idx}.txt")
except:
pass
try:
os.remove(f"input-list-{idx}.txt")
except:
pass
quit()
else:
# if the example folders are not specified from the command-line argument --example-folders
# then use the path from --example-top-folder, or from the input-list read from a text file
if len(example_subfolders) == 0:
# if the top level is specified
if len(example_toplevel) != 0:
# getting the list of all the input files because there are subfolders (e.g. PACKAGES) under the top level
cmd_str = f"find {example_toplevel} -name \"in.*\" "
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
input_list = p.stdout.split('\n')
input_list.remove("")
msg = f"\nThere are {len(input_list)} input scripts in total under the {example_toplevel} folder."
print(msg)
logger.info(msg)
# get the input file list
# TODO: generate a list of tuples, each tuple contains a folder list for a worker,
# then use multiprocessing.Pool starmap()
folder_list = []
for input in input_list:
folder = input.rsplit('/', 1)[0]
# unique folders in the list
if folder not in folder_list:
folder_list.append(folder)
# divide the list of folders into num_workers chunks
sublists = divide_into_N(folder_list, num_workers)
# write each chunk to a file
idx = 0
for list_input in sublists:
filename = f"folder-list-{idx}.txt"
with open(filename, "w") as f:
for folder in list_input:
# count the number of input scripts in each folder
cmd_str = f"ls {folder}/in.* | wc -l"
p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True)
num_input = p.stdout.split('\n')[0]
f.write(folder + ' ' + num_input + '\n')
f.close()
idx = idx + 1
# working on all the folders for now
example_subfolders = folder_list
# divide the list of input scripts into num_workers chunks
sublists = divide_into_N(input_list, num_workers)
# write each chunk to a file
idx = 0
for list_input in sublists:
filename = f"input-list-{idx}.txt"
with open(filename, "w") as f:
for inp in list_input:
f.write(inp + '\n')
f.close()
idx = idx + 1
# if a list of subfolders is provided from a text file (list_subfolders from the command-line argument)
elif len(list_subfolders) != 0:
num_inputscripts = 0
with open(list_subfolders, "r") as f:
all_subfolders = f.read().splitlines()
f.close()
for line in all_subfolders:
if len(line) > 0:
# skip subfolders
if line[0] == '#':
continue
folder = line.split()[0]
example_subfolders.append(folder)
num_inputscripts += int(line.split()[1])
msg = f"\nThere are {len(example_subfolders)} folders with {num_inputscripts} input scripts in total listed in {list_input}."
print(msg)
logger.info(msg)
# if a list of input scripts is provided from a text file (list_input from the command-line argument)
elif len(list_input) != 0:
num_inputscripts = 0
folder_list = []
with open(list_input, "r") as f:
all_inputs = f.read().splitlines()
f.close()
for line in all_inputs:
if len(line) > 0:
# skip input scripts
if line[0] == '#':
continue
input = line.split()[0]
folder = input.rsplit('/', 1)[0]
# unique folders in the list
if folder not in folder_list:
folder_list.append(folder)
example_inputs.append(input)
num_inputscripts += 1
example_subfolders = folder_list
msg = f"\nThere are {num_inputscripts} input scripts listed in {list_input}."
print(msg)
logger.info(msg)
else:
inplace_input = False
# if analyze the example folders (and split into separate lists for top-level examples), not running any test
if analyze == True: