diff --git a/doc/src/Tools.rst b/doc/src/Tools.rst index f651e93e32..6c12baf967 100644 --- a/doc/src/Tools.rst +++ b/doc/src/Tools.rst @@ -96,6 +96,7 @@ Miscellaneous tools * :ref:`LAMMPS-GUI ` * :ref:`LAMMPS magic patterns for file(1) ` * :ref:`Offline build tool ` + * :ref:`Regression tester ` * :ref:`singularity/apptainer ` * :ref:`SWIG interface ` * :ref:`valgrind ` @@ -991,6 +992,30 @@ README for more info on Pizza.py and how to use these scripts. ---------- +.. _regression: + +Regression tester tool +---------------------- + +The regression-tests subdirectory contains a tool for performing +regression tests with a given LAMMPS binary. The tool launches the +LAMMPS binary with any given input script under one of the `examples` +subdirectories, and compares the thermo output in the generated log file +with those in the provided log file with the same number of processors +ub the same subdirectory. If the differences between the actual and +reference values are within specified tolerances, the test is considered +passed. For each test batch, that is, a set of example input scripts, +the mpirun command, the LAMMPS command line arguments, and the +tolerances for individual thermo quantities can be specified in a +configuration file in YAML format. + +The tool also reports if and how the run fails, and if a reference log file +is missing. See the README file for more information. + +This tool was written by Trung Nguyen at U of Chicago (ndactrung at gmail.com). + +---------- + .. _replica: replica tool diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 200770fa5e..f1d9c0b748 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -2472,6 +2472,7 @@ ncol ncorr ncount nd +ndactrung ndescriptors ndihedrals Ndihedraltype diff --git a/examples/bpm/impact/log.17Feb2022.impact.spring.g++.4 b/examples/bpm/impact/log.17Feb2022.bpm.impact.spring.g++.4 similarity index 100% rename from examples/bpm/impact/log.17Feb2022.impact.spring.g++.4 rename to examples/bpm/impact/log.17Feb2022.bpm.impact.spring.g++.4 diff --git a/examples/bpm/impact/log.4May2022.impact.rotational.g++.4 b/examples/bpm/impact/log.4May2022.impact.rotational.g++.4 deleted file mode 100644 index 8eb87d3c2e..0000000000 --- a/examples/bpm/impact/log.4May2022.impact.rotational.g++.4 +++ /dev/null @@ -1,219 +0,0 @@ -LAMMPS (4 May 2022) -units lj -dimension 3 -boundary f f f -atom_style bpm/sphere -special_bonds lj 0.0 1.0 1.0 coul 0.0 1.0 1.0 -newton on off -comm_modify vel yes cutoff 2.6 -lattice fcc 1.0 -Lattice spacing in x,y,z = 1.5874011 1.5874011 1.5874011 -region box block -25 15 -22 22 -22 22 -create_box 1 box bond/types 2 extra/bond/per/atom 20 extra/special/per/atom 50 -Created orthogonal box = (-39.685026 -34.922823 -34.922823) to (23.811016 34.922823 34.922823) - 1 by 2 by 2 MPI processor grid - -region disk cylinder x 0.0 0.0 20.0 -0.5 0.5 -create_atoms 1 region disk -Created 7529 atoms - using lattice units in orthogonal box = (-39.685026 -34.922823 -34.922823) to (23.811016 34.922823 34.922823) - create_atoms CPU = 0.002 seconds -group plate region disk -7529 atoms in group plate - -region ball sphere 8.0 0.0 0.0 6.0 -create_atoms 1 region ball -Created 3589 atoms - using lattice units in orthogonal box = (-39.685026 -34.922823 -34.922823) to (23.811016 34.922823 34.922823) - create_atoms CPU = 0.001 seconds -group projectile region ball -3589 atoms in group projectile - -displace_atoms all random 0.1 0.1 0.1 134598738 -Displacing atoms ... - -neighbor 1.0 bin -pair_style gran/hooke/history 1.0 NULL 0.5 NULL 0.1 1 -pair_coeff 1 1 - -fix 1 all nve/bpm/sphere - -create_bonds many plate plate 1 0.0 1.5 -Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule -Neighbor list info ... - update every 1 steps, delay 10 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 2 - ghost atom cutoff = 2.6 - binsize = 1, bins = 64 70 70 - 2 neighbor lists, perpetual/occasional/extra = 1 1 0 - (1) command create_bonds, occasional - attributes: full, newton on - pair build: full/bin - stencil: full/bin/3d - bin: standard - (2) pair gran/hooke/history, perpetual - attributes: half, newton on, size, history - pair build: half/size/bin/newton - stencil: half/bin/3d - bin: standard -Added 38559 bonds, new total = 38559 -Finding 1-2 1-3 1-4 neighbors ... - special bond factors lj: 0 1 1 - special bond factors coul: 0 1 1 - 15 = max # of 1-2 neighbors - 101 = max # of special neighbors - special bonds CPU = 0.001 seconds -create_bonds many projectile projectile 2 0.0 1.5 -Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule -WARNING: Bonds are defined but no bond style is set (../force.cpp:192) -WARNING: Likewise 1-2 special neighbor interactions != 1.0 (../force.cpp:194) -Added 21869 bonds, new total = 60428 -Finding 1-2 1-3 1-4 neighbors ... - special bond factors lj: 0 1 1 - special bond factors coul: 0 1 1 - 16 = max # of 1-2 neighbors - 101 = max # of special neighbors - special bonds CPU = 0.002 seconds - -neighbor 0.3 bin -special_bonds lj 0.0 1.0 1.0 coul 1.0 1.0 1.0 - -bond_style bpm/rotational store/local brkbond 100 time id1 id2 -bond_coeff 1 1.0 0.2 0.02 0.02 0.05 0.01 0.01 0.01 0.1 0.2 0.002 0.002 -bond_coeff 2 1.0 0.2 0.02 0.02 0.20 0.04 0.04 0.04 0.1 0.2 0.002 0.002 - -velocity projectile set -0.05 0.0 0.0 -compute nbond all nbond/atom -compute tbond all reduce sum c_nbond - -timestep 0.05 -thermo_style custom step ke pe pxx pyy pzz c_tbond -thermo 100 -thermo_modify lost ignore lost/bond ignore -#dump 1 all custom 100 atomDump id radius x y z c_nbond - -dump 2 all local 100 brokenDump f_brkbond[1] f_brkbond[2] f_brkbond[3] -dump_modify 2 header no - -run 7500 -Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule -Neighbor list info ... - update every 1 steps, delay 10 steps, check yes - max neighbors/atom: 2000, page size: 100000 - master list distance cutoff = 1.3 - ghost atom cutoff = 2.6 - binsize = 0.65, bins = 98 108 108 - 1 neighbor lists, perpetual/occasional/extra = 1 0 0 - (1) pair gran/hooke/history, perpetual - attributes: half, newton on, size, history - pair build: half/size/bin/newton - stencil: half/bin/3d - bin: standard -Per MPI rank memory allocation (min/avg/max) = 33.22 | 33.22 | 33.22 Mbytes - Step KinEng PotEng Pxx Pyy Pzz c_tbond - 0 0.00053238861 0 3.8217307e-05 -7.6534847e-14 1.9906102e-13 10.8703 - 100 0.00053238861 0 3.8217307e-05 -4.2831948e-13 5.7428612e-13 10.8703 - 200 0.00053238861 0 3.8217307e-05 -1.4067913e-12 1.4975493e-12 10.8703 - 300 0.00053238861 0 3.8217307e-05 -8.77782e-13 3.8245851e-13 10.8703 - 400 0.00053238861 0 3.8217307e-05 -8.5835238e-13 6.5448014e-13 10.8703 - 500 0.00053093549 0 4.0340893e-05 4.501394e-07 5.3690512e-07 10.8703 - 600 0.00051485902 0 6.6761034e-05 4.6258948e-06 5.2285428e-06 10.869221 - 700 0.00049942978 0 9.5499005e-05 9.3031413e-07 4.5389354e-06 10.85501 - 800 0.00049465262 0 5.6810167e-05 -5.5619903e-06 -1.6010295e-06 10.820651 - 900 0.00048784775 0 1.5747004e-05 2.0522042e-05 2.5481542e-05 10.769563 - 1000 0.00048345699 0 2.1159666e-05 1.2232747e-05 1.4767665e-05 10.730347 - 1100 0.00047945073 0 5.2779833e-05 -2.6136504e-05 -2.7689007e-05 10.703184 - 1200 0.00047512604 0 6.3234312e-05 -1.7082136e-05 -2.9178979e-05 10.686634 - 1300 0.00047401428 0 2.5474717e-05 -7.4782876e-06 -1.9808965e-05 10.678 - 1400 0.00047619121 0 2.5189461e-05 2.9476227e-06 -4.4149511e-06 10.671704 - 1500 0.0004668728 0 5.8798861e-05 -2.6192143e-06 1.0805172e-06 10.666127 - 1600 0.00045088371 0 5.8377694e-05 2.2911269e-06 4.339717e-06 10.66073 - 1700 0.00044275099 0 4.0766018e-05 2.7748031e-05 2.8202527e-05 10.6458 - 1800 0.0004424362 0 3.0460351e-05 2.9690484e-05 3.3464889e-05 10.620615 - 1900 0.00043678957 0 3.809598e-05 2.4448755e-06 1.2651201e-05 10.603166 - 2000 0.00042747562 0 4.2315209e-05 -7.6783024e-06 -3.3357359e-06 10.576003 - 2100 0.0004214344 0 2.6171505e-05 5.5424035e-06 -4.1153085e-06 10.539845 - 2200 0.00041712779 0 3.0796803e-05 1.1362383e-05 7.7103875e-07 10.49937 - 2300 0.00041095769 0 4.141148e-05 1.4142023e-06 -1.0982633e-05 10.471668 - 2400 0.00040883568 0 3.5835323e-05 -1.0216635e-05 -2.3669763e-05 10.45116 - 2500 0.00040762685 0 2.879385e-05 -1.4074395e-05 -1.9314875e-05 10.437309 - 2600 0.00040579873 0 2.771644e-05 -2.316844e-05 -2.2961097e-05 10.422378 - 2700 0.00040340975 0 3.4482945e-05 -3.075929e-05 -2.3321344e-05 10.410505 - 2800 0.00040170914 0 3.1147943e-05 -2.4329639e-05 -1.1787678e-05 10.400792 - 2900 0.00040015113 0 2.3214992e-05 -1.8068374e-05 3.8127871e-06 10.393416 - 3000 0.00040029253 0 2.7275906e-05 -7.0762689e-06 1.3782424e-05 10.385321 - 3100 0.00040037329 0 2.962113e-05 -1.3795312e-06 5.3267315e-06 10.378485 - 3200 0.00040142612 0 2.86096e-05 4.4289601e-06 -3.3950404e-06 10.373988 - 3300 0.00040105092 0 2.5423615e-05 9.5689359e-06 -2.5296344e-06 10.370031 - 3400 0.00039950673 0 2.6405258e-05 9.5776239e-06 -7.3789982e-07 10.364274 - 3500 0.00039715236 0 3.115696e-05 1.0986224e-05 6.6898207e-06 10.360496 - 3600 0.00039446552 0 2.8426837e-05 9.6296098e-06 1.5057875e-05 10.353121 - 3700 0.00039263296 0 2.567768e-05 6.347291e-06 2.4842157e-05 10.346645 - 3800 0.00039061341 0 2.7635193e-05 5.0165135e-06 2.5989901e-05 10.341069 - 3900 0.00038985051 0 2.8639932e-05 1.056365e-05 2.4463803e-05 10.329196 - 4000 0.00038815347 0 2.6613146e-05 2.0316396e-05 2.1434368e-05 10.318043 - 4100 0.00038651302 0 2.4759493e-05 1.9632349e-05 1.3524912e-05 10.311027 - 4200 0.00038565809 0 2.5290845e-05 1.9908233e-05 8.3895516e-06 10.299155 - 4300 0.0003847255 0 2.6461182e-05 1.9385332e-05 5.664874e-06 10.292319 - 4400 0.0003844421 0 2.4464435e-05 1.4675545e-05 6.8223863e-06 10.286203 - 4500 0.0003842788 0 2.3080348e-05 7.1469159e-06 6.8395849e-06 10.278287 - 4600 0.00038365139 0 2.4937615e-05 2.3854793e-06 4.6100509e-06 10.270732 - 4700 0.00038271503 0 2.431281e-05 -3.127707e-06 3.8840673e-07 10.264796 - 4800 0.00038233688 0 2.3727372e-05 -3.9575833e-06 1.5658614e-06 10.25742 - 4900 0.00038223496 0 2.3842519e-05 2.6005447e-06 4.5031882e-06 10.246987 - 5000 0.00038219402 0 2.4705111e-05 8.2018492e-06 4.0121467e-06 10.240511 - 5100 0.00038195153 0 2.5112089e-05 9.1687723e-06 3.3825795e-06 10.236014 - 5200 0.00038170903 0 2.4166312e-05 4.6680528e-06 3.0359762e-06 10.232236 - 5300 0.00038194303 0 2.4293657e-05 3.067111e-06 2.1353614e-06 10.230617 - 5400 0.00038147407 0 2.472142e-05 5.2915485e-06 -1.7472423e-06 10.230098 - 5500 0.00038156894 0 2.4839317e-05 6.6216457e-06 -2.7544087e-06 10.227759 - 5600 0.00038169434 0 2.4600407e-05 3.8679618e-06 1.2622251e-07 10.2256 - 5700 0.00038219734 0 2.4459589e-05 2.0025919e-07 -1.1917672e-08 10.223621 - 5800 0.00038268758 0 2.4768428e-05 8.7913744e-07 -1.4000329e-06 10.222902 - 5900 0.00038300658 0 2.4827866e-05 3.6621944e-06 -2.2178729e-07 10.222182 - 6000 0.00038250316 0 2.4309432e-05 4.3755483e-06 2.2736019e-06 10.221123 - 6100 0.0003821526 0 2.4396115e-05 3.3855804e-06 4.4742551e-06 10.219503 - 6200 0.00038185711 0 2.4795348e-05 1.7569948e-06 4.3229904e-06 10.219503 - 6300 0.00038197679 0 2.4817115e-05 1.237731e-07 3.9285574e-06 10.218604 - 6400 0.00038232311 0 2.4723664e-05 -8.7946112e-07 2.6215619e-06 10.217884 - 6500 0.00038255543 0 2.4821878e-05 -4.8948257e-07 3.9392146e-07 10.217704 - 6600 0.00038251887 0 2.4923895e-05 -1.1107041e-07 -8.3900047e-07 10.217344 - 6700 0.00038177389 0 2.4664007e-05 -6.4474733e-07 -2.1004826e-06 10.218084 - 6800 0.00038096291 0 2.4262293e-05 -1.7159941e-06 -2.8149643e-06 10.218103 - 6900 0.00038090601 0 2.4179172e-05 -2.2622259e-06 -2.1268271e-06 10.217024 - 7000 0.00038088094 0 2.4363443e-05 -2.4652531e-06 -8.209416e-07 10.215944 - 7100 0.00038094624 0 2.5119358e-05 -1.5432706e-06 1.1465135e-06 10.214684 - 7200 0.00038168738 0 2.5565338e-05 -1.4052753e-07 3.3146851e-06 10.212705 - 7300 0.00038200854 0 2.5436565e-05 4.353807e-07 3.3504276e-06 10.212345 - 7400 0.00038187543 0 2.4764819e-05 6.7297502e-07 1.5923471e-06 10.213084 - 7500 0.00038165297 0 2.4015684e-05 7.8657712e-07 1.0138435e-06 10.214742 -Loop time of 32.2292 on 4 procs for 7500 steps with 11111 atoms - -Performance: 1005300.744 tau/day, 232.709 timesteps/s -96.2% CPU use with 4 MPI tasks x no OpenMP threads - -MPI task timing breakdown: -Section | min time | avg time | max time |%varavg| %total ---------------------------------------------------------------- -Pair | 0.29763 | 0.30464 | 0.31393 | 1.1 | 0.95 -Bond | 25.936 | 26.533 | 27.431 | 11.7 | 82.33 -Neigh | 0.60911 | 0.63557 | 0.66475 | 2.8 | 1.97 -Comm | 1.7247 | 2.7138 | 3.3823 | 40.5 | 8.42 -Output | 0.020408 | 0.020935 | 0.02227 | 0.5 | 0.06 -Modify | 1.8299 | 1.8724 | 1.9325 | 2.9 | 5.81 -Other | | 0.1491 | | | 0.46 - -Nlocal: 2777.75 ave 2887 max 2682 min -Histogram: 1 0 0 0 2 0 0 0 0 1 -Nghost: 1152.5 ave 1189 max 1125 min -Histogram: 1 0 1 0 0 1 0 0 0 1 -Neighs: 11515.5 ave 12520 max 10831 min -Histogram: 1 1 0 0 1 0 0 0 0 1 - -Total # of neighbors = 46062 -Ave neighs/atom = 4.1456215 -Ave special neighs/atom = 10.214742 -Neighbor list builds = 408 -Dangerous builds = 0 -Total wall time: 0:00:32 diff --git a/examples/bpm/pour/log.4May2022.pour.g++.4 b/examples/bpm/pour/log.4May2022.bpm.pour.g++.4 similarity index 100% rename from examples/bpm/pour/log.4May2022.pour.g++.4 rename to examples/bpm/pour/log.4May2022.bpm.pour.g++.4 diff --git a/tools/README b/tools/README index d7bf605726..c1f49c48c6 100644 --- a/tools/README +++ b/tools/README @@ -40,6 +40,7 @@ phonon post-process output of the fix phonon command polybond Python tool for programmable polymer bonding pymol_asphere convert LAMMPS output of ellipsoids to PyMol format python Python scripts for post-processing LAMMPS output +regression-tests Python script to run regression tests using the example input replica tool to reorder LAMMPS replica trajectories according to temperature singularity Singularity container descriptions suitable for LAMMPS development smd convert Smooth Mach Dynamics triangles to VTK diff --git a/tools/regression-tests/README b/tools/regression-tests/README new file mode 100644 index 0000000000..810b96e87c --- /dev/null +++ b/tools/regression-tests/README @@ -0,0 +1,123 @@ +The script `run_tests.py` in this folder is used to perform regression tests +using in-place example scripts. + +What this single script does is to launch the selected LAMMPS binary +using a testing configuration defined in a `.yaml` file (e.g., `config.yaml`) +for the set of input scripts inside the given `examples/` subfolders, +and compare the output thermo with that in the existing log file with the same number of procs. +If there are multiple log files with the same input script (e.g., `log.melt.*.g++.1` and `log.melt.*.g++.4`), +the one with the highest number of procs is chosen. + +The output includes the number of passed and failed tests and +an `output.xml` file in the JUnit XML format for downstream reporting. +The output and error of any crashed runs are logged. + +A test with an input script is considered passed when the given LAMMPS binary produces +thermo output quantities consistent with those in the reference log file +within the specified tolerances in the test configuration `config.yaml` file. + +With the current features, users can: + + + specify which LAMMPS binary version to test (e.g., the version from a commit, or those from `lammps-testing`) + + specify the examples subfolders (thus the reference log files) seperately (e.g. from other LAMMPS versions or commits) + + specify tolerances for individual quantities for any input script to override the global values + + launch tests with `mpirun` with all supported command line features (multiple procs, multiple paritions, and suffices) + + skip certain input files if not interested, or no reference log file exists + + simplify the main LAMMPS builds, as long as a LAMMPS binary is available + +Limitations: + + - input scripts use thermo style multi (e.g., examples/peptide) do not work with the expected thermo output format + - input scripts that require partition runs (e.g. examples/neb) need a separate config file, e.g. "args: --partition 2x1" + - testing accelerator packages (GPU, INTEL, KOKKOS, OPENMP) need separate config files, "args: -sf omp -pk omp 4" + +TODO: + + + keep track of the testing progress to resume the testing from the last checkpoint + + distribute the input list across multiple processes via multiprocessing, or + split the list of input scripts into separate runs (there are 800+ input script under the top-level examples) + + be able to be invoked from run_tests in the lammps-testing infrastruture + + +The following Python packages need to be installed into an activated environment: + + python3 -m venv testing-env + source testing-env/bin/activate + pip install numpy pyyaml junit_xml + + +Example uses: + + 1) Simple use (using the provided tools/regression-tests/config.yaml and the examples/ folder at the top level) + python3 run_tests.py --lmp-bin=/path/to/lmp_binary + + 2) Use a custom testing configuration + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml + + 3) Specify a list of example folders + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml \ + --example-folders="/path/to/examples/folder1;/path/to/examples/folder2" + + The example folders can also be loaded from a text file list_subfolders1.txt: + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml \ + --list-input=list_subfolders1.txt --output-file=output1.txt --progress-file=progress1.yaml \ + --log-file=run1.log + + 4) Test a LAMMPS binary with the whole top-level /examples folder in a LAMMPS source tree + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples + + 5) Analyze (dry run) the LAMMPS binary annd whole top-level /examples folder in a LAMMPS source tree + and generate separate input lists for 8 workers: + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples \ + --dry-run --num-workers=8 + + This is used for splitting the subfolders into separate input lists and launching different instances + of run_tests.py simultaneously. + +An example of the test configuration `config.yaml` is given as below. + + --- + lmp_binary: "" + nprocs: "4" + args: "-cite none" + mpiexec: "mpirun" + mpiexec_numproc_flag: "-np" + tolerance: + PotEng: + abs: 1e-4 + rel: 1e-7 + TotEng: + abs: 1e-4 + rel: 1e-7 + Press: + abs: 1e-4 + rel: 1e-7 + Temp: + abs: 1e-4 + rel: 1e-7 + E_vdwl: + abs: 1e-3 + rel: 1e-7 + overrides: + in.rigid.tnr: + Temp: + abs: 1e-3 + rel: 1e-5 + Press: + abs: 1e-2 + rel: 1e-4 + skip: + [ in.rigid.poems3, + in.rigid.poems4, + in.rigid.poems5, + ] + + nugget: 1.0 + epsilon: 1e-16 + +An example of the list of input scripts in a text file `list_subfolders1.txt` + +/home/codes/lammps/examples/melt +/home/codes/lammps/examples/body +/home/codes/lammps/examples/PACKAGES/dielectric +/home/codes/lammps/examples/PACKAGES/tally diff --git a/tools/regression-tests/config.yaml b/tools/regression-tests/config.yaml new file mode 100644 index 0000000000..24f1ab0d67 --- /dev/null +++ b/tools/regression-tests/config.yaml @@ -0,0 +1,46 @@ +--- + lmp_binary: "" + nprocs: "4" + args: "-cite none" + mpiexec: "mpirun" + mpiexec_numproc_flag: "-np" + tolerance: + PotEng: + abs: 1e-4 + rel: 1e-7 + TotEng: + abs: 1e-4 + rel: 1e-7 + Press: + abs: 1e-4 + rel: 1e-7 + Temp: + abs: 1e-4 + rel: 1e-7 + E_vdwl: + abs: 1e-3 + rel: 1e-7 + overrides: + in.rigid.tnr: + Temp: + abs: 1e-3 + rel: 1e-5 + Press: + abs: 1e-2 + rel: 1e-4 + skip: + [ in.rigid.poems3, + in.rigid.poems4, + in.rigid.poems5, + in.peptide, + in.voronoi, + in.voronoi.2d, + in.voronoi.data, + in.*_imd*, + in.bucky-plus-cnt*, + ] + + nugget: 1.0 + epsilon: 1e-16 + + diff --git a/tools/regression-tests/config_gpu.yaml b/tools/regression-tests/config_gpu.yaml new file mode 100644 index 0000000000..2ecf6785a0 --- /dev/null +++ b/tools/regression-tests/config_gpu.yaml @@ -0,0 +1,33 @@ +--- + lmp_binary: "" + nprocs: "4" + args: "-cite none -sf gpu -pk gpu 0" + mpiexec: "mpirun" + mpiexec_numproc_flag: "-np" + tolerance: + PotEng: + abs: 1e-4 + rel: 1e-7 + TotEng: + abs: 1e-4 + rel: 1e-7 + Press: + abs: 1e-4 + rel: 1e-7 + Temp: + abs: 1e-4 + rel: 1e-7 + E_vdwl: + abs: 1e-3 + rel: 1e-7 + overrides: + in.rigid.tnr: + Temp: + abs: 1e-3 + rel: 1e-5 + Press: + abs: 1e-2 + rel: 1e-4 + + nugget: 1.0 + epsilon: 1e-16 diff --git a/tools/regression-tests/config_kokkos.yaml b/tools/regression-tests/config_kokkos.yaml new file mode 100644 index 0000000000..8c94e04071 --- /dev/null +++ b/tools/regression-tests/config_kokkos.yaml @@ -0,0 +1,33 @@ +--- + lmp_binary: "" + nprocs: "4" + args: "-cite none -k on g 1 -sf kk -pk kokkos newton on neigh half" + mpiexec: "mpirun" + mpiexec_numproc_flag: "-np" + tolerance: + PotEng: + abs: 1e-4 + rel: 1e-7 + TotEng: + abs: 1e-4 + rel: 1e-7 + Press: + abs: 1e-4 + rel: 1e-7 + Temp: + abs: 1e-4 + rel: 1e-7 + E_vdwl: + abs: 1e-3 + rel: 1e-7 + overrides: + in.rigid.tnr: + Temp: + abs: 1e-3 + rel: 1e-5 + Press: + abs: 1e-2 + rel: 1e-4 + + nugget: 1.0 + epsilon: 1e-16 diff --git a/tools/regression-tests/config_valgrind.yaml b/tools/regression-tests/config_valgrind.yaml new file mode 100644 index 0000000000..e8a2a3a956 --- /dev/null +++ b/tools/regression-tests/config_valgrind.yaml @@ -0,0 +1,35 @@ +--- + lmp_binary: "" + nprocs: "1" + args: "-cite none" + mpiexec: "mpirun valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes" + mpiexec_numproc_flag: "-np" + tolerance: + PotEng: + abs: 1e-4 + rel: 1e-7 + TotEng: + abs: 1e-4 + rel: 1e-7 + Press: + abs: 1e-4 + rel: 1e-7 + Temp: + abs: 1e-4 + rel: 1e-7 + E_vdwl: + abs: 1e-3 + rel: 1e-7 + overrides: + in.rigid.tnr: + Temp: + abs: 1e-3 + rel: 1e-5 + Press: + abs: 1e-2 + rel: 1e-4 + + nugget: 1.0 + epsilon: 1e-16 + + diff --git a/tools/regression-tests/in.lj b/tools/regression-tests/in.lj new file mode 100644 index 0000000000..464f000f76 --- /dev/null +++ b/tools/regression-tests/in.lj @@ -0,0 +1,31 @@ +# 3d Lennard-Jones melt + +variable x index 1 +variable y index 1 +variable z index 1 + +variable xx equal 20*$x +variable yy equal 20*$y +variable zz equal 20*$z + +units lj +atom_style atomic + +lattice fcc 0.8442 +region box block 0 ${xx} 0 ${yy} 0 ${zz} +create_box 1 box +create_atoms 1 box +mass 1 1.0 + +velocity all create 1.44 87287 loop geom + +pair_style lj/cut 2.5 +pair_coeff 1 1 1.0 1.0 2.5 + +neighbor 0.3 bin +neigh_modify delay 0 every 20 check no + +fix 1 all nve + #REG:ADD thermo 10 + #REG:ADD thermo_style yaml +run 100 diff --git a/tools/regression-tests/run_tests.py b/tools/regression-tests/run_tests.py new file mode 100644 index 0000000000..b2144478ec --- /dev/null +++ b/tools/regression-tests/run_tests.py @@ -0,0 +1,1116 @@ +#!/usr/bin/env python3 +''' +UPDATE: August 13, 2024: + Launching the LAMMPS binary under testing using a configuration defined in a yaml file (e.g. config.yaml). + Comparing the output thermo with that in the existing log file (with the same nprocs) + + data in the log files are extracted and converted into yaml data structure + + using the in place input scripts, no need to add REG markers to the input scripts + +With the current features, users can: + + specify which LAMMPS binary version to test (e.g., the version from a commit, or those from `lammps-testing`) + + specify the examples subfolders (thus the reference log files) seperately (e.g. from other LAMMPS versions or commits) + + specify tolerances for individual quantities for any input script to override the global values + + launch tests with `mpirun` with all supported command line features (multiple procs, multiple paritions, and suffices) + + skip certain input files (whose names match specified patterns) if not interested, or packaged not installed, or no reference log file exists + + simplify the main LAMMPS builds, as long as a LAMMPS binary is available + + keep track of the testing progress to resume the testing from the last checkpoint (skipping completed runs) + + distribute the input list across multiple processes via multiprocessing, or + split the list of input scripts into separate runs (there are 800+ input script under the top-level examples) + +Limitations: + - input scripts use thermo style multi (e.g., examples/peptide) do not work with the expected thermo output format + - input scripts that require partition runs (e.g. examples/neb) need a separate config file, e.g. args: "--partition 3x1" + - testing accelerator packages (GPU, INTEL, KOKKOS, OPENMP) need separate config files, "args: -sf omp -pk omp 4" + +TODO: + + be able to be invoked from run_tests in the lammps-testing infrastruture + +The following Python packages need to be installed into an activated environment: + + python3 -m venv testing-env + source testing-env/bin/activate + pip install numpy pyyaml junit_xml + +Example usage: + + 1) Simple use (using the provided tools/regression-tests/config.yaml and the examples/ folder at the top level) + python3 run_tests.py --lmp-bin=/path/to/lmp_binary + + 2) Use a custom testing configuration + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml + + 3) Specify a list of example folders + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml \ + --example-folders="/path/to/examples/folder1;/path/to/examples/folder2" + + The example folders can also be loaded from a text file list_subfolders1.txt: + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --config-file=/path/to/config/file/config.yaml \ + --list-input=list_subfolders1.txt --output-file=output1.txt --progress-file=progress1.yaml \ + --log-file=run1.log + + 4) Test a LAMMPS binary with the whole top-level /examples folder in a LAMMPS source tree + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples + + 5) Analyze the LAMMPS binary annd whole top-level /examples folder in a LAMMPS source tree + and generate separate input lists for 8 workers: + python3 run_tests.py --lmp-bin=/path/to/lmp_binary --examples-top-level=/path/to/lammps/examples \ + --analyze --num-workers=8 + + This is used for splitting the subfolders into separate input lists and launching different instances + of run_tests.py simultaneously. +''' + +from argparse import ArgumentParser +import datetime +import fnmatch +import logging +import os +import re +import subprocess +#from multiprocessing import Pool + +# need "pip install numpy pyyaml" +import numpy as np +import yaml + +# need "pip install junit_xml" +from junit_xml import TestSuite, TestCase + +try: + from yaml import CSafeLoader as Loader +except ImportError: + from yaml import SafeLoader as Loader + +''' + data structure to store the test result +''' +class TestResult: + def __init__(self, name, output=None, time=None, checks=0, status=None): + self.name = name + self.output = output + self.time = time + self.checks = 0 + self.status = status + +''' + Iterate over a list of input folders and scripts using the given lmp_binary and the testing configuration + + lmp_binary : full path to the LAMMPS binary + input_folder : the absolute path to the input files + input_list : list of the input scripts under the input_folder + config : the dict that contains the test configuration + + output_buf: placeholder for storing the output of a given worker + + return + results : a list of TestResult objects + stat : a dictionary that lists the number of passed, skipped, failed tests + progress_file: yaml file that stores the tested input script and status + last_progress: the dictionary that shows the status of the last tests +''' +def iterate(lmp_binary, input_folder, input_list, config, results, progress_file, last_progress=None, output_buf=None): + + num_tests = len(input_list) + num_completed = 0 + num_passed = 0 + num_skipped = 0 + num_error = 0 + num_memleak = 0 + test_id = 0 + + # using REG-commented input scripts, now turned off (False) + using_markers = False + EPSILON = np.float64(config['epsilon']) + nugget = float(config['nugget']) + use_valgrind = False + if 'valgrind' in config['mpiexec']: + use_valgrind = True + + # iterate over the input scripts + for input in input_list: + + # check if the progress file exists to append or create a new one + if os.path.isfile(progress_file) == True: + progress = open(progress_file, "a") + else: + progress = open(progress_file, "w") + + # skip the input file if listed in the config file or matched with a pattern + if 'skip' in config: + if input in config['skip']: + msg = " + " + input + f" ({test_id+1}/{num_tests}): skipped as specified in {configFileName}" + print(msg) + logger.info(msg) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"skipped\" }}\n") + progress.close() + num_skipped = num_skipped + 1 + test_id = test_id + 1 + continue + + matched_pattern = False + for skipped_files in config['skip']: + if '*' in skipped_files: + if fnmatch.fnmatch(input, skipped_files): + matched_pattern = True + break + + if matched_pattern == True: + msg = " + " + input + f" ({test_id+1}/{num_tests}): skipped as specified in {configFileName}" + print(msg) + logger.info(msg) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"skipped\" }}\n") + progress.close() + num_skipped = num_skipped + 1 + test_id = test_id + 1 + continue + + # also skip if the test already completed as marked in the progress file + if input in last_progress: + status = last_progress[input]['status'] + if 'completed' in status or 'numerical checks skipped' in status: + msg = " + " + input + f" ({test_id+1}/{num_tests}): marked as completed or numerical checks skipped (see {progress_file})" + logger.info(msg) + print(msg) + # No need to write to progress again that the run is completed + progress.close() + num_skipped = num_skipped + 1 + test_id = test_id + 1 + continue + + if 'packaged not installed' in status: + msg = " + " + input + f" ({test_id+1}/{num_tests}): due to package not installed (see {progress_file})" + logger.info(msg) + print(msg) + # No need to write to progress again that the run gets error due to missing packages + progress.close() + num_skipped = num_skipped + 1 + test_id = test_id + 1 + continue + + # if annotating input scripts with REG markers is True + if using_markers == True: + input_test = 'test.' + input + if os.path.isfile(input) == True: + if has_markers(input): + process_markers(input, input_test) + + else: + print(f"WARNING: {input} does not have REG markers") + input_markers = input + '.markers' + # if the input file with the REG markers does not exist + # attempt to plug in the REG markers before each run command + if os.path.isfile(input_markers) == False: + cmd_str = "cp " + input + " " + input_markers + os.system(cmd_str) + generate_markers(input, input_markers) + process_markers(input_markers, input_test) + + else: + # else the same file name for testing + input_test = input + + str_t = " + " + input_test + f" ({test_id+1}/{num_tests})" + logger.info(str_t) + print(str_t) + + # check if a reference log file exists in the current folder: log.DDMMMYY.basename.g++.[nprocs] + # assuming that input file names start with "in." (except in.disp, in.disp2 and in.dos in phonon/) + basename = input_test[3:] + ref_logfile_exist = False + + # if there are multiple log files for different number of procs, pick the maximum number + cmd_str = "ls log.*" + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + logfile_list = p.stdout.split('\n') + logfile_list.remove('') + + max_np = 1 + for file in logfile_list: + # looks for pattern log.{date}.{basename}.{compiler}.{nprocs}: log.[date].min.box.[compiler]].* vs log.[date].min.[compiler].* + # get the date from the log files + date = file.split('.',2)[1] + compiler = file.rsplit('.',2)[1] + pattern = f'log.{date}.{basename}.{compiler}.*' + if fnmatch.fnmatch(file, pattern): + p = file.rsplit('.', 1) + if p[1].isnumeric(): + if use_valgrind == True: + if int(p[1]) == 1: + max_np = int(p[1]) + ref_logfile_exist = True + thermo_ref_file = file + break + else: + if max_np <= int(p[1]): + max_np = int(p[1]) + ref_logfile_exist = True + thermo_ref_file = file + + # if there is no ref log file and not running with valgrind + if ref_logfile_exist == False and use_valgrind == False: + max_np = 4 + + # if the maximum number of procs is different from the value in the configuration file + # then override the setting for this input script + saved_nprocs = config['nprocs'] + if max_np != int(config['nprocs']): + config['nprocs'] = str(max_np) + + # store the value of nprocs + nprocs = int(config['nprocs']) + + # if valgrind is used for mem check, the run command will be + # mpirun -np 1 valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes /path/to/lmp_binary -in in.txt + # so both mpiexec_numproc_flag and nprocs are empty + if use_valgrind == True: + config['nprocs'] = "" + config['mpiexec_numproc_flag'] = "" + nprocs = 1 + + result = TestResult(name=input, output="", time="", status="passed") + + # run the LAMMPS binary with the input script + cmd_str, output, error, returncode = execute(lmp_binary, config, input_test) + + # restore the nprocs value in the configuration + config['nprocs'] = saved_nprocs + + # check if the output contains ERROR + # there might not be a log.lammps generated at this point, or only log.lammps contains only the date line + if "ERROR" in output: + error_line = "" + for line in output.split('\n'): + if "ERROR" in line: + error_line = line + break + logger.info(f" The run terminated with {input_test} gives the following output:") + logger.info(f" {error_line}") + if "Unrecognized" in output: + result.status = f"error, unrecognized command, package not installed, {error_line}" + elif "Unknown" in output: + result.status = f"error, unknown command, package not installed, {error_line}" + else: + result.status = f"error, {error_line}." + + logger.info(f" Output:") + logger.info(f" {output}") + logger.info(f" Failed with {input_test}.\n") + num_error = num_error + 1 + + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"{result.status}\" }}\n") + progress.close() + + test_id = test_id + 1 + continue + + # if there is no ERROR in the output, but there is no Total wall time printed out + if "Total wall time" not in output: + logger.info(f" ERROR: no Total wall time in the output.\n") + logger.info(f"\n{input_test}:") + logger.info(f"\n Output:\n{output}") + logger.info(f"\n Error:\n{error}") + progress.write(f"{input}: {{ folder: {input_folder}, status: \"error, no Total wall time in the output.\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + + # if there is no Step or no Loop printed out + if "Step" not in output or "Loop" not in output: + logger.info(f" ERROR: no Step nor Loop in the output.\n") + logger.info(f"\n{input_test}:") + logger.info(f"\n Output:\n{output}") + logger.info(f"\n Error:\n{error}") + progress.write(f"{input}: {{ folder: {input_folder}, status: \"error, no Step nor Loop in the output.\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + + # check if a log.lammps file exists in the current folder + if os.path.isfile("log.lammps") == False: + logger.info(f" ERROR: No log.lammps generated with {input_test} with return code {returncode}.\n") + logger.info(f" Output:") + logger.info(f" {output}") + logger.info(f" Error:\n{error}") + progress.write(f"{input}: {{ folder: {input_folder}, status: \"error, no log.lammps\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + else: + # save a copy of the log file for further inspection + cmd_str = f"cp log.lammps log.{basename}.{nprocs}" + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + + # parse thermo output in log.lammps from the run + thermo = extract_data_to_yaml("log.lammps") + num_runs = len(thermo) + + # the run completed normally but log.lammps may not be friendly for parsing into YAML format + if num_runs == 0: + logger.info(f" The run terminated with {input_test} gives the following output:") + logger.info(f" {output}") + + msg = "completed" + if 'valgrind' in config['mpiexec']: + if "All heap blocks were free" in error: + msg += ", no memory leak" + else: + msg += ", memory leaks detected" + num_memleak = num_memleak + 1 + + result.status = msg + ", error parsing log.lammps into YAML" + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"{result.status}\" }}\n") + progress.close() + + num_completed = num_completed + 1 + test_id = test_id + 1 + continue + + # At this point, the run completed without trivial errors, proceed with numerical checks + # check if there is a reference log file for this input + if ref_logfile_exist: + # parse the thermo output in reference log file + thermo_ref = extract_data_to_yaml(thermo_ref_file) + if thermo_ref: + num_runs_ref = len(thermo_ref) + else: + # dictionary is empty + logger.info(f" ERROR: Error parsing the reference log file {thermo_ref_file}.") + result.status = "skipped numerical checks due to parsing the reference log file" + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"completed, numerical checks skipped, unsupported log file format\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + else: + msg = f" Cannot find the reference log file for {input_test} with the expected format log.[date].{basename}.*.[nprocs]" + logger.info(msg) + print(msg) + # attempt to read in the thermo yaml output from the working directory (the following section will be deprecated) + thermo_ref_file = 'thermo.' + input + '.yaml' + file_exist = os.path.isfile(thermo_ref_file) + if file_exist == True: + thermo_ref = extract_thermo(thermo_ref_file) + num_runs_ref = len(thermo_ref) + else: + # mostly will come to here if the reference log file does not exist + logger.info(f" {thermo_ref_file} also does not exist in the working directory.") + result.status = "skipped due to missing the reference log file" + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"completed, numerical checks skipped, missing the reference log file\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + + logger.info(f" Comparing thermo output from log.lammps against the reference log file {thermo_ref_file}") + + # check if the number of runs matches with that in the reference log file + # maybe due to some changes to the input where the ref log file is not updated yet + if num_runs != num_runs_ref: + logger.info(f" ERROR: Number of runs in log.lammps ({num_runs}) is different from that in the reference log ({num_runs_ref})." + " Check README in the folder, possibly due to using mpirun with partitions or parsing the wrong reference log file.") + result.status = "error, incomplete runs" + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"{result.status}\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + + # check if the number of fields match with that in the reference log file in the first run + # due to some changes to the input where the ref log file is not updated yet + # for early exit + num_fields = len(thermo[0]['keywords']) + num_fields_ref = len(thermo_ref[0]['keywords']) + if num_fields != num_fields_ref: + logger.info(f" ERROR: Number of thermo colums in log.lammps ({num_fields}) is different from that in the reference log ({num_fields_ref}) in the first run.") + logger.info(f" Check both log files for more details.") + result.status = "error, mismatched columns in the log files" + results.append(result) + progress.write(f"{input}: {{ folder: {input_folder}, status: \"{result.status}\" }}\n") + progress.close() + num_error = num_error + 1 + test_id = test_id + 1 + continue + + # comparing output vs reference values + width = 20 + if verbose == True: + print("Quantities".ljust(width) + "Output".center(width) + "Reference".center(width) + + "Abs Diff Check".center(width) + "Rel Diff Check".center(width)) + + # check if overrides for this input scipt is specified + overrides = {} + if 'overrides' in config: + if input_test in config['overrides']: + overrides = config['overrides'][input_test] + + # iterate through num_runs + + num_abs_failed = 0 + num_rel_failed = 0 + failed_abs_output = [] + failed_rel_output = [] + num_checks = 0 + mismatched_columns = False + + for irun in range(num_runs): + num_fields = len(thermo[irun]['keywords']) + num_fields_ref = len(thermo_ref[irun]['keywords']) + if num_fields != num_fields_ref: + logger.info(f" ERROR: Number of thermo columns in log.lammps ({num_fields})") + logger.info(f" is different from that in the reference log ({num_fields_ref}) in run {irun}.") + mismatched_columns = True + continue + + # get the total number of the thermo output lines + nthermo_steps = len(thermo[irun]['data']) + + # get the output at the last timestep + thermo_step = nthermo_steps - 1 + + # iterate over the fields + for i in range(num_fields): + quantity = thermo[irun]['keywords'][i] + + val = thermo[irun]['data'][thermo_step][i] + ref = thermo_ref[irun]['data'][thermo_step][i] + abs_diff = abs(float(val) - float(ref)) + + if abs(float(ref)) > EPSILON: + rel_diff = abs(float(val) - float(ref))/abs(float(ref)) + else: + rel_diff = abs(float(val) - float(ref))/abs(float(ref)+nugget) + + abs_diff_check = "PASSED" + rel_diff_check = "PASSED" + + if quantity in config['tolerance'] or quantity in overrides: + + if quantity in config['tolerance']: + abs_tol = float(config['tolerance'][quantity]['abs']) + rel_tol = float(config['tolerance'][quantity]['rel']) + + # overrides the global tolerance values if specified + if quantity in overrides: + abs_tol = float(overrides[quantity]['abs']) + rel_tol = float(overrides[quantity]['rel']) + + num_checks = num_checks + 2 + if abs_diff > abs_tol: + abs_diff_check = "FAILED" + reason = f"Run {irun}: {quantity}: actual ({abs_diff:0.2e}) > expected ({abs_tol:0.2e})" + failed_abs_output.append(f"{reason}") + num_abs_failed = num_abs_failed + 1 + if rel_diff > rel_tol: + rel_diff_check = "FAILED" + reason = f"Run {irun}: {quantity}: actual ({rel_diff:0.2e}) > expected ({rel_tol:0.2e})" + failed_rel_output.append(f"{reason}") + num_rel_failed = num_rel_failed + 1 + else: + # N/A means that tolerances are not defined in the config file + abs_diff_check = "N/A" + rel_diff_check = "N/A" + + if verbose == True and abs_diff_check != "N/A" and rel_diff_check != "N/A": + print(f"{thermo[irun]['keywords'][i].ljust(width)} {str(val).rjust(20)} {str(ref).rjust(20)} " + "{abs_diff_check.rjust(20)} {rel_diff_check.rjust(20)}") + + # after all runs completed, or are interrupted in one of the runs (mismatched_columns = True) + if mismatched_columns == True: + msg = f" mismatched log files after the first run. Check both log files for more details." + print(msg) + logger.info(msg) + result.status = "failed" + + if num_abs_failed > 0: + msg = f" {num_abs_failed} abs diff thermo checks failed." + print(msg) + logger.info(msg) + result.status = "failed" + if verbose == True: + for i in failed_abs_output: + print(f"- {i}") + if num_rel_failed > 0: + msg = f" {num_rel_failed} rel diff thermo checks failed." + print(msg) + logger.info(msg) + result.status = "failed" + if verbose == True: + for i in failed_rel_output: + print(f"- {i}") + if num_abs_failed == 0 and num_rel_failed == 0: + msg = f" all {num_checks} thermo checks passed." + print(msg) + logger.info(msg) + result.status = "passed" + num_passed = num_passed + 1 + + results.append(result) + + # check if memleak detects from valgrind run (need to replace "mpirun" -> valgrind --leak-check=yes mpirun") + msg = "completed" + if use_valgrind == True: + if "All heap blocks were freed" in error: + msg += ", no memory leak" + else: + msg += ", memory leaks detected" + num_memleak = num_memleak + 1 + + progress.write(f"{input}: {{ folder: {input_folder}, status: \"{msg}\" }}\n") + progress.close() + + # count the number of completed runs + num_completed = num_completed + 1 + test_id = test_id + 1 + + stat = { 'num_completed': num_completed, + 'num_passed': num_passed, + 'num_skipped': num_skipped, + 'num_error': num_error, + 'num_memleak': num_memleak, + } + return stat + +# HELPER FUNCTIONS +''' + get the thermo output from a log file with thermo style yaml + + yamlFileName: input YAML file with thermo structured + as described in https://docs.lammps.org/Howto_structured_data.html + return: thermo, which is a list containing a dictionary for each run + where the tag "keywords" maps to the list of thermo header strings + and the tag data has a list of lists where the outer list represents the lines + of output and the inner list the values of the columns matching the header keywords for that step. +''' +def extract_thermo(yamlFileName): + docs = "" + with open(yamlFileName) as f: + for line in f: + m = re.search(r"^(keywords:.*$|data:$|---$|\.\.\.$| - \[.*\]$)", line) + if m: docs += m.group(0) + '\n' + thermo = list(yaml.load_all(docs, Loader=Loader)) + return thermo + + +''' + Convert an existing log file into a thermo yaml style log + inputFileName = a provided log file in an examples folder (e.g. examples/melt/log.8Apr21.melt.g++.4) + return a YAML data structure as if loaded from a thermo yaml file +''' +def extract_data_to_yaml(inputFileName): + with open(inputFileName, 'r') as file: + data = file.read() + lines = data.splitlines() + reading = False + data = [] + docs = "" + num_thermo_cols = 0 + for line in lines: + if "Step" in line and line[0] != '#': + line.strip() + keywords = line.split() + num_thermo_cols = len(keywords) + reading = True + docs += "---\n" + docs += str("keywords: [") + for word in enumerate(keywords): + docs += "'" + word[1] + "', " + docs += "]\n" + docs += "data:\n" + if "Loop" in line: + reading = False + docs += "...\n" + + if reading == True and "Step" not in line: + if "WARNING" in line: + continue + data = line.split() + if len(data) != num_thermo_cols: + continue + docs += " - [" + for field in enumerate(data): + docs += field[1] + ", " + docs += "]\n" + + # load the docs into a YAML data struture + #print(docs) + thermo = {} + try: + yaml_struct = yaml.load_all(docs, Loader=Loader) + thermo = list(yaml_struct) + except yaml.YAMLError as exc: + if hasattr(exc, 'problem_mark'): + mark = exc.problem_mark + msg = f" Error parsing {inputFileName} at line {mark.line}, column {mark.column+1}." + print(msg) + logger.info(msg) + logger.info(docs) + return thermo + else: + msg = f" Something went wrong while parsing {inputFileName}." + print(msg) + logger.info(msg) + logger.info(docs) + return thermo + return thermo + +''' + return a tuple of the list of installed packages, OS, GitInfo and compile_flags +''' +def get_lammps_build_configuration(lmp_binary): + cmd_str = lmp_binary + " -h" + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + output = p.stdout.split('\n') + packages = "" + reading = False + operating_system = "" + GitInfo = "" + row = 0 + for line in output: + if line != "": + if line == "Installed packages:": + reading = True + n = row + if "List of individual style options" in line: + reading = False + if reading == True and row > n: + packages += line.strip() + " " + + if "OS:" in line: + operating_system = line + if "Git info" in line: + GitInfo = line + + row += 1 + + packages = packages.strip() + + row = 0 + compile_flags = "" + for line in output: + if line != "": + if "-DLAMMPS" in line: + compile_flags += " " + line.strip() + + row += 1 + + return packages.split(" "), operating_system, GitInfo, compile_flags + +''' + launch LAMMPS using the configuration defined in the dictionary config with an input file + TODO: + - generate new reference values if needed + - wrap subprocess with try/catch to handle exceptions +''' +def execute(lmp_binary, config, input_file_name, generate_ref_yaml=False): + cmd_str = config['mpiexec'] + " " + config['mpiexec_numproc_flag'] + " " + config['nprocs'] + " " + cmd_str += lmp_binary + " -in " + input_file_name + " " + config['args'] + logger.info(f" Executing: {cmd_str}") + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + + return cmd_str, p.stdout, p.stderr, p.returncode + +''' + split a list into a list of N sublists + + NOTE: + To map a function to individual workers with multiprocessing.Pool: + + def func(input1, input2, output_buf): + # do smth + return result + + # args is a list of num_workers tuples, each tuple contains the arguments passed to the function executed by a worker + args = [] + for i in range(num_workers): + args.append((input1, input2, output_buf)) + + with Pool(num_workers) as pool: + results = pool.starmap(func, args) +''' +def divide_into_N(original_list, N): + size = np.ceil(len(original_list) / N) + b = [] + for i in range(0, N): + start = int(i * size) + end = int(start + size) + l = original_list[start:end] + b.append(l) + return b + +''' + process the #REG markers in an input script, add/replace with what follows each marker + + inputFileName: LAMMPS input file with comments #REG:ADD and #REG:SUB as markers + outputFileName: modified input file ready for testing +''' +def process_markers(inputFileName, outputFileName): + # read in the script + with open(inputFileName, 'r') as file: + data = file.read() + + # replace #REG:ADD with empty string (i.e. adding the text at the end of the line) + data = data.replace("#REG:ADD", "") + + # replace the line contaning #REG:SUB with a line with the text that follows this marker + data = data.splitlines() + separator="#REG:SUB" + out = [] + for line in data: + s = line.split(separator) + if len(s) < 2: + out.append(line) + else: + out.append(s[1]) + + # write data to the new script + with open(outputFileName, 'w') as file: + for line in out: + file.write(line + "\n") + + +''' + attempt to insert the #REG markers before each run command + #REG:ADD thermo 10 + #REG:ADD thermo_style yaml + + inputFileName: provided LAMMPS input file + outputFileName: modified input file ready for testing +''' +def generate_markers(inputFileName, outputFileName): + # read in the script + with open(inputFileName, 'r') as file: + data = file.read() + + lines = data.splitlines() + out = [] + for line in lines: + s = line.split() + if len(s) > 0: + if s[0] == "run": + out.append(" #REG:ADD thermo 10") + out.append(" #REG:ADD thermo_style yaml") + out.append(line) + + # write data to the new script + with open(outputFileName, 'w') as file: + for line in out: + file.write(line + "\n") + +''' + check if any input script has any #REG markers +''' +def has_markers(inputFileName): + with open(inputFileName) as f: + if '#REG' in f.read(): + return True + return False + + +''' + Main entry +''' +if __name__ == "__main__": + + # default values + lmp_binary = "" + configFileName = "config.yaml" + example_subfolders = [] + example_toplevel = "" + genref = False + verbose = False + output_file = "output.xml" + progress_file = "progress.yaml" + log_file = "run.log" + list_input = "" + analyze = False + + # distribute the total number of input scripts over the workers + num_workers = 1 + + # parse the arguments + parser = ArgumentParser() + parser.add_argument("--lmp-bin", dest="lmp_binary", default="", help="LAMMPS binary") + parser.add_argument("--config-file", dest="config_file", default=configFileName, + help="Configuration YAML file") + parser.add_argument("--examples-top-level", dest="example_toplevel", default="", help="Examples top-level") + parser.add_argument("--example-folders", dest="example_folders", default="", help="Example subfolders") + parser.add_argument("--list-input", dest="list_input", default="", help="File that lists the subfolders") + parser.add_argument("--num-workers", dest="num_workers", default=1, help="Number of workers") + parser.add_argument("--gen-ref",dest="genref", action='store_true', default=False, + help="Generating reference data") + parser.add_argument("--verbose",dest="verbose", action='store_true', default=False, + help="Verbose output") + parser.add_argument("--resume",dest="resume", action='store_true', default=False, + help="Resume the test run") + parser.add_argument("--output-file",dest="output", default=output_file, help="Output file") + parser.add_argument("--log-file",dest="logfile", default=log_file, help="Log file") + parser.add_argument("--progress-file",dest="progress_file", default=progress_file, help="Progress file") + parser.add_argument("--analyze",dest="analyze", action='store_true', default=False, + help="Analyze the testing folders and report statistics, not running the tests") + + args = parser.parse_args() + + lmp_binary = os.path.abspath(args.lmp_binary) + configFileName = args.config_file + output_file = args.output + if int(args.num_workers) > 0: + num_workers = int(args.num_workers) + list_input = args.list_input + + # example_toplevel is where all the examples subfolders reside + if args.example_toplevel != "": + example_toplevel = args.example_toplevel + if args.example_folders != "": + example_subfolders = args.example_folders.split(';') + + genref = args.genref + verbose = args.verbose + log_file = args.logfile + analyze = args.analyze + resume = args.resume + progress_file = args.progress_file + + # logging + logger = logging.getLogger(__name__) + logging.basicConfig(filename=log_file, level=logging.INFO, filemode="w") + + # read in the configuration of the tests + with open(configFileName, 'r') as f: + config = yaml.load(f, Loader=Loader) + absolute_path = os.path.abspath(configFileName) + print(f"\nRegression tests with the settings defined in the configuration file:\n {absolute_path}") + f.close() + + # check if lmp_binary is specified in the config yaml + if lmp_binary == "": + if config['lmp_binary'] == "": + print("Needs a valid LAMMPS binary") + quit() + else: + lmp_binary = os.path.abspath(config['lmp_binary']) + + # print out the binary info + packages, operating_system, GitInfo, compile_flags = get_lammps_build_configuration(lmp_binary) + print("\nLAMMPS build info:") + print(f" - {operating_system}") + print(f" - {GitInfo}") + print(f" - Active compile flags: {compile_flags}") + print(f" - List of {len(packages)} installed packages:") + all_pkgs = "" + for p in packages: + all_pkgs += p + " " + print(all_pkgs) + + if len(example_subfolders) > 0: + print("\nExample folders to test:") + print(*example_subfolders, sep='\n') + if example_toplevel != "": + print("\nTop-level example folder:") + print(f" {example_toplevel}") + + # Using in place input scripts + inplace_input = True + test_cases = [] + + # if the example folders are not specified from the command-line argument --example-folders + # then use the path from --example-top-folder, or from the input-list read from a text file + if len(example_subfolders) == 0: + + # need top level specified + if len(example_toplevel) != 0: + # getting the list of all the input files because there are subfolders (e.g. PACKAGES) under the top level + cmd_str = f"find {example_toplevel} -name \"in.*\" " + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + input_list = p.stdout.split('\n') + input_list.remove("") + msg = f"\nThere are {len(input_list)} input scripts in total under the {example_toplevel} folder." + print(msg) + logger.info(msg) + + # get the input file list + # TODO: generate a list of tuples, each tuple contains a folder list for a worker, + # then use multiprocessing.Pool starmap() + folder_list = [] + for input in input_list: + folder = input.rsplit('/', 1)[0] + # unique folders in the list + if folder not in folder_list: + folder_list.append(folder) + + # divide the list of folders into num_workers chunks + sublists = divide_into_N(folder_list, num_workers) + + # write each chunk to a file + idx = 0 + for list_input in sublists: + filename = f"input-list-{idx}.txt" + with open(filename, "w") as f: + for folder in list_input: + # count the number of input scripts in each folder + cmd_str = f"ls {folder}/in.* | wc -l" + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + num_input = p.stdout.split('\n')[0] + f.write(folder + ' ' + num_input + '\n') + f.close() + idx = idx + 1 + + # working on all the folders for now + example_subfolders = folder_list + + # if a list of subfolders are provided from a text file (list_input from the command-line argument) + elif len(list_input) != 0: + num_inputscripts = 0 + with open(list_input, "r") as f: + all_subfolders = f.read().splitlines() + f.close() + for line in all_subfolders: + if len(line) > 0: + if line[0] == '#': + continue + folder = line.split()[0] + example_subfolders.append(folder) + num_inputscripts += int(line.split()[1]) + msg = f"\nThere are {len(example_subfolders)} folders with {num_inputscripts} input scripts in total listed in {list_input}." + print(msg) + logger.info(msg) + else: + inplace_input = False + + # if analyze the example folders (and split into separate lists for top-level examples), not running any test + if analyze == True: + quit() + + all_results = [] + + # save current working dir + p = subprocess.run("pwd", shell=True, text=True, capture_output=True) + pwd = p.stdout.split('\n')[0] + pwd = os.path.abspath(pwd) + print("\nWorking directory: " + pwd) + + progress_file_abs = pwd + "/" + progress_file + last_progress = {} + if resume == False: + progress = open(progress_file_abs, "w") + progress.close() + else: + try: + progress = open(progress_file_abs, "r") + last_progress = yaml.load(progress, Loader=Loader) + progress.close() + except Exception: + print(f" Cannot open progress file {progress_file_abs} to resume, rerun all the tests") + + # initialize all the counters + total_tests = 0 + completed_tests = 0 + passed_tests = 0 + skipped_tests = 0 + error_tests = 0 + memleak_tests = 0 + + # default setting is to use inplace_input + if inplace_input == True: + + # change dir to a folder under examples/ + # TODO: loop through the subfolders under examples/, depending on the installed packages + + ''' + args = [] + for i in range(num_workers): + args.append((input1, input2, output)) + + with Pool(num_workers) as pool: + results = pool.starmap(func, args) + ''' + + for directory in example_subfolders: + + # change to the directory where the input script and data files are located + print("-"*80) + print("Entering " + directory) + logger.info("Entering " + directory) + os.chdir(directory) + + cmd_str = "ls in.*" + p = subprocess.run(cmd_str, shell=True, text=True, capture_output=True) + input_list = p.stdout.split('\n') + input_list.remove('') + + print(f"{len(input_list)} input script(s): {input_list}") + total_tests += len(input_list) + + # iterate through the input scripts + results = [] + stat = iterate(lmp_binary, directory, input_list, config, results, progress_file_abs, last_progress) + + completed_tests += stat['num_completed'] + skipped_tests += stat['num_skipped'] + passed_tests += stat['num_passed'] + error_tests += stat['num_error'] + memleak_tests += stat['num_memleak'] + + # append the results to the all_results list + all_results.extend(results) + + # get back to the working dir + os.chdir(pwd) + + else: + # or using the input scripts in the working directory -- for debugging purposes + input_list=['in.lj'] + total_tests = len(input_list) + results = [] + stat = iterate(lmp_binary, pwd, input_list, config, results, progress_file_abs) + + completed_tests = stat['num_completed'] + skipped_tests = stat['num_skipped'] + passed_tests = stat['num_passed'] + error_tests = stat['num_error'] + memleak_tests = stat['num_memleak'] + + all_results.extend(results) + + # print out summary + msg = "\nSummary:\n" + msg += f" Total number of input scripts: {total_tests}\n" + msg += f" - Skipped : {skipped_tests}\n" + msg += f" - Failed : {error_tests}\n" + msg += f" - Completed: {completed_tests}\n" + if memleak_tests < completed_tests and 'valgrind' in config['mpiexec']: + msg += f" - memory leak detected : {memleak_tests}\n" + if passed_tests <= completed_tests: + msg += f" - numerical tests passed: {passed_tests}\n" + msg += "\nOutput:\n" + msg += f" - Running log with screen output: {log_file}\n" + msg += f" - Progress with the input list : {progress_file}\n" + msg += f" - Regression test results : {output_file}\n" + + print(msg) + + # optional: need to check if junit_xml packaged is already installed in the env + # generate a JUnit XML file + with open(output_file, 'w') as f: + test_cases = [] + for result in all_results: + #print(f"{result.name}: {result.status}") + case = TestCase(name=result.name, classname=result.name) + if result.status == "failed": + case.add_failure_info(message="Actual values did not match expected ones.") + if result.status == "skipped": + case.add_skipped_info(message="Test was skipped.") + if result.status == "error": + case.add_skipped_info(message="Test run had errors.") + test_cases.append(case) + + current_timestamp = datetime.datetime.now() + ts = TestSuite(f"{configFileName}", test_cases, timestamp=current_timestamp) + TestSuite.to_file(f, [ts], prettyprint=True) diff --git a/tools/regression-tests/thermo.in.lj.yaml b/tools/regression-tests/thermo.in.lj.yaml new file mode 100644 index 0000000000..d8f93cd377 --- /dev/null +++ b/tools/regression-tests/thermo.in.lj.yaml @@ -0,0 +1,15 @@ +--- +keywords: ['Step', 'Temp', 'KinEng', 'PotEng', 'E_bond', 'E_angle', 'E_dihed', 'E_impro', 'E_vdwl', 'E_coul', 'E_long', 'Press', ] +data: + - [0, 1.44000000000001, 2.15993250000001, -6.77336805323422, 0, 0, 0, 0, -6.77336805323422, 0, 0, -5.01970725908556, ] + - [10, 1.12539487029313, 1.68803955255514, -6.30005271976029, 0, 0, 0, 0, -6.30005271976029, 0, 0, -2.55968522600129, ] + - [20, 0.625793798302192, 0.938661363368992, -5.55655653922756, 0, 0, 0, 0, -5.55655653922756, 0, 0, 0.973517658007722, ] + - [30, 0.745927295413064, 1.11885597777762, -5.73951278150759, 0, 0, 0, 0, -5.73951278150759, 0, 0, 0.339284096694852, ] + - [40, 0.731026217827733, 1.09650505988764, -5.71764564663628, 0, 0, 0, 0, -5.71764564663628, 0, 0, 0.388973418756238, ] + - [50, 0.740091517740786, 1.11010258482128, -5.73150426762886, 0, 0, 0, 0, -5.73150426762886, 0, 0, 0.335273324523691, ] + - [60, 0.750500641591031, 1.12571578266897, -5.74713299283555, 0, 0, 0, 0, -5.74713299283555, 0, 0, 0.26343139026926, ] + - [70, 0.755436366857812, 1.13311913920702, -5.75480059117447, 0, 0, 0, 0, -5.75480059117447, 0, 0, 0.224276619217515, ] + - [80, 0.759974280364828, 1.13992579675285, -5.76187162670983, 0, 0, 0, 0, -5.76187162670983, 0, 0, 0.191626237124102, ] + - [90, 0.760464250735042, 1.14066072934081, -5.76280209529731, 0, 0, 0, 0, -5.76280209529731, 0, 0, 0.189478083345243, ] + - [100, 0.757453103239936, 1.13614414924569, -5.75850548601596, 0, 0, 0, 0, -5.75850548601596, 0, 0, 0.207261053624723, ] +...