From ae29b299af40ad765ac5b1d58d0cd84601f40a29 Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 15 May 2017 21:18:41 -0400 Subject: [PATCH 1/5] Start basic unit test runner script --- applications/test/test.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 applications/test/test.py diff --git a/applications/test/test.py b/applications/test/test.py new file mode 100755 index 0000000..d9b79a8 --- /dev/null +++ b/applications/test/test.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +"""Automatically collect and run unit tests.""" + +from __future__ import division, print_function +import subprocess +import argparse +import os +import glob +from subprocess import STDOUT + +# Run from this directory +this_dir = os.path.abspath(os.path.dirname(__file__)) +os.chdir(this_dir) + + +def collect_test_dirs(): + """Loop through test application directories looking for ``Make`` + subdirectories. + """ + return sorted([d for d in os.listdir(".") if os.path.isdir(d) \ + and "Make" in os.listdir(d)]) + + +def get_test_exe_name(test_dir): + """Get the name of the test executable.""" + with open("Make/files") as f: + for line in f.readlines(): + if line.strip().startswith("EXE"): + return line.strip().split("/")[-1] + + +def test_single(test_dir, autopar=True): + """Run one unit test. + + If ``autopar`` is ``True`` and test executable has ``parallel` in its name, + it will be run in parallel. + """ + os.chdir(test_dir) + run_out = "" + compile_out = "" + try: + compile_out = subprocess.check_output("wmake", stderr=STDOUT).decode() + test_compiled = True + except subprocess.CalledProcessError: + test_compiled = False + if test_compiled: + test_exe = get_test_exe_name(test_dir) + if autopar and "parallel" in test_exe.lower(): + test_exe = "mpirun -np 2 " + test_exe + try: + run_out = subprocess.check_output(test_exe, stderr=STDOUT, + shell=True).decode() + test_ran = True + except subprocess.CalledProcessError: + test_ran = False + else: + test_ran = False + os.chdir(this_dir) + return test_compiled, test_ran, compile_out, run_out + + +def clean(test_dirs="all"): + """Go through all directories and clean executables.""" + if test_dirs == "all": + test_dirs = collect_test_dirs() + for d in test_dirs: + subprocess.call(["wclean", d]) + + +def test_multiple(test_dirs="all", verbose=False): + """Run multiple tests.""" + if test_dirs == "all": + test_dirs = collect_test_dirs() + print("Collected {} tests\n".format(len(test_dirs))) + status_str = "" + errored = [] + passed = [] + failed = [] + for test_dir in test_dirs: + if verbose: + print("Testing", test_dir) + c, p, cout, rout = test_single(test_dir) + if not c: + if verbose: + print("ERROR") + errored.append(test_dir) + status_str += "E" + elif not p: + if verbose: + print("FAIL") + failed.append(test_dir) + status_str += "F" + else: + if verbose: + print("PASS") + passed.append(test_dir) + status_str += "." + print(status_str, end="") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run OpenFOAM unit tests") + parser.add_argument("--tests", "-t", help="Which tests to run or clean", + default="all", choices=collect_test_dirs() + ["all"], + metavar="tests") + parser.add_argument("--clean", "-c", action="store_true", default=False, + help="Clean all unit test executables") + parser.add_argument("--verbose", "-v", action="store_true", default=False, + help="Print verbose output") + args = parser.parse_args() + if args.clean: + clean_all(args.tests) + else: + test_multiple(args.tests) -- 2.7.4 From d5c559505c3414f6708a72874a273a974dde82f5 Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 15 May 2017 21:53:28 -0400 Subject: [PATCH 2/5] Improve summary and print output --- applications/test/test.py | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/applications/test/test.py b/applications/test/test.py index d9b79a8..f3d0ebb 100755 --- a/applications/test/test.py +++ b/applications/test/test.py @@ -41,7 +41,8 @@ def test_single(test_dir, autopar=True): try: compile_out = subprocess.check_output("wmake", stderr=STDOUT).decode() test_compiled = True - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: + compile_out += e.output.decode() test_compiled = False if test_compiled: test_exe = get_test_exe_name(test_dir) @@ -51,7 +52,8 @@ def test_single(test_dir, autopar=True): run_out = subprocess.check_output(test_exe, stderr=STDOUT, shell=True).decode() test_ran = True - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: + run_out += e.output.decode() test_ran = False else: test_ran = False @@ -71,38 +73,58 @@ def test_multiple(test_dirs="all", verbose=False): """Run multiple tests.""" if test_dirs == "all": test_dirs = collect_test_dirs() - print("Collected {} tests\n".format(len(test_dirs))) + print("\nRunning OpenFOAM unit tests") + print("\nCollected {} tests\n".format(len(test_dirs))) status_str = "" errored = [] passed = [] failed = [] + err_out = "" + fail_out = "" + std_out = "" for test_dir in test_dirs: if verbose: - print("Testing", test_dir) + print("Testing {:24s}".format(test_dir + "..."), end="", sep=" ") c, p, cout, rout = test_single(test_dir) if not c: if verbose: print("ERROR") errored.append(test_dir) status_str += "E" + err_out += cout elif not p: if verbose: print("FAIL") failed.append(test_dir) status_str += "F" - else: + fail_out += rout + elif c and p: if verbose: print("PASS") passed.append(test_dir) status_str += "." - print(status_str, end="") + std_out += rout + if not verbose: + print(status_str, end="", flush=True) + print("\n") + # Now print a summary + print("Passed: {}/{}".format(len(passed), len(test_dirs))) + if errored: + print("Errored: {}/{}:".format(len(errored), len(test_dirs)), errored) + print("\n====== COMPILATION ERRORS ======\n") + print(err_out) + if failed: + print("Failed: {}/{}:".format(len(failed), len(test_dirs)), failed) + print("\n====== RUN ERRORS ======\n") + print(fail_out) + print() if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run OpenFOAM unit tests") - parser.add_argument("--tests", "-t", help="Which tests to run or clean", + parser.add_argument("--tests", "-n", help="Which tests to run or clean", default="all", choices=collect_test_dirs() + ["all"], - metavar="tests") + metavar="tests", nargs="+") parser.add_argument("--clean", "-c", action="store_true", default=False, help="Clean all unit test executables") parser.add_argument("--verbose", "-v", action="store_true", default=False, @@ -111,4 +133,4 @@ if __name__ == "__main__": if args.clean: clean_all(args.tests) else: - test_multiple(args.tests) + test_multiple(args.tests, verbose=args.verbose) -- 2.7.4 From 49056d345e69b2c4c017307fdeffcb2b0c58f4d8 Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 15 May 2017 22:32:37 -0400 Subject: [PATCH 3/5] Test script should be run with Python 3 --- applications/test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/test/test.py b/applications/test/test.py index f3d0ebb..1683ead 100755 --- a/applications/test/test.py +++ b/applications/test/test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Automatically collect and run unit tests.""" from __future__ import division, print_function -- 2.7.4 From 792ca2a5169eaa997f443c25ffcd17758fd5a2ba Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 15 May 2017 23:10:49 -0400 Subject: [PATCH 4/5] Fix status str printing and raise exceptions if failed tests --- applications/test/test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/applications/test/test.py b/applications/test/test.py index 1683ead..e9f178b 100755 --- a/applications/test/test.py +++ b/applications/test/test.py @@ -75,7 +75,6 @@ def test_multiple(test_dirs="all", verbose=False): test_dirs = collect_test_dirs() print("\nRunning OpenFOAM unit tests") print("\nCollected {} tests\n".format(len(test_dirs))) - status_str = "" errored = [] passed = [] failed = [] @@ -90,19 +89,19 @@ def test_multiple(test_dirs="all", verbose=False): if verbose: print("ERROR") errored.append(test_dir) - status_str += "E" + status_str = "E" err_out += cout elif not p: if verbose: print("FAIL") failed.append(test_dir) - status_str += "F" + status_str = "F" fail_out += rout elif c and p: if verbose: print("PASS") passed.append(test_dir) - status_str += "." + status_str = "." std_out += rout if not verbose: print(status_str, end="", flush=True) @@ -112,11 +111,11 @@ def test_multiple(test_dirs="all", verbose=False): if errored: print("Errored: {}/{}:".format(len(errored), len(test_dirs)), errored) print("\n====== COMPILATION ERRORS ======\n") - print(err_out) + raise RuntimeError(err_out) if failed: print("Failed: {}/{}:".format(len(failed), len(test_dirs)), failed) print("\n====== RUN ERRORS ======\n") - print(fail_out) + raise RuntimeError(fail_out) print() -- 2.7.4 From 1f83cac6cebc0ad8d61681ebe51c988f75344fd8 Mon Sep 17 00:00:00 2001 From: Pete Bachant Date: Mon, 15 May 2017 23:22:49 -0400 Subject: [PATCH 5/5] Add exit on first error option --- applications/test/test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/applications/test/test.py b/applications/test/test.py index e9f178b..d01a5bc 100755 --- a/applications/test/test.py +++ b/applications/test/test.py @@ -69,7 +69,7 @@ def clean(test_dirs="all"): subprocess.call(["wclean", d]) -def test_multiple(test_dirs="all", verbose=False): +def test_multiple(test_dirs="all", exit_on_first=False, verbose=False): """Run multiple tests.""" if test_dirs == "all": test_dirs = collect_test_dirs() @@ -105,6 +105,8 @@ def test_multiple(test_dirs="all", verbose=False): std_out += rout if not verbose: print(status_str, end="", flush=True) + if exit_on_first and (not c or not p): + break print("\n") # Now print a summary print("Passed: {}/{}".format(len(passed), len(test_dirs))) @@ -126,10 +128,13 @@ if __name__ == "__main__": metavar="tests", nargs="+") parser.add_argument("--clean", "-c", action="store_true", default=False, help="Clean all unit test executables") + parser.add_argument("--existfirst", "-x", action="store_true", + default=False, help="Exit on first failure") parser.add_argument("--verbose", "-v", action="store_true", default=False, help="Print verbose output") args = parser.parse_args() if args.clean: clean_all(args.tests) else: - test_multiple(args.tests, verbose=args.verbose) + test_multiple(args.tests, exit_on_first=args.existfirst, + verbose=args.verbose) -- 2.7.4