automated-test.patch (12,850 bytes)
From ae29b299af40ad765ac5b1d58d0cd84601f40a29 Mon Sep 17 00:00:00 2001
From: Pete Bachant <petebachant@gmail.com>
Date: Mon, 15 May 2017 21:18:41 -0400
Subject: [PATCH 1/5] Start basic unit test runner script
---
applications/test/test.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 114 insertions(+)
create mode 100755 applications/test/test.py
diff --git a/applications/test/test.py b/applications/test/test.py
new file mode 100755
index 0000000..d9b79a8
--- /dev/null
+++ b/applications/test/test.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""Automatically collect and run unit tests."""
+
+from __future__ import division, print_function
+import subprocess
+import argparse
+import os
+import glob
+from subprocess import STDOUT
+
+# Run from this directory
+this_dir = os.path.abspath(os.path.dirname(__file__))
+os.chdir(this_dir)
+
+
+def collect_test_dirs():
+ """Loop through test application directories looking for ``Make``
+ subdirectories.
+ """
+ return sorted([d for d in os.listdir(".") if os.path.isdir(d) \
+ and "Make" in os.listdir(d)])
+
+
+def get_test_exe_name(test_dir):
+ """Get the name of the test executable."""
+ with open("Make/files") as f:
+ for line in f.readlines():
+ if line.strip().startswith("EXE"):
+ return line.strip().split("/")[-1]
+
+
+def test_single(test_dir, autopar=True):
+ """Run one unit test.
+
+ If ``autopar`` is ``True`` and test executable has ``parallel` in its name,
+ it will be run in parallel.
+ """
+ os.chdir(test_dir)
+ run_out = ""
+ compile_out = ""
+ try:
+ compile_out = subprocess.check_output("wmake", stderr=STDOUT).decode()
+ test_compiled = True
+ except subprocess.CalledProcessError:
+ test_compiled = False
+ if test_compiled:
+ test_exe = get_test_exe_name(test_dir)
+ if autopar and "parallel" in test_exe.lower():
+ test_exe = "mpirun -np 2 " + test_exe
+ try:
+ run_out = subprocess.check_output(test_exe, stderr=STDOUT,
+ shell=True).decode()
+ test_ran = True
+ except subprocess.CalledProcessError:
+ test_ran = False
+ else:
+ test_ran = False
+ os.chdir(this_dir)
+ return test_compiled, test_ran, compile_out, run_out
+
+
+def clean(test_dirs="all"):
+ """Go through all directories and clean executables."""
+ if test_dirs == "all":
+ test_dirs = collect_test_dirs()
+ for d in test_dirs:
+ subprocess.call(["wclean", d])
+
+
+def test_multiple(test_dirs="all", verbose=False):
+ """Run multiple tests."""
+ if test_dirs == "all":
+ test_dirs = collect_test_dirs()
+ print("Collected {} tests\n".format(len(test_dirs)))
+ status_str = ""
+ errored = []
+ passed = []
+ failed = []
+ for test_dir in test_dirs:
+ if verbose:
+ print("Testing", test_dir)
+ c, p, cout, rout = test_single(test_dir)
+ if not c:
+ if verbose:
+ print("ERROR")
+ errored.append(test_dir)
+ status_str += "E"
+ elif not p:
+ if verbose:
+ print("FAIL")
+ failed.append(test_dir)
+ status_str += "F"
+ else:
+ if verbose:
+ print("PASS")
+ passed.append(test_dir)
+ status_str += "."
+ print(status_str, end="")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Run OpenFOAM unit tests")
+ parser.add_argument("--tests", "-t", help="Which tests to run or clean",
+ default="all", choices=collect_test_dirs() + ["all"],
+ metavar="tests")
+ parser.add_argument("--clean", "-c", action="store_true", default=False,
+ help="Clean all unit test executables")
+ parser.add_argument("--verbose", "-v", action="store_true", default=False,
+ help="Print verbose output")
+ args = parser.parse_args()
+ if args.clean:
+ clean_all(args.tests)
+ else:
+ test_multiple(args.tests)
--
2.7.4
From d5c559505c3414f6708a72874a273a974dde82f5 Mon Sep 17 00:00:00 2001
From: Pete Bachant <petebachant@gmail.com>
Date: Mon, 15 May 2017 21:53:28 -0400
Subject: [PATCH 2/5] Improve summary and print output
---
applications/test/test.py | 40 +++++++++++++++++++++++++++++++---------
1 file changed, 31 insertions(+), 9 deletions(-)
diff --git a/applications/test/test.py b/applications/test/test.py
index d9b79a8..f3d0ebb 100755
--- a/applications/test/test.py
+++ b/applications/test/test.py
@@ -41,7 +41,8 @@ def test_single(test_dir, autopar=True):
try:
compile_out = subprocess.check_output("wmake", stderr=STDOUT).decode()
test_compiled = True
- except subprocess.CalledProcessError:
+ except subprocess.CalledProcessError as e:
+ compile_out += e.output.decode()
test_compiled = False
if test_compiled:
test_exe = get_test_exe_name(test_dir)
@@ -51,7 +52,8 @@ def test_single(test_dir, autopar=True):
run_out = subprocess.check_output(test_exe, stderr=STDOUT,
shell=True).decode()
test_ran = True
- except subprocess.CalledProcessError:
+ except subprocess.CalledProcessError as e:
+ run_out += e.output.decode()
test_ran = False
else:
test_ran = False
@@ -71,38 +73,58 @@ def test_multiple(test_dirs="all", verbose=False):
"""Run multiple tests."""
if test_dirs == "all":
test_dirs = collect_test_dirs()
- print("Collected {} tests\n".format(len(test_dirs)))
+ print("\nRunning OpenFOAM unit tests")
+ print("\nCollected {} tests\n".format(len(test_dirs)))
status_str = ""
errored = []
passed = []
failed = []
+ err_out = ""
+ fail_out = ""
+ std_out = ""
for test_dir in test_dirs:
if verbose:
- print("Testing", test_dir)
+ print("Testing {:24s}".format(test_dir + "..."), end="", sep=" ")
c, p, cout, rout = test_single(test_dir)
if not c:
if verbose:
print("ERROR")
errored.append(test_dir)
status_str += "E"
+ err_out += cout
elif not p:
if verbose:
print("FAIL")
failed.append(test_dir)
status_str += "F"
- else:
+ fail_out += rout
+ elif c and p:
if verbose:
print("PASS")
passed.append(test_dir)
status_str += "."
- print(status_str, end="")
+ std_out += rout
+ if not verbose:
+ print(status_str, end="", flush=True)
+ print("\n")
+ # Now print a summary
+ print("Passed: {}/{}".format(len(passed), len(test_dirs)))
+ if errored:
+ print("Errored: {}/{}:".format(len(errored), len(test_dirs)), errored)
+ print("\n====== COMPILATION ERRORS ======\n")
+ print(err_out)
+ if failed:
+ print("Failed: {}/{}:".format(len(failed), len(test_dirs)), failed)
+ print("\n====== RUN ERRORS ======\n")
+ print(fail_out)
+ print()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run OpenFOAM unit tests")
- parser.add_argument("--tests", "-t", help="Which tests to run or clean",
+ parser.add_argument("--tests", "-n", help="Which tests to run or clean",
default="all", choices=collect_test_dirs() + ["all"],
- metavar="tests")
+ metavar="tests", nargs="+")
parser.add_argument("--clean", "-c", action="store_true", default=False,
help="Clean all unit test executables")
parser.add_argument("--verbose", "-v", action="store_true", default=False,
@@ -111,4 +133,4 @@ if __name__ == "__main__":
if args.clean:
clean_all(args.tests)
else:
- test_multiple(args.tests)
+ test_multiple(args.tests, verbose=args.verbose)
--
2.7.4
From 49056d345e69b2c4c017307fdeffcb2b0c58f4d8 Mon Sep 17 00:00:00 2001
From: Pete Bachant <petebachant@gmail.com>
Date: Mon, 15 May 2017 22:32:37 -0400
Subject: [PATCH 3/5] Test script should be run with Python 3
---
applications/test/test.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/applications/test/test.py b/applications/test/test.py
index f3d0ebb..1683ead 100755
--- a/applications/test/test.py
+++ b/applications/test/test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
"""Automatically collect and run unit tests."""
from __future__ import division, print_function
--
2.7.4
From 792ca2a5169eaa997f443c25ffcd17758fd5a2ba Mon Sep 17 00:00:00 2001
From: Pete Bachant <petebachant@gmail.com>
Date: Mon, 15 May 2017 23:10:49 -0400
Subject: [PATCH 4/5] Fix status str printing and raise exceptions if failed
tests
---
applications/test/test.py | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/applications/test/test.py b/applications/test/test.py
index 1683ead..e9f178b 100755
--- a/applications/test/test.py
+++ b/applications/test/test.py
@@ -75,7 +75,6 @@ def test_multiple(test_dirs="all", verbose=False):
test_dirs = collect_test_dirs()
print("\nRunning OpenFOAM unit tests")
print("\nCollected {} tests\n".format(len(test_dirs)))
- status_str = ""
errored = []
passed = []
failed = []
@@ -90,19 +89,19 @@ def test_multiple(test_dirs="all", verbose=False):
if verbose:
print("ERROR")
errored.append(test_dir)
- status_str += "E"
+ status_str = "E"
err_out += cout
elif not p:
if verbose:
print("FAIL")
failed.append(test_dir)
- status_str += "F"
+ status_str = "F"
fail_out += rout
elif c and p:
if verbose:
print("PASS")
passed.append(test_dir)
- status_str += "."
+ status_str = "."
std_out += rout
if not verbose:
print(status_str, end="", flush=True)
@@ -112,11 +111,11 @@ def test_multiple(test_dirs="all", verbose=False):
if errored:
print("Errored: {}/{}:".format(len(errored), len(test_dirs)), errored)
print("\n====== COMPILATION ERRORS ======\n")
- print(err_out)
+ raise RuntimeError(err_out)
if failed:
print("Failed: {}/{}:".format(len(failed), len(test_dirs)), failed)
print("\n====== RUN ERRORS ======\n")
- print(fail_out)
+ raise RuntimeError(fail_out)
print()
--
2.7.4
From 1f83cac6cebc0ad8d61681ebe51c988f75344fd8 Mon Sep 17 00:00:00 2001
From: Pete Bachant <petebachant@gmail.com>
Date: Mon, 15 May 2017 23:22:49 -0400
Subject: [PATCH 5/5] Add exit on first error option
---
applications/test/test.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/applications/test/test.py b/applications/test/test.py
index e9f178b..d01a5bc 100755
--- a/applications/test/test.py
+++ b/applications/test/test.py
@@ -69,7 +69,7 @@ def clean(test_dirs="all"):
subprocess.call(["wclean", d])
-def test_multiple(test_dirs="all", verbose=False):
+def test_multiple(test_dirs="all", exit_on_first=False, verbose=False):
"""Run multiple tests."""
if test_dirs == "all":
test_dirs = collect_test_dirs()
@@ -105,6 +105,8 @@ def test_multiple(test_dirs="all", verbose=False):
std_out += rout
if not verbose:
print(status_str, end="", flush=True)
+ if exit_on_first and (not c or not p):
+ break
print("\n")
# Now print a summary
print("Passed: {}/{}".format(len(passed), len(test_dirs)))
@@ -126,10 +128,13 @@ if __name__ == "__main__":
metavar="tests", nargs="+")
parser.add_argument("--clean", "-c", action="store_true", default=False,
help="Clean all unit test executables")
+ parser.add_argument("--existfirst", "-x", action="store_true",
+ default=False, help="Exit on first failure")
parser.add_argument("--verbose", "-v", action="store_true", default=False,
help="Print verbose output")
args = parser.parse_args()
if args.clean:
clean_all(args.tests)
else:
- test_multiple(args.tests, verbose=args.verbose)
+ test_multiple(args.tests, exit_on_first=args.existfirst,
+ verbose=args.verbose)
--
2.7.4
