tests: refactor the test runner

.github/workflows/ci-run-tests.sh

@@ -3,7 +3,8 @@
 set -euo pipefail
 
 run () {
-  tests/tools/run-tests build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64
+  export PYTHONPATH=tests/tools
+  python -m libkcov build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64
 }
 
 run "$@"

tests/tools/testbase.py → tests/tools/libkcov/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 import errno
 import os
 import os.path
@@ -12,43 +10,21 @@
 
 PIPE = subprocess.PIPE
 
-kcov = ""
-outbase = ""
-testbuild = ""
-sources = ""
-
 default_timeout = 10 * 60
 
 
-# Normalize path, also ensuring that it is not empty.
-def normalize(path):
-    assert len(path) != 0, "path must be not empty"
-
-    path = os.path.normpath(path)
-    return path
-
-
-def configure(k, o, t, s):
-    global kcov, outbase, testbuild, sources, kcov_system_daemon
+class TestCase(unittest.TestCase):
+    def __init__(self, kcov, outbase, binaries, sources):
+        super().__init__()
 
-    kcov = normalize(k)
-    outbase = normalize(o)
-    testbuild = normalize(t)
-    sources = normalize(s)
-
-    assert os.path.abspath(outbase) != os.getcwd(), "'outbase' cannot be the current directory"
-
-
-class KcovTestCase(unittest.TestCase):
-    def setUp(self):
-        # Make the configuration values available as class members.
         self.kcov = kcov
         self.kcov_system_daemon = self.kcov + "-system-daemon"
         self.outbase = outbase
         self.outdir = self.outbase + "/" + "kcov"
-        self.testbuild = testbuild
+        self.binaries = binaries
         self.sources = sources
 
+    def setUp(self):
         # Intentionally fails if target directory exists.
         os.makedirs(self.outdir)
 
@@ -83,7 +59,7 @@ def do(self, cmdline, /, kcovKcov=True, *, timeout=default_timeout):
             and platform.machine() in ["x86_64", "i386", "i686"]
         ):
             extra = (
-                kcov
+                self.kcov
                 + " --include-pattern=kcov --exclude-pattern=helper.cc,library.cc,html-data-files.cc "
                 + "/tmp/kcov-kcov "
             )

tests/tools/libkcov/__main__.py

@@ -0,0 +1,5 @@
+"""Main entry point."""
+
+from .main import main
+
+main()

tests/tools/cobertura.py → tests/tools/libkcov/cobertura.py

@@ -1,10 +1,5 @@
-#!/usr/bin/env python3
-
-import sys
 import xml.dom.minidom
 
-# all these imports are standard on most modern python implementations
-
 
 def readFile(name):
     f = open(name, "r")
@@ -61,25 +56,3 @@ def hitsPerLine(dom, fileName, lineNr):
             return hits
 
     return None
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 4:
-        print("Usage: lookup-class-line <in-file> <filename> <lineNr>")
-        sys.exit(1)
-
-    fileName = sys.argv[2]
-    line = int(sys.argv[3])
-
-    data = readFile(sys.argv[1])
-
-    dom = parse(data)
-    fileTag = lookupClassName(dom, fileName)
-
-    if fileTag is not None:
-        hits = lookupHitsByLine(fileTag, line)
-        if hits is not None:
-            print(hits)
-            sys.exit(0)
-
-    print("nocode")

tests/tools/libkcov/main.py

@@ -0,0 +1,231 @@
+"""A custom simplified implementation of the test runner.
+
+This is necessary, since `unittest.main` does not support correctly how kcov
+tests are imported.  It will also make it easy to add additional features.
+
+Most of the test runner code has been copied from the `unittest.main` module.
+"""
+
+import argparse
+import importlib
+import os
+import os.path
+import platform
+import random
+import sys
+import unittest
+from collections import namedtuple
+from fnmatch import fnmatchcase
+
+# Copied from unittest.main.
+_NO_TESTS_EXITCODE = 5
+
+
+class TestLoader:
+    """A simplified test loader.
+
+    The implementation assumes that each test case runs only one test method:
+    `runTest`.
+    """
+
+    def __init__(self, config, patterns):
+        self.tests = []
+        self.config = config
+        self.patterns = patterns
+
+    def add_tests_from_module(self, name):
+        """Add all test cases from the named module."""
+
+        module = importlib.import_module(name)
+        for name in dir(module):
+            obj = getattr(module, name)
+            if (
+                isinstance(obj, type)
+                and issubclass(obj, unittest.TestCase)
+                and obj not in (unittest.TestCase, unittest.FunctionTestCase)
+                and hasattr(obj, "runTest")
+            ):
+                if not self.match_test(obj):
+                    continue
+
+                cfg = self.config
+                test = obj(cfg.kcov, cfg.outbase, cfg.binaries, cfg.sources)
+                self.tests.append(test)
+
+    def match_test(self, test_case_class):
+        if not self.patterns:
+            return True
+
+        full_name = f"{test_case_class.__module__}.{test_case_class.__qualname__}"
+        return any(fnmatchcase(full_name, pattern) for pattern in self.patterns)
+
+
+Config = namedtuple("Config", ["kcov", "outbase", "binaries", "sources"])
+
+
+def fatal(msg):
+    sys.stderr.write(f"error: {msg}\n")
+    sys.stderr.flush()
+    os._exit(1)
+
+
+def normalized_not_empty(path):
+    """Normalize path, also ensuring that it is not empty."""
+
+    if len(path) == 0:
+        raise ValueError("path must be not empty")
+
+    path = os.path.normpath(path)
+    return path
+
+
+# Implementation copied from unittest.main.
+def to_fnmatch(pattern):
+    if "*" not in pattern:
+        pattern = "*%s*" % pattern
+
+    return pattern
+
+
+def addTests(config, patterns):
+    """Add all the kcov test modules.
+
+    Discovery is not possible, since some modules need to be excluded,
+    depending on the os and arch.
+    """
+
+    test_loader = TestLoader(config, patterns)
+
+    test_loader.add_tests_from_module("test_basic")
+    test_loader.add_tests_from_module("test_compiled_basic")
+
+    if platform.machine() in ["x86_64", "i386", "i686"]:
+        test_loader.add_tests_from_module("test_compiled")
+    if sys.platform.startswith("linux"):
+        test_loader.add_tests_from_module("test_bash_linux_only")
+
+        if platform.machine() in ["x86_64", "i386", "i686"]:
+            test_loader.add_tests_from_module("test_system_mode")
+
+    test_loader.add_tests_from_module("test_accumulate")
+    test_loader.add_tests_from_module("test_bash")
+    test_loader.add_tests_from_module("test_filter")
+    test_loader.add_tests_from_module("test_python")
+
+    return test_loader.tests
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("kcov", type=normalized_not_empty)
+    parser.add_argument("outbase", type=normalized_not_empty)
+    parser.add_argument("binaries", type=normalized_not_empty)
+    parser.add_argument("sources", type=normalized_not_empty)
+
+    # Code copied from unittest.main.
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbosity",
+        action="store_const",
+        const=2,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="verbosity",
+        action="store_const",
+        const=0,
+        help="Quiet output",
+    )
+    parser.add_argument(
+        "-f",
+        "--failfast",
+        dest="failfast",
+        action="store_true",
+        help="Stop on first fail or error",
+    )
+    parser.add_argument(
+        "-b",
+        "--buffer",
+        dest="buffer",
+        action="store_true",
+        help="Buffer stdout and stderr during tests",
+    )
+    parser.add_argument(
+        "-c",
+        "--catch",
+        dest="catchbreak",
+        action="store_true",
+        help="Catch Ctrl-C and display results so far",
+    )
+    parser.add_argument(
+        "-k",
+        dest="patterns",
+        action="append",
+        type=to_fnmatch,
+        help="Only run tests which match the given substring",
+    )
+    parser.add_argument(
+        "--locals",
+        dest="tb_locals",
+        action="store_true",
+        help="Show local variables in tracebacks",
+    )
+
+    # TODO: The --duration argument was added in 3.12.
+
+    # kcov test runner options
+
+    parser.add_argument(
+        "--shuffle",
+        dest="shuffle",
+        action="store_true",
+        help="Randomize the execution order of tests",
+    )
+
+    # TODO: The --duration argument was added in 3.12.
+
+    # kcov test runner custom options
+
+    return parser.parse_args()
+
+
+def main():
+    # Parse the command line and validate the configuration paths
+    args = parse_args()
+
+    if os.path.abspath(args.outbase) == os.getcwd():
+        fatal("'outbase' cannot be the current directory")
+
+    # Loads and configure tests
+    config = Config(args.kcov, args.outbase, args.binaries, args.sources)
+    tests = addTests(config, args.patterns)
+
+    if args.shuffle:
+        random.shuffle(tests)
+
+    # Run the tests
+    test_suite = unittest.TestSuite(tests)
+
+    # Code copied from unittest.main.
+    if args.catchbreak:
+        unittest.installHandler()
+
+    test_runner = unittest.TextTestRunner(
+        verbosity=args.verbosity,
+        failfast=args.failfast,
+        buffer=args.buffer,
+        tb_locals=args.tb_locals,
+    )
+
+    result = test_runner.run(test_suite)
+    if True:
+        if result.testsRun == 0 and len(result.skipped) == 0:
+            sys.exit(_NO_TESTS_EXITCODE)
+        elif result.wasSuccessful():
+            sys.exit(0)
+        else:
+            sys.exit(1)

tests/tools/parse_cobertura

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import sys
+
+from libkcov import cobertura
+
+if len(sys.argv) < 4:
+    print("Usage: lookup-class-line <in-file> <filename> <lineNr>")
+    sys.exit(1)
+
+fileName = sys.argv[2]
+line = int(sys.argv[3])
+
+data = cobertura.readFile(sys.argv[1])
+
+dom = cobertura.parse(data)
+fileTag = cobertura.lookupClassName(dom, fileName)
+
+if fileTag is not None:
+    hits = cobertura.lookupHitsByLine(fileTag, line)
+    if hits is not None:
+        print(hits)
+        sys.exit(0)
+
+print("nocode")