tests: refactor the test runner

The current implementation, using the unittest.main function, is not
working correctly with how kcov tests are loaded.  One annoying problem
is that filtering tests is not possible.

Rewrite the test runner, adding a new libkcov package.

The code from "testbase.py" is moved to libkcov.__init__ (the test API),
and the code from "run-tests" is moved to libkcov.main (the test
runner).

In the test API, remove the configure method and the global variables,
since the configuration paths are now processed by argparse as
positional arguments.  These paths are validated and then used to
initialize each test case, that now has an __init__ method.

In order to make running the tests convenient, add a __main__ entry
point to the libkcov package.

Rename the test names; "file.py" is renamed to "test_file.py", since
this is the preferred naming conventions.

Tests are loaded by a custom test loader, not derived from
unittest.TestCase, since the code is really simple.

Move the "cobertura.py" module to the libkcov package, but without the
main code.  The main code has been moved to "parse_cobertura", so that
it can be used as a tool.

The new implementation make is easy do separate the test API, test
runner, tests and tools.

In order to reduce the number of changes, in each test file libkcov is
imported using as testbase.  It will updated in a separate commit.

Update the "ci-run-tests.sh" script to run the tests with `python -m
linbkcov`.  It is necessary to set the PYTHONPATH environment variable.

The behavior of the new test runner implementation should be the same as
the old one, with the difference that the order of tests is different.
The new order is what users expect.

.github/workflows/ci-run-tests.sh

@@ -3,7 +3,8 @@
 set -euo pipefail
 
 run () {
-  tests/tools/run-tests build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64
+  export PYTHONPATH=tests/tools
+  python -m libkcov build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64
 }
 
 run "$@"

tests/tools/testbase.py → tests/tools/libkcov/__init__.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 import errno
 import os
 import os.path
@@ -12,43 +10,21 @@
 
 PIPE = subprocess.PIPE
 
-kcov = ""
-outbase = ""
-testbuild = ""
-sources = ""
-
 default_timeout = 10 * 60
 
 
-# Normalize path, also ensuring that it is not empty.
-def normalize(path):
-    assert len(path) != 0, "path must be not empty"
-
-    path = os.path.normpath(path)
-    return path
-
-
-def configure(k, o, t, s):
-    global kcov, outbase, testbuild, sources, kcov_system_daemon
-
-    kcov = normalize(k)
-    outbase = normalize(o)
-    testbuild = normalize(t)
-    sources = normalize(s)
-
-    assert os.path.abspath(outbase) != os.getcwd(), "'outbase' cannot be the current directory"
-
-
 class KcovTestCase(unittest.TestCase):
-    def setUp(self):
-        # Make the configuration values available as class members.
+    def __init__(self, kcov, outbase, testbuild, sources):
+        super().__init__()
+
         self.kcov = kcov
         self.kcov_system_daemon = self.kcov + "-system-daemon"
         self.outbase = outbase
         self.outdir = self.outbase + "/" + "kcov"
         self.testbuild = testbuild
         self.sources = sources
 
+    def setUp(self):
         # Intentionally fails if target directory exists.
         os.makedirs(self.outdir)
 
@@ -83,7 +59,7 @@ def do(self, cmdline, /, kcovKcov=True, *, timeout=default_timeout):
             and platform.machine() in ["x86_64", "i386", "i686"]
         ):
             extra = (
-                kcov
+                self.kcov
                 + " --include-pattern=kcov --exclude-pattern=helper.cc,library.cc,html-data-files.cc "
                 + "/tmp/kcov-kcov "
             )

tests/tools/libkcov/__main__.py

@@ -0,0 +1,5 @@
+"""Main entry point."""
+
+from .main import main
+
+main()

tests/tools/cobertura.py → tests/tools/libkcov/cobertura.py

@@ -1,10 +1,5 @@
-#!/usr/bin/env python3
-
-import sys
 import xml.dom.minidom
 
-# all these imports are standard on most modern python implementations
-
 
 def readFile(name):
     f = open(name, "r")
@@ -61,25 +56,3 @@ def hitsPerLine(dom, fileName, lineNr):
             return hits
 
     return None
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 4:
-        print("Usage: lookup-class-line <in-file> <filename> <lineNr>")
-        sys.exit(1)
-
-    fileName = sys.argv[2]
-    line = int(sys.argv[3])
-
-    data = readFile(sys.argv[1])
-
-    dom = parse(data)
-    fileTag = lookupClassName(dom, fileName)
-
-    if fileTag is not None:
-        hits = lookupHitsByLine(fileTag, line)
-        if hits is not None:
-            print(hits)
-            sys.exit(0)
-
-    print("nocode")

tests/tools/libkcov/main.py

@@ -0,0 +1,196 @@
+"""A custom simplified implementation of the test runner.
+
+This is necessary, since `unittest.main` does not support correctly how kcov
+tests are imported.  It will also make it easy to add additional features.
+
+Most of the test runner code has been copied from the `unittest.main` module.
+"""
+
+import argparse
+import importlib
+import os
+import os.path
+import platform
+import sys
+import unittest
+from collections import namedtuple
+
+# Copied from unittest.main.
+_NO_TESTS_EXITCODE = 5
+
+
+class TestLoader:
+    """A simplified test loader.
+
+    The implementation assumes that each test case runs only one test method:
+    `runTest`.
+    """
+
+    def __init__(self, config):
+        self.tests = []
+        self.config = config
+
+    def add_tests_from_module(self, name):
+        """Add all test cases from the named module."""
+
+        module = importlib.import_module(name)
+        for name in dir(module):
+            obj = getattr(module, name)
+            if (
+                isinstance(obj, type)
+                and issubclass(obj, unittest.TestCase)
+                and obj not in (unittest.TestCase, unittest.FunctionTestCase)
+                and hasattr(obj, "runTest")
+            ):
+                cfg = self.config
+
+                # TODO: add support for filtering.
+
+                test = obj(cfg.kcov, cfg.outbase, cfg.testbuild, cfg.sources)
+                self.tests.append(test)
+
+
+Config = namedtuple("Config", ["kcov", "outbase", "testbuild", "sources"])
+
+
+def fatal(msg):
+    sys.stderr.write(f"error: {msg}\n")
+    sys.stderr.flush()
+    os._exit(1)
+
+
+def normalized_not_empty(path):
+    """Normalize path, also ensuring that it is not empty."""
+
+    if len(path) == 0:
+        raise ValueError("path must be not empty")
+
+    path = os.path.normpath(path)
+    return path
+
+
+def addTests(config):
+    """Add all the kcov test modules.
+
+    Discovery is not possible, since some modules need to be excluded,
+    depending on the os and arch.
+    """
+
+    test_loader = TestLoader(config)
+
+    test_loader.add_tests_from_module("test_basic")
+    test_loader.add_tests_from_module("test_compiled_basic")
+
+    if platform.machine() in ["x86_64", "i386", "i686"]:
+        test_loader.add_tests_from_module("test_compiled")
+    if sys.platform.startswith("linux"):
+        test_loader.add_tests_from_module("test_bash_linux_only")
+
+        if platform.machine() in ["x86_64", "i386", "i686"]:
+            test_loader.add_tests_from_module("test_system_mode")
+
+    test_loader.add_tests_from_module("test_accumulate")
+    test_loader.add_tests_from_module("test_bash")
+    test_loader.add_tests_from_module("test_filter")
+    test_loader.add_tests_from_module("test_python")
+
+    return test_loader.tests
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("kcov", type=normalized_not_empty)
+    parser.add_argument("outbase", type=normalized_not_empty)
+    parser.add_argument("testbuild", type=normalized_not_empty)
+    parser.add_argument("sources", type=normalized_not_empty)
+
+    # Code copied from unittest.main.
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbosity",
+        action="store_const",
+        const=2,
+        help="Verbose output",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        dest="verbosity",
+        action="store_const",
+        const=0,
+        help="Quiet output",
+    )
+    parser.add_argument(
+        "-f",
+        "--failfast",
+        dest="failfast",
+        action="store_true",
+        help="Stop on first fail or error",
+    )
+    parser.add_argument(
+        "-b",
+        "--buffer",
+        dest="buffer",
+        action="store_true",
+        help="Buffer stdout and stderr during tests",
+    )
+    parser.add_argument(
+        "-c",
+        "--catch",
+        dest="catchbreak",
+        action="store_true",
+        help="Catch Ctrl-C and display results so far",
+    )
+    parser.add_argument(
+        "-k",
+        dest="patterns",
+        action="append",
+        help="Only run tests which match the given substring",
+    )
+    parser.add_argument(
+        "--locals",
+        dest="tb_locals",
+        action="store_true",
+        help="Show local variables in tracebacks",
+    )
+
+    # TODO: The --duration argument was added in 3.12.
+
+    return parser.parse_args()
+
+
+def main():
+    # Parse the command line and validate the configuration paths
+    args = parse_args()
+
+    if os.path.abspath(args.outbase) == os.getcwd():
+        fatal("'outbase' cannot be the current directory")
+
+    # Loads and configure tests
+    config = Config(args.kcov, args.outbase, args.testbuild, args.sources)
+    tests = addTests(config)
+
+    # Run the tests
+    test_suite = unittest.TestSuite(tests)
+
+    # Code copied from unittest.main.
+    if args.catchbreak:
+        unittest.installHandler()
+
+    test_runner = unittest.TextTestRunner(
+        verbosity=args.verbosity,
+        failfast=args.failfast,
+        buffer=args.buffer,
+        tb_locals=args.tb_locals,
+    )
+
+    result = test_runner.run(test_suite)
+    if True:
+        if result.testsRun == 0 and len(result.skipped) == 0:
+            sys.exit(_NO_TESTS_EXITCODE)
+        elif result.wasSuccessful():
+            sys.exit(0)
+        else:
+            sys.exit(1)

tests/tools/parse_cobertura

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import sys
+
+from libkcov import cobertura
+
+if len(sys.argv) < 4:
+    print("Usage: lookup-class-line <in-file> <filename> <lineNr>")
+    sys.exit(1)
+
+fileName = sys.argv[2]
+line = int(sys.argv[3])
+
+data = cobertura.readFile(sys.argv[1])
+
+dom = cobertura.parse(data)
+fileTag = cobertura.lookupClassName(dom, fileName)
+
+if fileTag is not None:
+    hits = cobertura.lookupHitsByLine(fileTag, line)
+    if hits is not None:
+        print(hits)
+        sys.exit(0)
+
+print("nocode")

tests/tools/accumulate.py → tests/tools/test_accumulate.py

@@ -1,5 +1,5 @@
-import cobertura
-import testbase
+import libkcov as testbase
+from libkcov import cobertura
 
 
 class accumulate_data(testbase.KcovTestCase):

tests/tools/bash.py → tests/tools/test_bash.py

@@ -3,8 +3,8 @@
 import sys
 import unittest
 
-import cobertura
-import testbase
+import libkcov as testbase
+from libkcov import cobertura
 
 
 class BashBase(testbase.KcovTestCase):