From 74cdbaa16ad30e5a0ab1af9890a29e6e1f4c1361 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Tue, 2 Apr 2024 21:11:52 +0200 Subject: [PATCH] tests: refactor the test runner The current implementation, using the unittest.main function, is not working correctly with how kcov tests are loaded. One annoying problem is that filtering tests is not possible. Rewrite the test runner, adding a new libkcov package. The code from "testbase.py" is moved to libkcov.__init__ (the test API), and the code from "run-tests" is moved to libkcov.main (the test runner). In the test API, remove the configure method and the global variables, since the configuration paths are now processed by argparse as positional arguments. These paths are validated and then used to initialize each test case, that now has an __init__ method. In order to make running the tests convenient, add a __main__ entry point to the libkcov package. Rename the test names; "file.py" is renamed to "test_file.py", since this is the preferred naming conventions. Tests are loaded by a custom test loader, not derived from unittest.TestCase, since the code is really simple. Move the "cobertura.py" module to the libkcov package, but without the main code. The main code has been moved to "parse_cobertura", so that it can be used as a tool. The new implementation make is easy do separate the test API, test runner, tests and tools. In order to reduce the number of changes, in each test file libkcov is imported using as testbase. It will updated in a separate commit. Update the "ci-run-tests.sh" script to run the tests with `python -m linbkcov`. It is necessary to set the PYTHONPATH environment variable. The behavior of the new test runner implementation should be the same as the old one, with the difference that the order of tests is different. The new order is what users expect. --- .github/workflows/ci-run-tests.sh | 3 +- .../{testbase.py => libkcov/__init__.py} | 34 +-- tests/tools/libkcov/__main__.py | 5 + tests/tools/{ => libkcov}/cobertura.py | 27 --- tests/tools/libkcov/main.py | 196 ++++++++++++++++++ tests/tools/parse_cobertura | 25 +++ tests/tools/run-tests | 36 ---- .../{accumulate.py => test_accumulate.py} | 4 +- tests/tools/{bash.py => test_bash.py} | 4 +- ..._linux_only.py => test_bash_linux_only.py} | 4 +- tests/tools/{basic.py => test_basic.py} | 4 +- tests/tools/{compiled.py => test_compiled.py} | 4 +- ...mpiled_basic.py => test_compiled_basic.py} | 4 +- tests/tools/{filter.py => test_filter.py} | 4 +- tests/tools/{python.py => test_python.py} | 4 +- .../{system_mode.py => test_system_mode.py} | 4 +- 16 files changed, 251 insertions(+), 111 deletions(-) rename tests/tools/{testbase.py => libkcov/__init__.py} (79%) create mode 100644 tests/tools/libkcov/__main__.py rename tests/tools/{ => libkcov}/cobertura.py (67%) mode change 100755 => 100644 create mode 100644 tests/tools/libkcov/main.py create mode 100755 tests/tools/parse_cobertura delete mode 100755 tests/tools/run-tests rename tests/tools/{accumulate.py => test_accumulate.py} (98%) rename tests/tools/{bash.py => test_bash.py} (99%) rename tests/tools/{bash_linux_only.py => test_bash_linux_only.py} (97%) rename tests/tools/{basic.py => test_basic.py} (96%) rename tests/tools/{compiled.py => test_compiled.py} (99%) rename tests/tools/{compiled_basic.py => test_compiled_basic.py} (98%) rename tests/tools/{filter.py => test_filter.py} (98%) rename tests/tools/{python.py => test_python.py} (99%) rename tests/tools/{system_mode.py => test_system_mode.py} (97%) diff --git a/.github/workflows/ci-run-tests.sh b/.github/workflows/ci-run-tests.sh index df9b1a3d..776fc62d 100755 --- a/.github/workflows/ci-run-tests.sh +++ b/.github/workflows/ci-run-tests.sh @@ -3,7 +3,8 @@ set -euo pipefail run () { - tests/tools/run-tests build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64 + export PYTHONPATH=tests/tools + python -m libkcov build/src/kcov /tmp/ build-tests/ "." -v "$@" || exit 64 } run "$@" diff --git a/tests/tools/testbase.py b/tests/tools/libkcov/__init__.py similarity index 79% rename from tests/tools/testbase.py rename to tests/tools/libkcov/__init__.py index 922612e2..e5c2200f 100644 --- a/tests/tools/testbase.py +++ b/tests/tools/libkcov/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import errno import os import os.path @@ -12,36 +10,13 @@ PIPE = subprocess.PIPE -kcov = "" -outbase = "" -testbuild = "" -sources = "" - default_timeout = 10 * 60 -# Normalize path, also ensuring that it is not empty. -def normalize(path): - assert len(path) != 0, "path must be not empty" - - path = os.path.normpath(path) - return path - - -def configure(k, o, t, s): - global kcov, outbase, testbuild, sources, kcov_system_daemon - - kcov = normalize(k) - outbase = normalize(o) - testbuild = normalize(t) - sources = normalize(s) - - assert os.path.abspath(outbase) != os.getcwd(), "'outbase' cannot be the current directory" - - class KcovTestCase(unittest.TestCase): - def setUp(self): - # Make the configuration values available as class members. + def __init__(self, kcov, outbase, testbuild, sources): + super().__init__() + self.kcov = kcov self.kcov_system_daemon = self.kcov + "-system-daemon" self.outbase = outbase @@ -49,6 +24,7 @@ def setUp(self): self.testbuild = testbuild self.sources = sources + def setUp(self): # Intentionally fails if target directory exists. os.makedirs(self.outdir) @@ -83,7 +59,7 @@ def do(self, cmdline, /, kcovKcov=True, *, timeout=default_timeout): and platform.machine() in ["x86_64", "i386", "i686"] ): extra = ( - kcov + self.kcov + " --include-pattern=kcov --exclude-pattern=helper.cc,library.cc,html-data-files.cc " + "/tmp/kcov-kcov " ) diff --git a/tests/tools/libkcov/__main__.py b/tests/tools/libkcov/__main__.py new file mode 100644 index 00000000..c510f37e --- /dev/null +++ b/tests/tools/libkcov/__main__.py @@ -0,0 +1,5 @@ +"""Main entry point.""" + +from .main import main + +main() diff --git a/tests/tools/cobertura.py b/tests/tools/libkcov/cobertura.py old mode 100755 new mode 100644 similarity index 67% rename from tests/tools/cobertura.py rename to tests/tools/libkcov/cobertura.py index 7ac97700..3046783e --- a/tests/tools/cobertura.py +++ b/tests/tools/libkcov/cobertura.py @@ -1,10 +1,5 @@ -#!/usr/bin/env python3 - -import sys import xml.dom.minidom -# all these imports are standard on most modern python implementations - def readFile(name): f = open(name, "r") @@ -61,25 +56,3 @@ def hitsPerLine(dom, fileName, lineNr): return hits return None - - -if __name__ == "__main__": - if len(sys.argv) < 4: - print("Usage: lookup-class-line ") - sys.exit(1) - - fileName = sys.argv[2] - line = int(sys.argv[3]) - - data = readFile(sys.argv[1]) - - dom = parse(data) - fileTag = lookupClassName(dom, fileName) - - if fileTag is not None: - hits = lookupHitsByLine(fileTag, line) - if hits is not None: - print(hits) - sys.exit(0) - - print("nocode") diff --git a/tests/tools/libkcov/main.py b/tests/tools/libkcov/main.py new file mode 100644 index 00000000..61148cdf --- /dev/null +++ b/tests/tools/libkcov/main.py @@ -0,0 +1,196 @@ +"""A custom simplified implementation of the test runner. + +This is necessary, since `unittest.main` does not support correctly how kcov +tests are imported. It will also make it easy to add additional features. + +Most of the test runner code has been copied from the `unittest.main` module. +""" + +import argparse +import importlib +import os +import os.path +import platform +import sys +import unittest +from collections import namedtuple + +# Copied from unittest.main. +_NO_TESTS_EXITCODE = 5 + + +class TestLoader: + """A simplified test loader. + + The implementation assumes that each test case runs only one test method: + `runTest`. + """ + + def __init__(self, config): + self.tests = [] + self.config = config + + def add_tests_from_module(self, name): + """Add all test cases from the named module.""" + + module = importlib.import_module(name) + for name in dir(module): + obj = getattr(module, name) + if ( + isinstance(obj, type) + and issubclass(obj, unittest.TestCase) + and obj not in (unittest.TestCase, unittest.FunctionTestCase) + and hasattr(obj, "runTest") + ): + cfg = self.config + + # TODO: add support for filtering. + + test = obj(cfg.kcov, cfg.outbase, cfg.testbuild, cfg.sources) + self.tests.append(test) + + +Config = namedtuple("Config", ["kcov", "outbase", "testbuild", "sources"]) + + +def fatal(msg): + sys.stderr.write(f"error: {msg}\n") + sys.stderr.flush() + os._exit(1) + + +def normalized_not_empty(path): + """Normalize path, also ensuring that it is not empty.""" + + if len(path) == 0: + raise ValueError("path must be not empty") + + path = os.path.normpath(path) + return path + + +def addTests(config): + """Add all the kcov test modules. + + Discovery is not possible, since some modules need to be excluded, + depending on the os and arch. + """ + + test_loader = TestLoader(config) + + test_loader.add_tests_from_module("test_basic") + test_loader.add_tests_from_module("test_compiled_basic") + + if platform.machine() in ["x86_64", "i386", "i686"]: + test_loader.add_tests_from_module("test_compiled") + if sys.platform.startswith("linux"): + test_loader.add_tests_from_module("test_bash_linux_only") + + if platform.machine() in ["x86_64", "i386", "i686"]: + test_loader.add_tests_from_module("test_system_mode") + + test_loader.add_tests_from_module("test_accumulate") + test_loader.add_tests_from_module("test_bash") + test_loader.add_tests_from_module("test_filter") + test_loader.add_tests_from_module("test_python") + + return test_loader.tests + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("kcov", type=normalized_not_empty) + parser.add_argument("outbase", type=normalized_not_empty) + parser.add_argument("testbuild", type=normalized_not_empty) + parser.add_argument("sources", type=normalized_not_empty) + + # Code copied from unittest.main. + parser.add_argument( + "-v", + "--verbose", + dest="verbosity", + action="store_const", + const=2, + help="Verbose output", + ) + parser.add_argument( + "-q", + "--quiet", + dest="verbosity", + action="store_const", + const=0, + help="Quiet output", + ) + parser.add_argument( + "-f", + "--failfast", + dest="failfast", + action="store_true", + help="Stop on first fail or error", + ) + parser.add_argument( + "-b", + "--buffer", + dest="buffer", + action="store_true", + help="Buffer stdout and stderr during tests", + ) + parser.add_argument( + "-c", + "--catch", + dest="catchbreak", + action="store_true", + help="Catch Ctrl-C and display results so far", + ) + parser.add_argument( + "-k", + dest="patterns", + action="append", + help="Only run tests which match the given substring", + ) + parser.add_argument( + "--locals", + dest="tb_locals", + action="store_true", + help="Show local variables in tracebacks", + ) + + # TODO: The --duration argument was added in 3.12. + + return parser.parse_args() + + +def main(): + # Parse the command line and validate the configuration paths + args = parse_args() + + if os.path.abspath(args.outbase) == os.getcwd(): + fatal("'outbase' cannot be the current directory") + + # Loads and configure tests + config = Config(args.kcov, args.outbase, args.testbuild, args.sources) + tests = addTests(config) + + # Run the tests + test_suite = unittest.TestSuite(tests) + + # Code copied from unittest.main. + if args.catchbreak: + unittest.installHandler() + + test_runner = unittest.TextTestRunner( + verbosity=args.verbosity, + failfast=args.failfast, + buffer=args.buffer, + tb_locals=args.tb_locals, + ) + + result = test_runner.run(test_suite) + if True: + if result.testsRun == 0 and len(result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + elif result.wasSuccessful(): + sys.exit(0) + else: + sys.exit(1) diff --git a/tests/tools/parse_cobertura b/tests/tools/parse_cobertura new file mode 100755 index 00000000..3ff642f1 --- /dev/null +++ b/tests/tools/parse_cobertura @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import sys + +from libkcov import cobertura + +if len(sys.argv) < 4: + print("Usage: lookup-class-line ") + sys.exit(1) + +fileName = sys.argv[2] +line = int(sys.argv[3]) + +data = cobertura.readFile(sys.argv[1]) + +dom = cobertura.parse(data) +fileTag = cobertura.lookupClassName(dom, fileName) + +if fileTag is not None: + hits = cobertura.lookupHitsByLine(fileTag, line) + if hits is not None: + print(hits) + sys.exit(0) + +print("nocode") diff --git a/tests/tools/run-tests b/tests/tools/run-tests deleted file mode 100755 index d8cbfd6a..00000000 --- a/tests/tools/run-tests +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -import platform -import sys -import unittest - -import testbase - -# The actual tests -from basic import * -from compiled_basic import * - -if platform.machine() in ["x86_64", "i386", "i686"]: - from compiled import * -if sys.platform.startswith("linux"): - from bash_linux_only import * - - if platform.machine() in ["x86_64", "i386", "i686"]: - from system_mode import * -from accumulate import * -from bash import * -from filter import * -from python import * - -if __name__ == "__main__": - if len(sys.argv) < 5: - print( - "Usage: run-tests " - ) - sys.exit(1) - - testbase.configure(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) - - sys.argv = sys.argv[4:] - - unittest.main() diff --git a/tests/tools/accumulate.py b/tests/tools/test_accumulate.py similarity index 98% rename from tests/tools/accumulate.py rename to tests/tools/test_accumulate.py index cec2681d..5a9879f5 100644 --- a/tests/tools/accumulate.py +++ b/tests/tools/test_accumulate.py @@ -1,5 +1,5 @@ -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class accumulate_data(testbase.KcovTestCase): diff --git a/tests/tools/bash.py b/tests/tools/test_bash.py similarity index 99% rename from tests/tools/bash.py rename to tests/tools/test_bash.py index 9cd44024..7733ac73 100644 --- a/tests/tools/bash.py +++ b/tests/tools/test_bash.py @@ -3,8 +3,8 @@ import sys import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class BashBase(testbase.KcovTestCase): diff --git a/tests/tools/bash_linux_only.py b/tests/tools/test_bash_linux_only.py similarity index 97% rename from tests/tools/bash_linux_only.py rename to tests/tools/test_bash_linux_only.py index 3fe9963f..ceb00890 100644 --- a/tests/tools/bash_linux_only.py +++ b/tests/tools/test_bash_linux_only.py @@ -1,5 +1,5 @@ -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class bash_sh_shebang(testbase.KcovTestCase): diff --git a/tests/tools/basic.py b/tests/tools/test_basic.py similarity index 96% rename from tests/tools/basic.py rename to tests/tools/test_basic.py index 9a2bbca7..cbb6c6d7 100644 --- a/tests/tools/basic.py +++ b/tests/tools/test_basic.py @@ -1,8 +1,8 @@ import os import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class too_few_arguments(testbase.KcovTestCase): diff --git a/tests/tools/compiled.py b/tests/tools/test_compiled.py similarity index 99% rename from tests/tools/compiled.py rename to tests/tools/test_compiled.py index ca52d098..2fca03fd 100644 --- a/tests/tools/compiled.py +++ b/tests/tools/test_compiled.py @@ -3,8 +3,8 @@ import sys import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class illegal_insn(testbase.KcovTestCase): diff --git a/tests/tools/compiled_basic.py b/tests/tools/test_compiled_basic.py similarity index 98% rename from tests/tools/compiled_basic.py rename to tests/tools/test_compiled_basic.py index 849129a1..0ff7c440 100644 --- a/tests/tools/compiled_basic.py +++ b/tests/tools/test_compiled_basic.py @@ -1,8 +1,8 @@ import sys import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class shared_library(testbase.KcovTestCase): diff --git a/tests/tools/filter.py b/tests/tools/test_filter.py similarity index 98% rename from tests/tools/filter.py rename to tests/tools/test_filter.py index 1ceed3fa..3e270360 100644 --- a/tests/tools/filter.py +++ b/tests/tools/test_filter.py @@ -1,5 +1,5 @@ -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class include_exclude_pattern(testbase.KcovTestCase): diff --git a/tests/tools/python.py b/tests/tools/test_python.py similarity index 99% rename from tests/tools/python.py rename to tests/tools/test_python.py index 969e584c..b3056666 100644 --- a/tests/tools/python.py +++ b/tests/tools/test_python.py @@ -1,7 +1,7 @@ import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class python_exit_status(testbase.KcovTestCase): diff --git a/tests/tools/system_mode.py b/tests/tools/test_system_mode.py similarity index 97% rename from tests/tools/system_mode.py rename to tests/tools/test_system_mode.py index 70629919..e3fae421 100644 --- a/tests/tools/system_mode.py +++ b/tests/tools/test_system_mode.py @@ -3,8 +3,8 @@ import time import unittest -import cobertura -import testbase +import libkcov as testbase +from libkcov import cobertura class SystemModeBase(testbase.KcovTestCase):