test: exercise the clar itself

While we have had the "clar_test" binary for a very long time already,
it was more for demonstration purposes than for real testing of the
clar. It could serve as sort of a smoke test if the developer remembered
to execute it, but that's about it.

In the preceding commits we have moved the "clar_test" into a separate
directory, renamed it to "selftest" and made its output deterministic.
We can thus now wire up proper testing of the clar by exercising the
"selftest" binary via a new "clar_test". These tests run the "selftest"
binary with various different arguments and check both its return code
as well as its output.

test/CMakeLists.txt

@@ -1 +1,63 @@
 add_subdirectory(selftest)
+
+find_package(Python COMPONENTS Interpreter REQUIRED)
+
+add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/clar.suite"
+	COMMAND "${Python_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/generate.py" --output "${CMAKE_CURRENT_BINARY_DIR}"
+	DEPENDS main.c clar.c
+	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+)
+
+add_executable(clar_test)
+set_target_properties(clar_test PROPERTIES
+	C_STANDARD 90
+	C_STANDARD_REQUIRED ON
+	C_EXTENSIONS OFF
+)
+
+# MSVC generates all kinds of warnings. We may want to fix these in the future
+# and then unconditionally treat warnings as errors.
+if (NOT MSVC)
+	set_target_properties(clar_test PROPERTIES
+		COMPILE_WARNING_AS_ERROR ON
+	)
+endif()
+
+target_sources(clar_test PRIVATE
+	main.c
+	clar.c
+	"${CMAKE_CURRENT_BINARY_DIR}/clar.suite"
+)
+target_compile_definitions(clar_test PRIVATE
+	CLAR_FIXTURE_PATH="${CMAKE_CURRENT_SOURCE_DIR}/expected/"
+	SELFTEST_BINARY_PATH="${CMAKE_CURRENT_BINARY_DIR}/selftest/selftest"
+	_DARWIN_C_SOURCE
+	_POSIX_C_SOURCE=200809L
+)
+target_compile_options(clar_test PRIVATE
+	$<IF:$<CXX_COMPILER_ID:MSVC>,/W4,-Wall>
+)
+target_include_directories(clar_test PRIVATE
+	"${CMAKE_SOURCE_DIR}"
+	"${CMAKE_CURRENT_BINARY_DIR}"
+)
+target_link_libraries(clar_test clar)
+
+add_test(build_selftest
+  "${CMAKE_COMMAND}"
+  --build "${CMAKE_BINARY_DIR}"
+  --config "$<CONFIG>"
+  --target selftest
+)
+set_tests_properties(build_selftest PROPERTIES FIXTURES_SETUP clar_test_fixture)
+
+add_test(build_test
+  "${CMAKE_COMMAND}"
+  --build "${CMAKE_BINARY_DIR}"
+  --config "$<CONFIG>"
+  --target clar_test
+)
+set_tests_properties(build_test PROPERTIES FIXTURES_SETUP clar_test_fixture)
+
+add_test(clar_test "${CMAKE_CURRENT_BINARY_DIR}/clar_test")
+set_tests_properties(clar_test PROPERTIES FIXTURES_REQUIRED clar_test_fixture)

test/clar.c

@@ -0,0 +1,165 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include "clar.h"
+
+static char *read_full(int fd)
+{
+	size_t data_bytes = 0;
+	char *data = NULL;
+
+	while (1) {
+		char buf[4096];
+		ssize_t n;
+
+		n = read(fd, buf, sizeof(buf));
+		if (n < 0) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
+			cl_fail("Failed reading from child process.");
+		}
+		if (!n)
+			break;
+
+		data = realloc(data, data_bytes + n);
+		cl_assert(data);
+
+		memcpy(data + data_bytes, buf, n);
+		data_bytes += n;
+	}
+
+	data = realloc(data, data_bytes + 1);
+	cl_assert(data);
+	data[data_bytes] = '\0';
+
+	return data;
+}
+
+static char *read_file(const char *path)
+{
+	char *data;
+	int fd;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		cl_fail("Failed reading expected file.");
+
+	data = read_full(fd);
+	cl_must_pass(close(fd));
+
+	return data;
+}
+
+static void run(const char *expected_output_file, int expected_error_code, ...)
+{
+	const char *argv[16];
+	int pipe_fds[2];
+	va_list ap;
+	pid_t pid;
+	int i;
+
+	va_start(ap, expected_error_code);
+	argv[0] = "selftest";
+	for (i = 1; ; i++) {
+		cl_assert(i < sizeof(argv) / sizeof(*argv));
+
+		argv[i] = va_arg(ap, const char *);
+		if (!argv[i])
+			break;
+	}
+	va_end(ap);
+
+	cl_must_pass(pipe(pipe_fds));
+
+	pid = fork();
+	if (!pid) {
+		if (dup2(pipe_fds[1], STDOUT_FILENO) < 0 ||
+		    dup2(pipe_fds[1], STDERR_FILENO) < 0 ||
+		    close(0) < 0 ||
+		    close(pipe_fds[0]) < 0 ||
+		    close(pipe_fds[1]) < 0)
+			exit(1);
+
+		execv(SELFTEST_BINARY_PATH, (char **) argv);
+		exit(1);
+	} else if (pid > 0) {
+		pid_t waited_pid;
+		char *expected_output, *output;
+		int stat;
+
+		cl_must_pass(close(pipe_fds[1]));
+
+		output = read_full(pipe_fds[0]);
+
+		waited_pid = waitpid(pid, &stat, 0);
+		cl_assert_equal_i(pid, waited_pid);
+		cl_assert(WIFEXITED(stat));
+		cl_assert_equal_i(WEXITSTATUS(stat), expected_error_code);
+
+		expected_output = read_file(cl_fixture(expected_output_file));
+		cl_assert_equal_s(output, expected_output);
+
+		free(expected_output);
+		free(output);
+	} else {
+		cl_fail("Fork failed.");
+	}
+}
+
+void test_clar__help(void)
+{
+	run("help", 255, "-h", NULL);
+}
+
+void test_clar__without_arguments(void)
+{
+	run("without_arguments", 8, NULL);
+}
+
+void test_clar__specific_test(void)
+{
+	run("specific_test", 1, "-sselftest::bool", NULL);
+}
+
+void test_clar__stop_on_failure(void)
+{
+	run("stop_on_failure", 1, "-Q", NULL);
+}
+
+void test_clar__quiet(void)
+{
+	run("quiet", 8, "-q", NULL);
+}
+
+void test_clar__tap(void)
+{
+	run("tap", 8, "-t", NULL);
+}
+
+void test_clar__suite_names(void)
+{
+	run("suite_names", 0, "-l", NULL);
+}
+
+void test_clar__summary_without_filename(void)
+{
+	struct stat st;
+	run("summary_without_filename", 8, "-r", NULL);
+	/* The summary contains timestamps, so we cannot verify its contents. */
+	cl_must_pass(stat("summary.xml", &st));
+}
+
+void test_clar__summary_with_filename(void)
+{
+	struct stat st;
+	run("summary_with_filename", 8, "-rdifferent.xml", NULL);
+	/* The summary contains timestamps, so we cannot verify its contents. */
+	cl_must_pass(stat("different.xml", &st));
+}

test/expected/help

@@ -0,0 +1,12 @@
+Usage: selftest [options]
+
+Options:
+  -sname        Run only the suite with `name` (can go to individual test name)
+  -iname        Include the suite with `name`
+  -xname        Exclude the suite with `name`
+  -v            Increase verbosity (show suite names)
+  -q            Only report tests that had an error
+  -Q            Quit as soon as a test fails
+  -t            Display results in tap format
+  -l            Print suite names
+  -r[filename]  Write summary file (to the optional filename)

test/expected/quiet

@@ -0,0 +1,80 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+  1) Failure:
+selftest::1 [file:42]
+  Function call failed: -1
+
+  1) Failure:
+selftest::2 [file:42]
+  Expression is not true: 100 == 101
+
+  1) Failure:
+selftest::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  1) Failure:
+selftest::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  1) Failure:
+selftest::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  1) Failure:
+selftest::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  1) Failure:
+selftest::bool [file:42]
+  0 != value
+  0 != 1
+
+  1) Failure:
+selftest::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+
+
+
+  1) Failure:
+selftest::1 [file:42]
+  Function call failed: -1
+
+  2) Failure:
+selftest::2 [file:42]
+  Expression is not true: 100 == 101
+
+  3) Failure:
+selftest::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  4) Failure:
+selftest::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  5) Failure:
+selftest::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  6) Failure:
+selftest::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  7) Failure:
+selftest::bool [file:42]
+  0 != value
+  0 != 1
+
+  8) Failure:
+selftest::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+

test/expected/specific_test

@@ -0,0 +1,9 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+F
+
+  1) Failure:
+selftest::bool [file:42]
+  0 != value
+  0 != 1
+

test/expected/stop_on_failure

@@ -0,0 +1,8 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+F
+
+  1) Failure:
+selftest::1 [file:42]
+  Function call failed: -1
+

test/expected/suite_names

@@ -0,0 +1,2 @@
+Test suites (use -s<name> to run just one):
+   0: selftest

test/expected/summary.xml

@@ -0,0 +1,36 @@
+<testsuites>
+	<testsuite id="0" name="selftest" hostname="localhost" timestamp="2024-09-06T10:04:08" tests="8" failures="8" errors="0">
+		<testcase name="1" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Function call failed: -1
+(null)]]></failure>
+		</testcase>
+		<testcase name="2" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Expression is not true: 100 == 101
+(null)]]></failure>
+		</testcase>
+		<testcase name="strings" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[String mismatch: "mismatched" != actual ("this one fails")
+'mismatched' != 'expected' (at byte 0)]]></failure>
+		</testcase>
+		<testcase name="strings_with_length" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[String mismatch: "exactly" != actual ("this one fails")
+'exa' != 'exp' (at byte 2)]]></failure>
+		</testcase>
+		<testcase name="int" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[101 != value ("extra note on failing test")
+101 != 100]]></failure>
+		</testcase>
+		<testcase name="int_fmt" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[022 != value
+0022 != 0144]]></failure>
+		</testcase>
+		<testcase name="bool" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[0 != value
+0 != 1]]></failure>
+		</testcase>
+		<testcase name="ptr" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Pointer mismatch: p1 != p2
+0x1 != 0x2]]></failure>
+		</testcase>
+	</testsuite>
+</testsuites>