Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

common: don't blanket-ignore sd-executor coredumps #668

Merged
merged 4 commits into from
Oct 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions agent/testsuite-alt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ if ! coredumpctl_init; then
exit 1
fi

# Collect any coredumps that happened during boot
exectask "coredumpctl_collect_boot" "coredumpctl_collect"

centos_ensure_qemu_symlink

# Since we don't install the built revision in this case, let's copy some files
Expand Down Expand Up @@ -80,6 +83,9 @@ exectask "ninja-test_sanitizers_$(uname -m)" "meson test -C $BUILD_DIR --print-e
exectask "check-meson-logs-for-sanitizer-errors" "cat $BUILD_DIR/meson-logs/testlog*.txt | check_for_sanitizer_errors"
[[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR"

# Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes
coredumpctl_set_ts
Fixed Show fixed Hide fixed

export QEMU_BIN="/bin/qemu-system-$(uname -m)"
# Set timeouts for QEMU and nspawn tests to kill them in case they get stuck
export QEMU_TIMEOUT=1200
Expand Down
6 changes: 6 additions & 0 deletions agent/testsuite.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ if ! coredumpctl_init; then
exit 1
fi

# Collect any coredumps that happened during boot
exectask "coredumpctl_collect_boot" "coredumpctl_collect"

centos_ensure_qemu_symlink

set +e
Expand All @@ -53,6 +56,9 @@ if ! git diff --quiet "$MAIN_BRANCH" HEAD && ! git diff "$(git merge-base "$MAIN
finish_and_exit
fi

# Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes
coredumpctl_set_ts
Fixed Show fixed Hide fixed

## Integration test suite ##
CHECK_LIST=()
FLAKE_LIST=(
Expand Down
21 changes: 11 additions & 10 deletions common/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ set -u
__COREDUMPCTL_TS=""
# Keep a map of test-specific excludes to avoid code duplication
declare -Arx COREDUMPCTL_EXCLUDE_MAP=(
# test-execute - certain subtests die with SIGSEGV intentionally
# systemd-executor - there's a couple of instances where test-execute blocks exec
# syscalls through a seccomp filter which then kills the test
# with SIGSYS, and with systemd/systemd#27890 the coredump
# executable will be set to the new systemd-executor binary
# python3.x - one of the test-execute subtests triggers SIGSYS in python3.x
# (since systemd/systemd#16675)
["test/TEST-02-UNITTESTS"]="/(bin/python3.[0-9]+|platform-python3.[0-9]+|systemd-executor|test-execute)$"
["test/TEST-17-UDEV"]="/(sleep|udevadm)$"
["test/TEST-59-RELOADING-RESTART"]="/(sleep|bash|systemd-notify)$"
)
Expand Down Expand Up @@ -297,7 +305,7 @@ coredumpctl_init() {
#
# $1: timestamp to set. If empty, the current date & time is used instead
coredumpctl_set_ts() {
__COREDUMPCTL_TS="${1:-$(date +"%Y-%m-%d %H:%M:%S")}"
__COREDUMPCTL_TS="$(date +"%Y-%m-%d %H:%M:%S")"
}

# Attempt to dump info about relevant coredumps using the coredumpctl utility.
Expand Down Expand Up @@ -350,20 +358,13 @@ coredumpctl_collect() {
# The filter can be overridden using the $COREDUMPCTL_EXCLUDE_RX env variable.
# See also the $COREDUMPCTL_EXCLUDE_MAP at the beginning of this file.
# EXCLUDE_RX:
# test-execute - certain subtests die with SIGSEGV intentionally
# dhcpcd - [temporary] keeps crashing intermittently with SIGABRT, needs
# further investigation
# python3.x - one of the test-execute subtests triggers SIGSYS in python3.x
# (since systemd/systemd#16675)
# sleep/bash - intentional SIGABRT caused by TEST-57
# systemd-notify - intermittent (and intentional) SIGABRT caused by TEST-59
# auditd - bug in C8S
# test(-usr)?-dump - intentional coredumps from systemd-coredump tests in TEST-74
# systemd-executor - there's a couple of instances where test-execute blocks exec
# syscalls through a seccomp filter which then kills the test
# with SIGSYS, and with systemd/systemd#27890 the coredump
# executable will be set to the new systemd-executor binary
local exclude_rx="${COREDUMPCTL_EXCLUDE_RX:-/(test-execute|dhcpcd|bin/python3.[0-9]+|platform-python3.[0-9]+|bash|sleep|systemd-notify|auditd|test(-usr)?-dump|systemd-executor)$}"
local exclude_rx="${COREDUMPCTL_EXCLUDE_RX:-/(dhcpcd|bash|sleep|systemd-notify|auditd|test(-usr)?-dump)$}"
_log "Excluding coredumps matching '$exclude_rx'"
if ! "$coredumpctl_bin" "${args[@]}" -F COREDUMP_EXE | grep -Ev "$exclude_rx" > "$tempfile"; then
_log "No relevant coredumps found"
Expand All @@ -380,7 +381,7 @@ coredumpctl_collect() {
# trace and other useful info
while read -r path; do
local exe
local gdb_cmd="bt full"
local gdb_cmd="set print pretty on\nbt full"

_log "Collecting coredumps for '$path'"
"$coredumpctl_bin" "${args[@]}" info "$path"
Expand Down
6 changes: 6 additions & 0 deletions vagrant/test_scripts/test-arch-coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ if ! coredumpctl_init; then
exit 1
fi

# Collect any coredumps that happened during boot
exectask "coredumpctl_collect_boot" "coredumpctl_collect"

# Disable swap, since it seems to cause CPU soft lock-ups in some cases
swapoff -av

Expand All @@ -49,6 +52,9 @@ exectask "ninja-test" "GCOV_ERROR_FILE=$LOGDIR/ninja-test-gcov-errors.log meson
exectask "lcov_collect-build_dir-ninja-test" "lcov_collect $COVERAGE_DIR/unit-tests.coverage-info $BUILD_DIR"
[[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR"

# Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes
coredumpctl_set_ts
Fixed Show fixed Hide fixed

## Integration test suite ##
# Prepare a custom-tailored initrd image (with the systemd module included).
# This is necessary, as the default mkinitcpio config includes only the udev module,
Expand Down
12 changes: 12 additions & 0 deletions vagrant/test_scripts/test-arch-sanitizers-clang.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@ export COREDUMPCTL_BIN="$BUILD_DIR/coredumpctl"

bootctl status

# Enable systemd-coredump
if ! coredumpctl_init; then
echo >&2 "Failed to configure systemd-coredump/coredumpctl"
exit 1
fi

# Collect any coredumps that happened during boot
exectask "coredumpctl_collect_boot" "coredumpctl_collect"

pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; }

## Sanitizer-specific options
Expand All @@ -43,6 +52,9 @@ exectask "ninja-test_sanitizers" "meson test -C $BUILD_DIR --print-errorlogs --t
exectask "check-meson-logs-for-sanitizer-errors" "cat $BUILD_DIR/meson-logs/testlog*.txt | check_for_sanitizer_errors"
[[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR"

# Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes
coredumpctl_set_ts
Fixed Show fixed Hide fixed

## Run TEST-01-BASIC under sanitizers
# Prepare a custom-tailored initrd image (with the systemd module included).
# This is necessary, as the default mkinitcpio config includes only the udev module,
Expand Down
6 changes: 6 additions & 0 deletions vagrant/test_scripts/test-arch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ if ! coredumpctl_init; then
exit 1
fi

# Collect any coredumps that happened during boot
exectask "coredumpctl_collect_boot" "coredumpctl_collect"

# Disable swap, since it seems to cause CPU soft lock-ups in some cases
swapoff -av

Expand All @@ -37,6 +40,9 @@ pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; }
exectask "ninja-test" "meson test -C $BUILD_DIR --print-errorlogs --timeout-multiplier=3"
[[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR"

# Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes
coredumpctl_set_ts
Fixed Show fixed Hide fixed

## Integration test suite ##
# Prepare a custom-tailored initrd image (with the systemd module included).
# This is necessary, as the default mkinitcpio config includes only the udev module,
Expand Down