From a362b862ac4eb1a734f6bc257b7a2df74ab6dd29 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Tue, 28 Jul 2026 16:04:47 +0200 Subject: [PATCH 1/4] Execute gdb in docker Provide core dump support for crashing processes and print backtrace of thread provoking the crash on console. --- .bazelrc | 10 + .gitignore | 4 + MODULE.bazel | 15 + config/BUILD | 23 + tests/integration/readme.md | 117 +++ tests/utils/bazel/integration.bzl | 17 +- tests/utils/environments/x86_64-linux/BUILD | 25 + .../x86_64-linux/gdb_apt.lock.json | 865 ++++++++++++++++++ .../environments/x86_64-linux/gdb_apt.yaml | 24 + tests/utils/plugins/integration.py | 354 +++++++ 10 files changed, 1452 insertions(+), 2 deletions(-) create mode 100644 tests/utils/environments/x86_64-linux/gdb_apt.lock.json create mode 100644 tests/utils/environments/x86_64-linux/gdb_apt.yaml diff --git a/.bazelrc b/.bazelrc index 2a35375ed5..7452c45711 100644 --- a/.bazelrc +++ b/.bazelrc @@ -72,6 +72,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce test:x86_64-linux --//config:integration_mode=docker test:x86_64-linux --//config:unit_mode=host +# Show a failing test's log (incl. the crash-dump banner) in the console. +test:x86_64-linux --test_output=errors # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!) build:arm64-linux --config=stub @@ -181,3 +183,11 @@ test:tsan --test_tag_filters=-no-tsan test:tsan --build_tests_only test:tsan --cxxopt=-Wno-maybe-uninitialized test:tsan --cxxopt=-Wno-redundant-move + +# Core-dump capture — opt in with --config=core_dump. The env var gates the +# plugin at runtime; the build flag selects the gdb-equipped debug image so cores +# can be analysed in-container and a backtrace is auto-captured. -c dbg is implied +# so the auto-captured backtrace carries file/line symbols. +build:core_dump --compilation_mode=dbg +test:core_dump --test_env=SCORE_ENABLE_CORE_DUMP=1 +test:core_dump --//config:core_dump=True diff --git a/.gitignore b/.gitignore index f3a58c2f51..5f3da465e0 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,10 @@ target/ tests/**/*.html tests/**/*.xml +# Backup of the kernel core_pattern saved by the integration test core-dump +# capture (--config=core_dump); auto-removed after restore +/.original_core_pattern + # IDE Code files *.orig .venv_docs diff --git a/MODULE.bazel b/MODULE.bazel index 44cb0f6481..7a42c2ecff 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5") bazel_dep(name = "rules_rust", version = "0.68.2-score") bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_oci", version = "2.3.0") +bazel_dep(name = "rules_distroless", version = "0.8.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "aspect_rules_lint", version = "2.3.0") bazel_dep(name = "buildifier_prebuilt", version = "8.5.1") @@ -175,6 +176,20 @@ oci.pull( ) use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64") +# gdb (+ dependency closure) layered onto the debug test image; see +# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile +# with: bazel run @gdb_apt//:lock +apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "gdb_apt", + lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json", + manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml", + # Normalize to a merged-usr layout so the layer does not clobber the base + # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh). + mergedusr = True, +) +use_repo(apt, "gdb_apt") + bazel_dep(name = "score_baselibs", version = "0.2.10") # Hedron's Compile Commands Extractor for Bazel diff --git a/config/BUILD b/config/BUILD index 1b432cad5a..4d3e197218 100644 --- a/config/BUILD +++ b/config/BUILD @@ -50,6 +50,29 @@ config_setting( }, ) +# Opt in with --config=core_dump: selects the gdb-equipped debug test image so +# core dumps can be analysed in-container (see .bazelrc, tests/integration/readme.md). +bool_flag( + name = "core_dump", + build_setting_default = False, +) + +config_setting( + name = "core_dump_enabled", + flag_values = { + ":core_dump": "True", + }, +) + +# Specialization of :integration_docker; select() prefers it when both match. +config_setting( + name = "integration_docker_core_dump", + flag_values = { + ":integration_mode": "docker", + ":core_dump": "True", + }, +) + # How to run unit tests: # # - qemu: in a QEMU virtual machine diff --git a/tests/integration/readme.md b/tests/integration/readme.md index 92d514dd3b..569e23d881 100644 --- a/tests/integration/readme.md +++ b/tests/integration/readme.md @@ -23,3 +23,120 @@ Currently the following configs are supported: - `host` - `x86_64-linux` +## Crash dumps (core dumps) + +Core-dump capture is **opt-in**: add `--config=core_dump` to include the support. **Attention: This influences the kernel `core_pattern` value of your host system!** + + +How it works: +- `--config=core_dump` forwards `SCORE_ENABLE_CORE_DUMP=1` into the test + environment and sets the `//config:core_dump` build flag (see `.bazelrc`); the + shared pytest plugin keys off the env var, individual tests need no adaptions. +- The build flag selects a **debug image variant** (`score_itf_examples_debug`, + the normal image plus `gdb`) so cores can be analysed inside the container. + Normal runs keep the slim `score_itf_examples` image, unchanged. +- The sandbox container runs privileged with an unlimited core-file `ulimit` and + a read-write bind-mount of the workspace root. +- A shared fixture sets the kernel `core_pattern` to a sandbox-local path + (`/tmp/score_cores/core.%e.%p.%s.%t`). On teardown it symbolizes each core + **inside the container** (where the binary and matching libraries live) into a + `.bt.txt` backtrace, copies the cores and backtraces into the Bazel test + outputs, and restores the original `core_pattern`. +- Before changing `core_pattern`, the fixture mirrors the original value to + `.original_core_pattern` in the workspace root. The sandboxed test process sees + the source tree read-only, so this file is written from inside the privileged + container via the workspace bind-mount (hence it is root-owned). It is removed + again once the value is restored, so it exists only if a run is force-killed. + +Further technical limitations are described in [Important: the `core_pattern` is a global kernel setting](#important-the-core_pattern-is-a-global-kernel-setting) + +### Getting a crash dump + +Run the (crashing) test with `--config=core_dump`, disabling the cache so it +actually executes: +``` +bazel test //tests/integration/ --config=x86_64-linux --config=core_dump --nocache_test_results +``` + +If a crash dump was created, a `CRASH DUMP` section is printed right under the +pytest `FAILURES` section at the end of the run (the `x86_64-linux` config +enables `--test_output=errors`, so the failing log is shown automatically): +``` +=================================== FAILURES =================================== +... +================================== CRASH DUMP ================================== +CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details. + +core.launch_manager.42.6.1787209649: + Program terminated with signal SIGABRT, Aborted. + #0 0x... in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + #1 0x... in raise () from /lib/x86_64-linux-gnu/libc.so.6 + #2 0x... in abort () from /lib/x86_64-linux-gnu/libc.so.6 + #3 0x... in at : + ... + Full backtrace (all threads): <.../cores/core.launch_manager.*.bt.txt> + Reopen in gdb inside the debug image: + docker run --rm -it -v <.../launch_manager>:/tmp/.../launch_manager:ro -v <.../cores>:/cores:ro score_itf_examples_debug:latest gdb /tmp/.../launch_manager /cores/core.launch_manager.* +=========================== short test summary info ============================ +``` +The crashing thread's stack is printed **inline** (symbolized inside the +container, so libraries match). The printed paths are absolute and +copy-pasteable. Core files are named `core....