diff --git a/.bazelrc b/.bazelrc index 2a35375ed5..69c3038ef1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -22,6 +22,8 @@ common --http_timeout_scaling=2.0 # directory is often deleted while the ANDROID_HOME environment variable remains set. common --action_env=ANDROID_HOME="" +common --cxxopt=-std=c++17 +build --cxxopt=-std=c++17 build --java_language_version=17 build --tool_java_language_version=17 build --java_runtime_version=remotejdk_17 @@ -72,6 +74,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce test:x86_64-linux --//config:integration_mode=docker test:x86_64-linux --//config:unit_mode=host +# Show a failing test's log (incl. the crash-dump banner) in the console. +test:x86_64-linux --test_output=errors # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!) build:arm64-linux --config=stub @@ -181,3 +185,12 @@ test:tsan --test_tag_filters=-no-tsan test:tsan --build_tests_only test:tsan --cxxopt=-Wno-maybe-uninitialized test:tsan --cxxopt=-Wno-redundant-move + +# Build variant. --config=release is the default (slim image, no debug support) +# — equivalent to specifying no variant. --config=debug adds gdb to the +# test image so cores can be analysed in-container and a backtrace is auto-captured. +# -c dbg is implied so the backtrace carries file/line symbols. +build:release --//config:debug=False +build:debug --compilation_mode=dbg +test:debug --test_env=SCORE_ENABLE_DEBUG=1 +test:debug --//config:debug=True diff --git a/.gitignore b/.gitignore index f3a58c2f51..5f3da465e0 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,10 @@ target/ tests/**/*.html tests/**/*.xml +# Backup of the kernel core_pattern saved by the integration test core-dump +# capture (--config=core_dump); auto-removed after restore +/.original_core_pattern + # IDE Code files *.orig .venv_docs diff --git a/MODULE.bazel b/MODULE.bazel index 44cb0f6481..1b46ebd726 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5") bazel_dep(name = "rules_rust", version = "0.68.2-score") bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_oci", version = "2.3.0") +bazel_dep(name = "rules_distroless", version = "0.8.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "aspect_rules_lint", version = "2.3.0") bazel_dep(name = "buildifier_prebuilt", version = "8.5.1") @@ -175,6 +176,20 @@ oci.pull( ) use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64") +# gdb (+ dependency closure) layered onto the test image; see +# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile +# with: bazel run @gdb_apt//:lock +apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "gdb_apt", + lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json", + manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml", + # Normalize to a merged-usr layout so the layer does not clobber the base + # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh). + mergedusr = True, +) +use_repo(apt, "gdb_apt") + bazel_dep(name = "score_baselibs", version = "0.2.10") # Hedron's Compile Commands Extractor for Bazel diff --git a/config/BUILD b/config/BUILD index 1b432cad5a..74444fb6c4 100644 --- a/config/BUILD +++ b/config/BUILD @@ -50,6 +50,21 @@ config_setting( }, ) +# Build variant. --config=release is the default (slim image); --config=debug adds +# the gdb layer to the test image so core dumps can be analysed in-container (see +# .bazelrc, tests/utils/environments/x86_64-linux/BUILD). +bool_flag( + name = "debug", + build_setting_default = False, +) + +config_setting( + name = "debug_enabled", + flag_values = { + ":debug": "True", + }, +) + # How to run unit tests: # # - qemu: in a QEMU virtual machine diff --git a/tests/integration/readme.md b/tests/integration/readme.md index 92d514dd3b..1e8c04fac9 100644 --- a/tests/integration/readme.md +++ b/tests/integration/readme.md @@ -23,3 +23,128 @@ Currently the following configs are supported: - `host` - `x86_64-linux` +## Debug environment + +The debug environment is **opt-in**: add `--config=debug` to include it. +It adds gdb to the docker image, builds the code in debug variant (using "-c dbg") and if process is crashing, a core dump is created. Furthermore the backtrace of the thread which provoked the crash is printed to the console. + +The +default (`--config=release`, equivalent to specifying no variant) builds the slim +image without it. + +**Attention: It is needed to influence the kernel `core_pattern` value of +your host system to have core dump support!** + + +How it works: +- `--config=debug` forwards `SCORE_ENABLE_DEBUG=1` into the test + environment and sets the `//config:debug` build flag (see `.bazelrc`); the + shared pytest plugin keys off the env var, individual tests need no adaptions. +- The build flag adds a `gdb` layer to the test image (`score_itf_examples`) so + cores can be analysed inside the container against matching libraries. Normal + runs build the image without `gdb`, unchanged. +- The sandbox container runs privileged with an unlimited core-file `ulimit` and + a read-write bind-mount of the workspace root. +- A shared fixture sets the kernel `core_pattern` to a sandbox-local path + (`/tmp/score_cores/core.%e.%p.%s.%t`). On teardown it symbolizes each core + **inside the container** (where the binary and matching libraries live) into a + `.bt.txt` backtrace, copies the cores and backtraces into the Bazel test + outputs, and restores the original `core_pattern`. +- Before changing `core_pattern`, the fixture mirrors the original value to + `.original_core_pattern` in the workspace root. The sandboxed test process sees + the source tree read-only, so this file is written from inside the privileged + container via the workspace bind-mount (hence it is root-owned). It is removed + again once the value is restored, so it exists only if a run is force-killed. + +Further technical limitations are described in [Important: the `core_pattern` is a global kernel setting](#important-the-core_pattern-is-a-global-kernel-setting) + +### Getting a crash dump + +Run the (crashing) test with `--config=debug`, disabling the cache so it +actually executes: +``` +bazel test //tests/integration/ --config=x86_64-linux --config=debug --nocache_test_results +``` + +If a crash dump was created, a `CRASH DUMP` section is printed right under the +pytest `FAILURES` section at the end of the run (the `x86_64-linux` config +enables `--test_output=errors`, so the failing log is shown automatically): +``` +=================================== FAILURES =================================== +... +================================== CRASH DUMP ================================== +CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details. + +core.launch_manager.42.6.1787209649: + Program terminated with signal SIGABRT, Aborted. + #0 0x... in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + #1 0x... in raise () from /lib/x86_64-linux-gnu/libc.so.6 + #2 0x... in abort () from /lib/x86_64-linux-gnu/libc.so.6 + #3 0x... in at : + ... + Full backtrace (all threads): <.../cores/core.launch_manager.*.bt.txt> + Reopen in gdb inside the test image: + docker run --rm -it -v <.../launch_manager>:/tmp/.../launch_manager:ro -v <.../cores>:/cores:ro score_itf_examples:latest gdb /tmp/.../launch_manager /cores/core.launch_manager.* +=========================== short test summary info ============================ +``` +The crashing thread's stack is printed **inline** (symbolized inside the +container, so libraries match). The printed paths are absolute and +copy-pasteable. Core files are named `core....