diff --git a/config/BUILD b/config/BUILD index 1b432cad5a..72ad3ab74f 100644 --- a/config/BUILD +++ b/config/BUILD @@ -12,6 +12,22 @@ # ******************************************************************************* load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") +# QEMU virtual machine used by the integration tests. Override on the command +# line to run the tests against a different image, e.g. +# bazel test --//config:integration_mode=qemu \ +# --//config:qemu_image=//my/package:my_image //tests/... +label_flag( + name = "qemu_image", + build_setting_default = "//tests/utils/environments/x86_64-qnx:qemu_image", + visibility = ["//visibility:public"], +) + +label_flag( + name = "qemu_config", + build_setting_default = "//tests/utils/environments/x86_64-qnx:qemu_config.json", + visibility = ["//visibility:public"], +) + # How to run integration tests: # # - docker: in a Docker container diff --git a/tests/README.md b/tests/README.md index 93bb83b56d..f171d857b7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,7 +30,8 @@ Note the arguments are written so that you can add them to the end of the bazel test command. -| Arg | What It Does | Example Command | -|:-------------------------------|:----------------------------------------------|:------------------------------------------------| -|`--test_arg=--no-local-cleanup` | Integration tests don't cleanup after running |`bazel test //... --test_arg=--no-local-cleanup` | -|`--test_arg=-s` | More logging from the python test framework |`bazel test //... --test_arg=-s` | +| Arg | What It Does | Example Command | +|:-------------------------------|:--------------------------------------------------|:------------------------------------------------| +|`--test_arg=--no-local-cleanup` | Integration tests don't cleanup after running |`bazel test //... --test_arg=--no-local-cleanup` | +|`--test_arg="--test_runner=XYZ"`| Integration tests run with the given test runner |`bazel test //... --test_arg="--test_runner=time` | +|`--test_arg=-s` | More logging from the python test framework |`bazel test //... --test_arg=-s` | diff --git a/tests/integration/crash_ignores_dependents/BUILD b/tests/integration/crash_ignores_dependents/BUILD index 039993802c..b2c44d8e14 100644 --- a/tests/integration/crash_ignores_dependents/BUILD +++ b/tests/integration/crash_ignores_dependents/BUILD @@ -38,7 +38,6 @@ integration_test( name = "crash_ignores_dependents", srcs = ["crash_ignores_dependents.py"], binaries = [ - ":config", ":test_process", ":process_crashing_once", "//score/launch_manager", diff --git a/tests/integration/fallback_to_same_target_restarts/BUILD b/tests/integration/fallback_to_same_target_restarts/BUILD index 9c39ec7713..140d95d8a7 100644 --- a/tests/integration/fallback_to_same_target_restarts/BUILD +++ b/tests/integration/fallback_to_same_target_restarts/BUILD @@ -38,7 +38,6 @@ integration_test( name = "fallback_to_same_target_restarts", srcs = ["fallback_to_same_target_restarts.py"], binaries = [ - ":config", ":control_client_test_driver", ":process_crashing_once", "//score/launch_manager", diff --git a/tests/integration/process_crash_monitoring/BUILD b/tests/integration/process_crash_monitoring/BUILD index 43865f42b1..b42a5cb514 100644 --- a/tests/integration/process_crash_monitoring/BUILD +++ b/tests/integration/process_crash_monitoring/BUILD @@ -38,7 +38,6 @@ integration_test( name = "process_crash_monitoring", srcs = ["process_crash_monitoring.py"], binaries = [ - ":config", ":control_client_test_driver", ":process_crashing_on_runtime", "//score/launch_manager", diff --git a/tests/utils/bazel/integration.bzl b/tests/utils/bazel/integration.bzl index be754b1947..79f43ccb8a 100644 --- a/tests/utils/bazel/integration.bzl +++ b/tests/utils/bazel/integration.bzl @@ -17,6 +17,9 @@ load("@score_lifecycle_pip//:requirements.bzl", "all_requirements") load("//:defs.bzl", "launch_manager_config") load("//tests/utils/bazel:constants.bzl", "SCORE_TEST_INSTALL_PREFIX") +DEFAULT_QEMU_CONFIG = "//config:qemu_config" +DEFAULT_QEMU_IMAGE = "//config:qemu_image" + def integration_test( name, srcs, @@ -24,6 +27,8 @@ def integration_test( files = [], config = None, install_prefix = SCORE_TEST_INSTALL_PREFIX, + qemu_config = DEFAULT_QEMU_CONFIG, + qemu_image = DEFAULT_QEMU_IMAGE, **kwargs): """Creates an integration test. @@ -41,11 +46,20 @@ def integration_test( files: Additional files config: Launch manager configuration file install_prefix: Installation prefix for the test environment + qemu_config: QEMU configuration file used by the QEMU test target. + Defaults to the `//config:qemu_config` label flag. + qemu_image: QEMU image used by the QEMU test target. + Defaults to the `//config:qemu_image` label flag. **kwargs: Miscellaneous arguments passed through to `py_itf_test` """ + bin_pkg_name = "{}_binaries_pkg".format(name) + config_name = "{}_config_pkg".format(name) + test_pkg_name = "{}_test_pkg".format(name) + test_tar_name = "{}_test_tar".format(name) + pkg_files( - name = "binaries", + name = bin_pkg_name, srcs = binaries, attributes = pkg_attributes(mode = "0555"), prefix = "tests/{}".format(name), @@ -53,40 +67,40 @@ def integration_test( if config: launch_manager_config( - name = "config", + name = config_name, config = config, flatbuffer_out_dir = "etc", ) - all_files = files + [":config"] + all_files = files + [":{}".format(config_name)] else: all_files = files pkg_files( - name = "files", + name = test_pkg_name, srcs = all_files, prefix = "tests/{}".format(name), attributes = pkg_attributes(mode = "0400"), ) - pkg_tar(name = "environment", srcs = [":binaries", ":files"]) + pkg_tar(name = test_tar_name, srcs = [":{}".format(bin_pkg_name), ":{}".format(test_pkg_name)]) final_deps = kwargs.pop("deps", []) + all_requirements + [ "@score_tooling//python_basics/score_pytest:attribute_plugin", "//tests/utils/testing_utils", ] - final_data = kwargs.pop("data", []) + [":environment"] + select({ + final_data = kwargs.pop("data", []) + [":{}".format(test_tar_name)] + select({ "//config:integration_docker": [ "//tests/utils/environments/x86_64-linux", ], "//config:integration_qemu": [ - "//tests/utils/environments/x86_64-qnx:qemu_config.json", - "//tests/utils/environments/x86_64-qnx:qemu_image", + qemu_config, + qemu_image, ], "//conditions:default": [], }) final_args = kwargs.pop("args", []) + [ "-p attribute_plugin", - "--score-test-binary-path=$(locations :environment)", + "--score-test-binary-path=$(location :{})".format(test_tar_name), "--score-test-remote-directory={}/tests/{}".format(install_prefix, name), ] + select({ "//config:integration_docker": [ @@ -94,8 +108,8 @@ def integration_test( "--docker-image=score_itf_examples:latest", ], "//config:integration_qemu": [ - "--qemu-config=$(location //tests/utils/environments/x86_64-qnx:qemu_config.json)", - "--qemu-image=$(location //tests/utils/environments/x86_64-qnx:qemu_image)", + "--qemu-config=$(location {})".format(qemu_config), + "--qemu-image=$(location {})".format(qemu_image), ], "//config:integration_host": [ "--local-dir=/tmp/score_itf_host/{}".format(name), diff --git a/tests/utils/environments/x86_64-qnx/BUILD b/tests/utils/environments/x86_64-qnx/BUILD index afd23652c8..276dbe72ee 100644 --- a/tests/utils/environments/x86_64-qnx/BUILD +++ b/tests/utils/environments/x86_64-qnx/BUILD @@ -17,6 +17,7 @@ load("@score_rules_imagefs//rules/qnx:ifs.bzl", "qnx_ifs") exports_files( ["qemu_config.json"], visibility = [ + "//config:__pkg__", "//examples:__subpackages__", "//tests:__subpackages__", ], @@ -34,6 +35,7 @@ qnx_ifs( build_file = ":init.build", target_compatible_with = ["@platforms//os:qnx"], visibility = [ + "//config:__pkg__", "//examples:__subpackages__", "//tests:__subpackages__", ], diff --git a/tests/utils/plugins/integration.py b/tests/utils/plugins/integration.py index 89bf90747d..c79e53ffcd 100644 --- a/tests/utils/plugins/integration.py +++ b/tests/utils/plugins/integration.py @@ -29,6 +29,15 @@ def pytest_addoption(parser): default=None, help="Absolute remote directory path used during test execution.", ) + parser.addoption( + "--test_runner", + action="store", + default=None, + help="""Wrapper program to run the test inside, for example a debugger. + The wrapper program will be called with the test binary as its first + argument, followed by any number of test arguments. It must execute the + test binary with the provided arguments and exit when complete.""", + ) @pytest.fixture @@ -51,3 +60,7 @@ def pytest_configure(config): remote_dir = config.getoption("--score-test-remote-directory", default=None) if remote_dir is not None: os.environ["SCORE_TEST_REMOTE_DIRECTORY"] = remote_dir + + test_runner = config.getoption("--test_runner", default=None) + if test_runner is not None: + os.environ["SCORE_TEST_RUNNER"] = test_runner diff --git a/tests/utils/testing_utils/run_test.py b/tests/utils/testing_utils/run_test.py index e8e4dfa8e0..9fc4d8e6bd 100644 --- a/tests/utils/testing_utils/run_test.py +++ b/tests/utils/testing_utils/run_test.py @@ -10,10 +10,23 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +from os import environ +from typing import Optional -def run_test(*, target, binary_path, args=None, cwd="/", timeout=15): +def run_test( + *, target, binary_path, args: Optional[list[str]] = None, cwd="/", timeout=15 +): """Run an integration test to completion.""" - process = target.execute_async(binary_path, args=args, cwd=cwd) + local_args = [] + if args: + local_args = args + + test_runner = environ.get("SCORE_TEST_RUNNER", None) + if test_runner: + local_args.insert(0, binary_path) + binary_path = test_runner + + process = target.execute_async(binary_path, args=local_args, cwd=cwd) assert process.wait(timeout) == 0