Exposing the //config:test_image to overload different images to test on

config/BUILD

@@ -12,6 +12,22 @@
 # *******************************************************************************
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
 
+# QEMU virtual machine used by the integration tests. Override on the command
+# line to run the tests against a different image, e.g.
+#   bazel test --//config:integration_mode=qemu \
+#       --//config:qemu_image=//my/package:my_image //tests/...
+label_flag(
+    name = "qemu_image",
+    build_setting_default = "//tests/utils/environments/x86_64-qnx:qemu_image",
+    visibility = ["//visibility:public"],
+)
+
+label_flag(
+    name = "qemu_config",
+    build_setting_default = "//tests/utils/environments/x86_64-qnx:qemu_config.json",
+    visibility = ["//visibility:public"],
+)
+
 # How to run integration tests:
 #
 # - docker: in a Docker container

tests/README.md

@@ -30,7 +30,8 @@
 Note the arguments are written so that you can add them to the end of the bazel
 test command.
 
-| Arg                            | What It Does                                  | Example Command                                 |
-|:-------------------------------|:----------------------------------------------|:------------------------------------------------|
-|`--test_arg=--no-local-cleanup` | Integration tests don't cleanup after running |`bazel test //... --test_arg=--no-local-cleanup` |
-|`--test_arg=-s`                 | More logging from the python test framework   |`bazel test //... --test_arg=-s`                 |
+| Arg                            | What It Does                                      | Example Command                                 |
+|:-------------------------------|:--------------------------------------------------|:------------------------------------------------|
+|`--test_arg=--no-local-cleanup` | Integration tests don't cleanup after running     |`bazel test //... --test_arg=--no-local-cleanup` |
+|`--test_arg="--test_runner=XYZ"`| Integration tests run with the given test runner  |`bazel test //... --test_arg="--test_runner=time` |
+|`--test_arg=-s`                 | More logging from the python test framework       |`bazel test //... --test_arg=-s`                 |

tests/integration/crash_ignores_dependents/BUILD

@@ -38,7 +38,6 @@ integration_test(
     name = "crash_ignores_dependents",
     srcs = ["crash_ignores_dependents.py"],
     binaries = [
-        ":config",
         ":test_process",
         ":process_crashing_once",
         "//score/launch_manager",

tests/integration/fallback_to_same_target_restarts/BUILD

@@ -38,7 +38,6 @@ integration_test(
     name = "fallback_to_same_target_restarts",
     srcs = ["fallback_to_same_target_restarts.py"],
     binaries = [
-        ":config",
         ":control_client_test_driver",
         ":process_crashing_once",
         "//score/launch_manager",

tests/integration/process_crash_monitoring/BUILD

@@ -38,7 +38,6 @@ integration_test(
     name = "process_crash_monitoring",
     srcs = ["process_crash_monitoring.py"],
     binaries = [
-        ":config",
         ":control_client_test_driver",
         ":process_crashing_on_runtime",
         "//score/launch_manager",

tests/utils/bazel/integration.bzl

@@ -17,13 +17,18 @@ load("@score_lifecycle_pip//:requirements.bzl", "all_requirements")
 load("//:defs.bzl", "launch_manager_config")
 load("//tests/utils/bazel:constants.bzl", "SCORE_TEST_INSTALL_PREFIX")
 
+DEFAULT_QEMU_CONFIG = "//config:qemu_config"
+DEFAULT_QEMU_IMAGE = "//config:qemu_image"
+
 def integration_test(
         name,
         srcs,
         binaries,
         files = [],
         config = None,
         install_prefix = SCORE_TEST_INSTALL_PREFIX,
+        qemu_config = DEFAULT_QEMU_CONFIG,
+        qemu_image = DEFAULT_QEMU_IMAGE,
         **kwargs):
     """Creates an integration test.
 
@@ -41,61 +46,70 @@ def integration_test(
         files: Additional files
         config: Launch manager configuration file
         install_prefix: Installation prefix for the test environment
+        qemu_config: QEMU configuration file used by the QEMU test target.
+            Defaults to the `//config:qemu_config` label flag.
+        qemu_image: QEMU image used by the QEMU test target.
+            Defaults to the `//config:qemu_image` label flag.
         **kwargs: Miscellaneous arguments passed through to `py_itf_test`
     """
 
+    bin_pkg_name = "{}_binaries_pkg".format(name)
+    config_name = "{}_config_pkg".format(name)
+    test_pkg_name = "{}_test_pkg".format(name)
+    test_tar_name = "{}_test_tar".format(name)
+
     pkg_files(
-        name = "binaries",
+        name = bin_pkg_name,
         srcs = binaries,
         attributes = pkg_attributes(mode = "0555"),
         prefix = "tests/{}".format(name),
     )
 
     if config:
         launch_manager_config(
-            name = "config",
+            name = config_name,
             config = config,
             flatbuffer_out_dir = "etc",
         )
-        all_files = files + [":config"]
+        all_files = files + [":{}".format(config_name)]
     else:
         all_files = files
 
     pkg_files(
-        name = "files",
+        name = test_pkg_name,
         srcs = all_files,
         prefix = "tests/{}".format(name),
         attributes = pkg_attributes(mode = "0400"),
     )
 
-    pkg_tar(name = "environment", srcs = [":binaries", ":files"])
+    pkg_tar(name = test_tar_name, srcs = [":{}".format(bin_pkg_name), ":{}".format(test_pkg_name)])
 
     final_deps = kwargs.pop("deps", []) + all_requirements + [
         "@score_tooling//python_basics/score_pytest:attribute_plugin",
         "//tests/utils/testing_utils",
     ]
-    final_data = kwargs.pop("data", []) + [":environment"] + select({
+    final_data = kwargs.pop("data", []) + [":{}".format(test_tar_name)] + select({
         "//config:integration_docker": [
             "//tests/utils/environments/x86_64-linux",
         ],
         "//config:integration_qemu": [
-            "//tests/utils/environments/x86_64-qnx:qemu_config.json",
-            "//tests/utils/environments/x86_64-qnx:qemu_image",
+            qemu_config,
+            qemu_image,
         ],
         "//conditions:default": [],
     })
     final_args = kwargs.pop("args", []) + [
         "-p attribute_plugin",
-        "--score-test-binary-path=$(locations :environment)",
+        "--score-test-binary-path=$(location :{})".format(test_tar_name),
         "--score-test-remote-directory={}/tests/{}".format(install_prefix, name),
     ] + select({
         "//config:integration_docker": [
             "--docker-image-bootstrap=$(location //tests/utils/environments/x86_64-linux)",
             "--docker-image=score_itf_examples:latest",
         ],
         "//config:integration_qemu": [
-            "--qemu-config=$(location //tests/utils/environments/x86_64-qnx:qemu_config.json)",
-            "--qemu-image=$(location //tests/utils/environments/x86_64-qnx:qemu_image)",
+            "--qemu-config=$(location {})".format(qemu_config),
+            "--qemu-image=$(location {})".format(qemu_image),
         ],
         "//config:integration_host": [
             "--local-dir=/tmp/score_itf_host/{}".format(name),

tests/utils/environments/x86_64-qnx/BUILD

@@ -17,6 +17,7 @@ load("@score_rules_imagefs//rules/qnx:ifs.bzl", "qnx_ifs")
 exports_files(
     ["qemu_config.json"],
     visibility = [
+        "//config:__pkg__",
         "//examples:__subpackages__",
         "//tests:__subpackages__",
     ],
@@ -34,6 +35,7 @@ qnx_ifs(
     build_file = ":init.build",
     target_compatible_with = ["@platforms//os:qnx"],
     visibility = [
+        "//config:__pkg__",
         "//examples:__subpackages__",
         "//tests:__subpackages__",
     ],

tests/utils/plugins/integration.py

@@ -29,6 +29,15 @@ def pytest_addoption(parser):
         default=None,
         help="Absolute remote directory path used during test execution.",
     )
+    parser.addoption(
+        "--test_runner",
+        action="store",
+        default=None,
+        help="""Wrapper program to run the test inside, for example a debugger.
+        The wrapper program will be called with the test binary as its first
+        argument, followed by any number of test arguments. It must execute the
+        test binary with the provided arguments and exit when complete.""",
+    )
 
 
 @pytest.fixture
@@ -51,3 +60,7 @@ def pytest_configure(config):
     remote_dir = config.getoption("--score-test-remote-directory", default=None)
     if remote_dir is not None:
         os.environ["SCORE_TEST_REMOTE_DIRECTORY"] = remote_dir
+
+    test_runner = config.getoption("--test_runner", default=None)
+    if test_runner is not None:
+        os.environ["SCORE_TEST_RUNNER"] = test_runner

tests/utils/testing_utils/run_test.py

@@ -10,10 +10,23 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 # *******************************************************************************
+from os import environ
+from typing import Optional
 
 
-def run_test(*, target, binary_path, args=None, cwd="/", timeout=15):
+def run_test(
+    *, target, binary_path, args: Optional[list[str]] = None, cwd="/", timeout=15
+):
     """Run an integration test to completion."""
 
-    process = target.execute_async(binary_path, args=args, cwd=cwd)
+    local_args = []
+    if args:
+        local_args = args
+
+    test_runner = environ.get("SCORE_TEST_RUNNER", None)
+    if test_runner:
+        local_args.insert(0, binary_path)
+        binary_path = test_runner
+
+    process = target.execute_async(binary_path, args=local_args, cwd=cwd)
     assert process.wait(timeout) == 0