From bbc6c62066f6107e70b41cf96fa2fa0f1a45493c Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Tue, 8 Sep 2026 19:24:22 +0300 Subject: [PATCH] runtime/lava: add a liveness probe and fail fast on connect Every device and queue query the scheduler makes against a lab that has gone away blocks for the full 30s request timeout, once per job and per platform. There is no way to ask "is this lab up?" without paying that cost on an endpoint that also does real work. Add LAVA.is_alive(), which polls /system/version/. lava-server answers it from a constant with no database access (SystemViewSet.version in lava_rest_app/v02/views.py) and the body is ~20 bytes, against ~16-78kB and a full device table scan for /devices/, so a caller can poll it regularly without adding any measurable load to the lab. Any HTTP answer counts as reachable, including the refusals: labs that return 401 or 403 to an anonymous or unprivileged request are up, and treating them as down would stop scheduling against them entirely. Only a transport failure or a 5xx is reported as unreachable. Also split the request timeout into a short connect timeout and the existing response timeout, so an unreachable host fails in seconds rather than after 30, and give Runtime a default is_alive() so callers can probe any runtime. Signed-off-by: Denys Fedoryshchenko --- kernelci/runtime/__init__.py | 9 ++++ kernelci/runtime/lava.py | 52 ++++++++++++++++++++-- tests/test_runtime.py | 86 ++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/kernelci/runtime/__init__.py b/kernelci/runtime/__init__.py index 3473378267..427a6e73aa 100644 --- a/kernelci/runtime/__init__.py +++ b/kernelci/runtime/__init__.py @@ -287,6 +287,15 @@ def get_job_id(self, job_object): def wait(self, job_object): """Wait for a job to complete and get the exit status code""" + def is_alive(self): + """Check whether the runtime is reachable + + Return a (alive, detail) tuple where *detail* describes the outcome + for logging. Runtimes that have no cheap way of answering this, or + that cannot become unreachable, report themselves as alive. + """ + return True, "liveness check not implemented" + def get_runtime( config, user=None, token=None, custom_template_dir=None, kcictx=None diff --git a/kernelci/runtime/lava.py b/kernelci/runtime/lava.py index 311f76ece8..ba6f76b618 100644 --- a/kernelci/runtime/lava.py +++ b/kernelci/runtime/lava.py @@ -346,6 +346,19 @@ class LAVA(Runtime): API_VERSION = "v0.2" RestAPIServer = namedtuple("RestAPIServer", ["url", "session"]) + # Connecting is capped well below the response timeout so an unreachable + # lab fails in seconds instead of tying up the caller for the full + # request timeout on every single call. + CONNECT_TIMEOUT = 5 + REQUEST_TIMEOUT = 30 + + # Liveness probe. lava-server answers /system/version/ from a constant + # without touching the database (SystemViewSet.version in + # lava_rest_app/v02/views.py) and the body is ~20 bytes, so this can be + # polled regularly without adding any measurable load to the lab. + LIVENESS_PATH = "system/version/" + LIVENESS_TIMEOUT = 10 + # LAVA supports 'high'/'medium'/'low' (100/50/0), but we define our own # values to allow scaling across labs with different priority ranges. PRIORITY_HIGHEST = 80 @@ -457,7 +470,7 @@ def wait(self, job_object): job_id = int(job_object) job_url = urljoin(self._server.url, "/".join(["jobs", str(job_id)])) while True: - resp = self._server.session.get(job_url, timeout=30) + resp = self._server.session.get(job_url, timeout=self._timeout()) resp.raise_for_status() data = resp.json() if data["state"] == "Finished": @@ -477,8 +490,41 @@ def _connect(self): } return rest_api + def _timeout(self, read_timeout=None): + """Timeout tuple for requests: fail fast on connect, wait on read""" + return (self.CONNECT_TIMEOUT, read_timeout or self.REQUEST_TIMEOUT) + + def is_alive(self): + """Check that the LAVA instance is reachable + + Any HTTP answer proves the instance is up, including the ones that + refuse the request: 401 and 403 mean the token or the ACL is wrong, + not that the lab is down. Only a transport failure or a server + error counts as unreachable. + """ + if self._server.url is None: + return True, "no server URL configured" + url = urljoin(self._server.url, self.LIVENESS_PATH) + try: + resp = self._server.session.get( + url, timeout=self._timeout(self.LIVENESS_TIMEOUT) + ) + except requests.RequestException as exc: + return False, str(exc) + if resp.status_code >= 500: + return False, f"HTTP {resp.status_code}" + if resp.status_code != 200: + return True, f"HTTP {resp.status_code} (reachable)" + try: + version = resp.json().get("version") + except ValueError: + version = None + return True, f"version {version}" if version else "reachable" + def _get_response(self, url, params=None): - resp = self._server.session.get(url, params=params, timeout=30) + resp = self._server.session.get( + url, params=params, timeout=self._timeout() + ) resp.raise_for_status() return resp.json() @@ -574,7 +620,7 @@ def _submit(self, job): jobs_url, json=job_data, allow_redirects=False, - timeout=30, + timeout=self._timeout(), ) if resp.status_code >= 400: print(f"Error submitting job: {resp.status_code}, {resp.text}") diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 81ea8b0644..5554ae9f2e 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -12,6 +12,7 @@ from pathlib import Path import pytest +import requests import yaml from jinja2 import Environment, FileSystemLoader from jinja2.exceptions import TemplateRuntimeError @@ -528,3 +529,88 @@ def test_compute_tuxrun_parameters_missing_branch(): """Nodes lacking kernel_revision.branch get an empty parameter set.""" assert compute_tuxrun_parameters("fvp-aemva", {}) == {} assert compute_tuxrun_parameters("fvp-aemva", {"data": {}}) == {} + + +class _LivenessSession: + """Session recording the liveness request and replaying a canned answer""" + + def __init__(self, response=None, error=None): + self.response = response + self.error = error + self.calls = [] + + def get(self, url, params=None, timeout=None): + self.calls.append((url, timeout)) + if self.error: + raise self.error + return self.response + + +def _liveness_lab(response=None, error=None): + config = kernelci.config.load("tests/configs/lava-runtimes.yaml") + runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"] + lab = kernelci.runtime.get_runtime(runtime_config) + lab._server = types.SimpleNamespace( + url="http://lava/api/v0.2/", + session=_LivenessSession(response=response, error=error), + ) + return lab + + +def test_lava_is_alive_reports_the_version(): + """A 200 from /system/version/ means the lab is up.""" + lab = _liveness_lab(_FakeResponse({"version": "2026.07"})) + + alive, detail = lab.is_alive() + + assert alive is True + assert "2026.07" in detail + url, timeout = lab._server.session.calls[0] + assert url == "http://lava/api/v0.2/system/version/" + # Connect fast, then allow the probe timeout for the answer. + assert timeout == ( + kernelci.runtime.lava.LAVA.CONNECT_TIMEOUT, + kernelci.runtime.lava.LAVA.LIVENESS_TIMEOUT, + ) + + +def test_lava_is_alive_treats_forbidden_as_reachable(): + """A lab that refuses the request has still answered it.""" + lab = _liveness_lab(_FakeResponse({}, status_code=403)) + + alive, detail = lab.is_alive() + + assert alive is True + assert "403" in detail + + +def test_lava_is_alive_reports_server_errors_as_down(): + """A 5xx means the instance cannot serve requests.""" + lab = _liveness_lab(_FakeResponse({}, status_code=502)) + + alive, detail = lab.is_alive() + + assert alive is False + assert "502" in detail + + +def test_lava_is_alive_reports_transport_failures_as_down(): + """An unreachable host is the case this probe exists for.""" + lab = _liveness_lab( + error=requests.ConnectionError("Network is unreachable") + ) + + alive, detail = lab.is_alive() + + assert alive is False + assert "Network is unreachable" in detail + + +def test_lava_is_alive_without_server_url(): + """A runtime storing jobs externally has no server to probe.""" + config = kernelci.config.load("tests/configs/lava-runtimes.yaml") + runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"] + lab = kernelci.runtime.get_runtime(runtime_config) + lab._server = types.SimpleNamespace(url=None, session=None) + + assert lab.is_alive() == (True, "no server URL configured")