diff --git a/kernelci/runtime/__init__.py b/kernelci/runtime/__init__.py index 3473378267..427a6e73aa 100644 --- a/kernelci/runtime/__init__.py +++ b/kernelci/runtime/__init__.py @@ -287,6 +287,15 @@ def get_job_id(self, job_object): def wait(self, job_object): """Wait for a job to complete and get the exit status code""" + def is_alive(self): + """Check whether the runtime is reachable + + Return a (alive, detail) tuple where *detail* describes the outcome + for logging. Runtimes that have no cheap way of answering this, or + that cannot become unreachable, report themselves as alive. + """ + return True, "liveness check not implemented" + def get_runtime( config, user=None, token=None, custom_template_dir=None, kcictx=None diff --git a/kernelci/runtime/lava.py b/kernelci/runtime/lava.py index 311f76ece8..ba6f76b618 100644 --- a/kernelci/runtime/lava.py +++ b/kernelci/runtime/lava.py @@ -346,6 +346,19 @@ class LAVA(Runtime): API_VERSION = "v0.2" RestAPIServer = namedtuple("RestAPIServer", ["url", "session"]) + # Connecting is capped well below the response timeout so an unreachable + # lab fails in seconds instead of tying up the caller for the full + # request timeout on every single call. + CONNECT_TIMEOUT = 5 + REQUEST_TIMEOUT = 30 + + # Liveness probe. lava-server answers /system/version/ from a constant + # without touching the database (SystemViewSet.version in + # lava_rest_app/v02/views.py) and the body is ~20 bytes, so this can be + # polled regularly without adding any measurable load to the lab. + LIVENESS_PATH = "system/version/" + LIVENESS_TIMEOUT = 10 + # LAVA supports 'high'/'medium'/'low' (100/50/0), but we define our own # values to allow scaling across labs with different priority ranges. PRIORITY_HIGHEST = 80 @@ -457,7 +470,7 @@ def wait(self, job_object): job_id = int(job_object) job_url = urljoin(self._server.url, "/".join(["jobs", str(job_id)])) while True: - resp = self._server.session.get(job_url, timeout=30) + resp = self._server.session.get(job_url, timeout=self._timeout()) resp.raise_for_status() data = resp.json() if data["state"] == "Finished": @@ -477,8 +490,41 @@ def _connect(self): } return rest_api + def _timeout(self, read_timeout=None): + """Timeout tuple for requests: fail fast on connect, wait on read""" + return (self.CONNECT_TIMEOUT, read_timeout or self.REQUEST_TIMEOUT) + + def is_alive(self): + """Check that the LAVA instance is reachable + + Any HTTP answer proves the instance is up, including the ones that + refuse the request: 401 and 403 mean the token or the ACL is wrong, + not that the lab is down. Only a transport failure or a server + error counts as unreachable. + """ + if self._server.url is None: + return True, "no server URL configured" + url = urljoin(self._server.url, self.LIVENESS_PATH) + try: + resp = self._server.session.get( + url, timeout=self._timeout(self.LIVENESS_TIMEOUT) + ) + except requests.RequestException as exc: + return False, str(exc) + if resp.status_code >= 500: + return False, f"HTTP {resp.status_code}" + if resp.status_code != 200: + return True, f"HTTP {resp.status_code} (reachable)" + try: + version = resp.json().get("version") + except ValueError: + version = None + return True, f"version {version}" if version else "reachable" + def _get_response(self, url, params=None): - resp = self._server.session.get(url, params=params, timeout=30) + resp = self._server.session.get( + url, params=params, timeout=self._timeout() + ) resp.raise_for_status() return resp.json() @@ -574,7 +620,7 @@ def _submit(self, job): jobs_url, json=job_data, allow_redirects=False, - timeout=30, + timeout=self._timeout(), ) if resp.status_code >= 400: print(f"Error submitting job: {resp.status_code}, {resp.text}") diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 81ea8b0644..5554ae9f2e 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -12,6 +12,7 @@ from pathlib import Path import pytest +import requests import yaml from jinja2 import Environment, FileSystemLoader from jinja2.exceptions import TemplateRuntimeError @@ -528,3 +529,88 @@ def test_compute_tuxrun_parameters_missing_branch(): """Nodes lacking kernel_revision.branch get an empty parameter set.""" assert compute_tuxrun_parameters("fvp-aemva", {}) == {} assert compute_tuxrun_parameters("fvp-aemva", {"data": {}}) == {} + + +class _LivenessSession: + """Session recording the liveness request and replaying a canned answer""" + + def __init__(self, response=None, error=None): + self.response = response + self.error = error + self.calls = [] + + def get(self, url, params=None, timeout=None): + self.calls.append((url, timeout)) + if self.error: + raise self.error + return self.response + + +def _liveness_lab(response=None, error=None): + config = kernelci.config.load("tests/configs/lava-runtimes.yaml") + runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"] + lab = kernelci.runtime.get_runtime(runtime_config) + lab._server = types.SimpleNamespace( + url="http://lava/api/v0.2/", + session=_LivenessSession(response=response, error=error), + ) + return lab + + +def test_lava_is_alive_reports_the_version(): + """A 200 from /system/version/ means the lab is up.""" + lab = _liveness_lab(_FakeResponse({"version": "2026.07"})) + + alive, detail = lab.is_alive() + + assert alive is True + assert "2026.07" in detail + url, timeout = lab._server.session.calls[0] + assert url == "http://lava/api/v0.2/system/version/" + # Connect fast, then allow the probe timeout for the answer. + assert timeout == ( + kernelci.runtime.lava.LAVA.CONNECT_TIMEOUT, + kernelci.runtime.lava.LAVA.LIVENESS_TIMEOUT, + ) + + +def test_lava_is_alive_treats_forbidden_as_reachable(): + """A lab that refuses the request has still answered it.""" + lab = _liveness_lab(_FakeResponse({}, status_code=403)) + + alive, detail = lab.is_alive() + + assert alive is True + assert "403" in detail + + +def test_lava_is_alive_reports_server_errors_as_down(): + """A 5xx means the instance cannot serve requests.""" + lab = _liveness_lab(_FakeResponse({}, status_code=502)) + + alive, detail = lab.is_alive() + + assert alive is False + assert "502" in detail + + +def test_lava_is_alive_reports_transport_failures_as_down(): + """An unreachable host is the case this probe exists for.""" + lab = _liveness_lab( + error=requests.ConnectionError("Network is unreachable") + ) + + alive, detail = lab.is_alive() + + assert alive is False + assert "Network is unreachable" in detail + + +def test_lava_is_alive_without_server_url(): + """A runtime storing jobs externally has no server to probe.""" + config = kernelci.config.load("tests/configs/lava-runtimes.yaml") + runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"] + lab = kernelci.runtime.get_runtime(runtime_config) + lab._server = types.SimpleNamespace(url=None, session=None) + + assert lab.is_alive() == (True, "no server URL configured")