runtime/lava: add a liveness probe and fail fast on connect

kernelci/runtime/__init__.py

@@ -287,6 +287,15 @@ def get_job_id(self, job_object):
     def wait(self, job_object):
         """Wait for a job to complete and get the exit status code"""
 
+    def is_alive(self):
+        """Check whether the runtime is reachable
+
+        Return a (alive, detail) tuple where *detail* describes the outcome
+        for logging.  Runtimes that have no cheap way of answering this, or
+        that cannot become unreachable, report themselves as alive.
+        """
+        return True, "liveness check not implemented"
+
 
 def get_runtime(
     config, user=None, token=None, custom_template_dir=None, kcictx=None

kernelci/runtime/lava.py

@@ -346,6 +346,19 @@ class LAVA(Runtime):
     API_VERSION = "v0.2"
     RestAPIServer = namedtuple("RestAPIServer", ["url", "session"])
 
+    # Connecting is capped well below the response timeout so an unreachable
+    # lab fails in seconds instead of tying up the caller for the full
+    # request timeout on every single call.
+    CONNECT_TIMEOUT = 5
+    REQUEST_TIMEOUT = 30
+
+    # Liveness probe.  lava-server answers /system/version/ from a constant
+    # without touching the database (SystemViewSet.version in
+    # lava_rest_app/v02/views.py) and the body is ~20 bytes, so this can be
+    # polled regularly without adding any measurable load to the lab.
+    LIVENESS_PATH = "system/version/"
+    LIVENESS_TIMEOUT = 10
+
     # LAVA supports 'high'/'medium'/'low' (100/50/0), but we define our own
     # values to allow scaling across labs with different priority ranges.
     PRIORITY_HIGHEST = 80
@@ -457,7 +470,7 @@ def wait(self, job_object):
         job_id = int(job_object)
         job_url = urljoin(self._server.url, "/".join(["jobs", str(job_id)]))
         while True:
-            resp = self._server.session.get(job_url, timeout=30)
+            resp = self._server.session.get(job_url, timeout=self._timeout())
             resp.raise_for_status()
             data = resp.json()
             if data["state"] == "Finished":
@@ -477,8 +490,41 @@ def _connect(self):
         }
         return rest_api
 
+    def _timeout(self, read_timeout=None):
+        """Timeout tuple for requests: fail fast on connect, wait on read"""
+        return (self.CONNECT_TIMEOUT, read_timeout or self.REQUEST_TIMEOUT)
+
+    def is_alive(self):
+        """Check that the LAVA instance is reachable
+
+        Any HTTP answer proves the instance is up, including the ones that
+        refuse the request: 401 and 403 mean the token or the ACL is wrong,
+        not that the lab is down.  Only a transport failure or a server
+        error counts as unreachable.
+        """
+        if self._server.url is None:
+            return True, "no server URL configured"
+        url = urljoin(self._server.url, self.LIVENESS_PATH)
+        try:
+            resp = self._server.session.get(
+                url, timeout=self._timeout(self.LIVENESS_TIMEOUT)
+            )
+        except requests.RequestException as exc:
+            return False, str(exc)
+        if resp.status_code >= 500:
+            return False, f"HTTP {resp.status_code}"
+        if resp.status_code != 200:
+            return True, f"HTTP {resp.status_code} (reachable)"
+        try:
+            version = resp.json().get("version")
+        except ValueError:
+            version = None
+        return True, f"version {version}" if version else "reachable"
+
     def _get_response(self, url, params=None):
-        resp = self._server.session.get(url, params=params, timeout=30)
+        resp = self._server.session.get(
+            url, params=params, timeout=self._timeout()
+        )
         resp.raise_for_status()
         return resp.json()
 
@@ -574,7 +620,7 @@ def _submit(self, job):
             jobs_url,
             json=job_data,
             allow_redirects=False,
-            timeout=30,
+            timeout=self._timeout(),
         )
         if resp.status_code >= 400:
             print(f"Error submitting job: {resp.status_code}, {resp.text}")

tests/test_runtime.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import pytest
+import requests
 import yaml
 from jinja2 import Environment, FileSystemLoader
 from jinja2.exceptions import TemplateRuntimeError
@@ -528,3 +529,88 @@ def test_compute_tuxrun_parameters_missing_branch():
     """Nodes lacking kernel_revision.branch get an empty parameter set."""
     assert compute_tuxrun_parameters("fvp-aemva", {}) == {}
     assert compute_tuxrun_parameters("fvp-aemva", {"data": {}}) == {}
+
+
+class _LivenessSession:
+    """Session recording the liveness request and replaying a canned answer"""
+
+    def __init__(self, response=None, error=None):
+        self.response = response
+        self.error = error
+        self.calls = []
+
+    def get(self, url, params=None, timeout=None):
+        self.calls.append((url, timeout))
+        if self.error:
+            raise self.error
+        return self.response
+
+
+def _liveness_lab(response=None, error=None):
+    config = kernelci.config.load("tests/configs/lava-runtimes.yaml")
+    runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"]
+    lab = kernelci.runtime.get_runtime(runtime_config)
+    lab._server = types.SimpleNamespace(
+        url="http://lava/api/v0.2/",
+        session=_LivenessSession(response=response, error=error),
+    )
+    return lab
+
+
+def test_lava_is_alive_reports_the_version():
+    """A 200 from /system/version/ means the lab is up."""
+    lab = _liveness_lab(_FakeResponse({"version": "2026.07"}))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is True
+    assert "2026.07" in detail
+    url, timeout = lab._server.session.calls[0]
+    assert url == "http://lava/api/v0.2/system/version/"
+    # Connect fast, then allow the probe timeout for the answer.
+    assert timeout == (
+        kernelci.runtime.lava.LAVA.CONNECT_TIMEOUT,
+        kernelci.runtime.lava.LAVA.LIVENESS_TIMEOUT,
+    )
+
+
+def test_lava_is_alive_treats_forbidden_as_reachable():
+    """A lab that refuses the request has still answered it."""
+    lab = _liveness_lab(_FakeResponse({}, status_code=403))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is True
+    assert "403" in detail
+
+
+def test_lava_is_alive_reports_server_errors_as_down():
+    """A 5xx means the instance cannot serve requests."""
+    lab = _liveness_lab(_FakeResponse({}, status_code=502))
+
+    alive, detail = lab.is_alive()
+
+    assert alive is False
+    assert "502" in detail
+
+
+def test_lava_is_alive_reports_transport_failures_as_down():
+    """An unreachable host is the case this probe exists for."""
+    lab = _liveness_lab(
+        error=requests.ConnectionError("Network is unreachable")
+    )
+
+    alive, detail = lab.is_alive()
+
+    assert alive is False
+    assert "Network is unreachable" in detail
+
+
+def test_lava_is_alive_without_server_url():
+    """A runtime storing jobs externally has no server to probe."""
+    config = kernelci.config.load("tests/configs/lava-runtimes.yaml")
+    runtime_config = config["runtimes"]["lab-min-12-max-40-new-runtime"]
+    lab = kernelci.runtime.get_runtime(runtime_config)
+    lab._server = types.SimpleNamespace(url=None, session=None)
+
+    assert lab.is_alive() == (True, "no server URL configured")