diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml new file mode 100644 index 00000000..48dabb94 --- /dev/null +++ b/.github/workflows/lvh-integration-tests.yml @@ -0,0 +1,179 @@ +name: QEMU Integration Tests + +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.run_id }}-qemu + cancel-in-progress: true + +env: + VM_DIR: /tmp/fact-vm + SSH_PORT: '2222' + +jobs: + build: + name: Build fact image + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + with: + submodules: true + fetch-depth: 0 + persist-credentials: false + + - name: Build image + run: | + FACT_REGISTRY=localhost/fact make image + + - name: Export image + run: | + docker save -o /tmp/fact-image.tar "$(FACT_REGISTRY=localhost/fact make image-name)" + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: fact-image + path: /tmp/fact-image.tar + retention-days: 1 + + integration-tests: + name: Integration tests (${{ matrix.name }}) + needs: build + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - name: centos-9-stream + image_url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 + checksum_url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM + cloud_init: hack/cloud-init/centos.yml + - name: centos-10-stream + image_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2 + checksum_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2.SHA256SUM + cloud_init: hack/cloud-init/centos.yml + - name: ubuntu-24.04 + image_url: https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img + checksum_url: https://cloud-images.ubuntu.com/releases/24.04/release/SHA256SUMS + cloud_init: hack/cloud-init/ubuntu.yml + - name: ubuntu-26.04 + image_url: https://cloud-images.ubuntu.com/releases/26.04/release/ubuntu-26.04-server-cloudimg-amd64.img + checksum_url: https://cloud-images.ubuntu.com/releases/26.04/release/SHA256SUMS + cloud_init: hack/cloud-init/ubuntu.yml + - name: fedora-44 + image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2 + checksum_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM + cloud_init: hack/cloud-init/fedora.yml + steps: + - name: Checkout fact + uses: actions/checkout@v7 + with: + submodules: true + fetch-depth: 0 + persist-credentials: false + + - name: Install QEMU + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + qemu-system-x86 qemu-utils genisoimage virtiofsd + [ -w /dev/kvm ] || sudo chmod 666 /dev/kvm + + - name: Download VM base image + run: | + mkdir -p "${VM_DIR}" + image_name="$(basename "${{ matrix.image_url }}")" + curl -fSL --progress-bar -o "${VM_DIR}/${image_name}" "${{ matrix.image_url }}" + + # Checksum files may list multiple images (Ubuntu SHA256SUMS) or + # use either GNU ("hash filename") or BSD ("SHA256 (filename) = + # hash") tagged format. sha256sum -c auto-detects both, so we + # only need to filter the file down to our image's line. + curl -fsSL "${{ matrix.checksum_url }}" | grep -F "${image_name}" > "${VM_DIR}/${image_name}.sum" + (cd "${VM_DIR}" && sha256sum -c "${image_name}.sum") + rm -f "${VM_DIR}/${image_name}.sum" + + mv "${VM_DIR}/${image_name}" "${VM_DIR}/base.qcow2" + + - name: Download fact image artifact + uses: actions/download-artifact@v4 + with: + name: fact-image + + - name: Start VM + run: | + hack/qemu-vm.sh start \ + --image "${VM_DIR}/base.qcow2" \ + --cloud-init "${{ matrix.cloud_init }}" \ + --vm-dir "${VM_DIR}" \ + --ssh-port "${SSH_PORT}" \ + --host-mount "${{ github.workspace }}" + + - name: Load fact image into VM + run: | + hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \ + docker load -i /host/fact-image.tar + + - name: Setup test environment in VM + run: | + hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \ + bash -c ' + set -euo pipefail + cd /host/tests + python3 -m venv .venv + source .venv/bin/activate + pip install -q -r requirements.txt + make grpc-gen + ' + + - name: Run integration tests + run: | + # NOTE: --no-local-builds is intentionally omitted. The editor, + # fedora, and self-deleter test containers are normally pulled + # from quay.io/rhacs-eng/qa-multi-arch, which is a private + # registry that fork PR runs have no credentials for. Without + # --no-local-builds, tests/containers.py::pull_or_build() falls + # back to building those images locally from their Containerfiles + # on a pull error. + FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make --no-print-directory image-name)" + hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \ + bash -c " + set -euo pipefail + mkdir -p /tmp/fact-tmp + cd /host/tests + source .venv/bin/activate + pytest \ + --image='${FACT_IMAGE}' \ + --output=grpc \ + --tmp-dir=/tmp/fact-tmp \ + --junit-xml=results.xml \ + --tb=short + " + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qemu-test-results-${{ matrix.name }} + path: | + tests/results.xml + tests/logs/ + if-no-files-found: ignore + + - name: Test summary + uses: test-summary/action@v2 + if: always() + with: + paths: tests/results.xml + + - name: Dump VM console on failure + if: failure() + run: cat "${VM_DIR}/console.log" 2>/dev/null || true + + - name: Stop VM + if: always() + run: hack/qemu-vm.sh stop --vm-dir "${VM_DIR}" diff --git a/hack/cloud-init/centos.yml b/hack/cloud-init/centos.yml new file mode 100644 index 00000000..c99ae7be --- /dev/null +++ b/hack/cloud-init/centos.yml @@ -0,0 +1,20 @@ +#cloud-config +# Cloud-init user-data for CentOS Stream 10 (and RHEL 10). +# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time. + +users: + - name: root + lock_passwd: false + ssh_authorized_keys: + - __SSH_PUBKEY__ + +ssh_pwauth: false + +runcmd: + - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf" + - dnf -y install dnf-plugins-core + - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin + - systemctl enable --now docker + - dnf -y install python3 python3-pip python3-devel gcc + - touch /var/lib/cloud/instance/boot-finished-user diff --git a/hack/cloud-init/fedora.yml b/hack/cloud-init/fedora.yml new file mode 100644 index 00000000..13030a2e --- /dev/null +++ b/hack/cloud-init/fedora.yml @@ -0,0 +1,20 @@ +#cloud-config +# Cloud-init user-data for Fedora (41+, dnf5-based). +# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time. + +users: + - name: root + lock_passwd: false + ssh_authorized_keys: + - __SSH_PUBKEY__ + +ssh_pwauth: false + +runcmd: + - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf" + - dnf -y install dnf5-plugins + - dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo + - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin + - systemctl enable --now docker + - dnf -y install python3 python3-pip python3-devel gcc + - touch /var/lib/cloud/instance/boot-finished-user diff --git a/hack/cloud-init/ubuntu.yml b/hack/cloud-init/ubuntu.yml new file mode 100644 index 00000000..9e39b315 --- /dev/null +++ b/hack/cloud-init/ubuntu.yml @@ -0,0 +1,38 @@ +#cloud-config +# Cloud-init user-data for Ubuntu (24.04+). +# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time. +# +# NOTE: Ubuntu uses AppArmor by default; the LSM list must include it +# alongside bpf. The grubby tool is not available — use a GRUB +# drop-in config file instead (see runcmd below for why a plain +# /etc/default/grub edit does not work). + +users: + - name: root + lock_passwd: false + ssh_authorized_keys: + - __SSH_PUBKEY__ + +ssh_pwauth: false + +runcmd: + # Ubuntu cloud images ship /etc/default/grub.d/50-cloudimg-settings.cfg, + # which sets GRUB_CMDLINE_LINUX_DEFAULT and is sourced by grub-mkconfig + # AFTER /etc/default/grub, clobbering any edits made there. Write our + # own drop-in with a higher-sorting filename so it wins instead. + - | + cat > /etc/default/grub.d/99-fact-lsm.cfg <<'EOF' + GRUB_CMDLINE_LINUX_DEFAULT="console=tty1 console=ttyS0 lsm=lockdown,capability,yama,apparmor,bpf" + EOF + - update-grub + - apt-get update -qq + - apt-get install -y -qq ca-certificates curl + - install -m 0755 -d /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - chmod a+r /etc/apt/keyrings/docker.asc + - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list + - apt-get update -qq + - apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin + - systemctl enable --now docker + - apt-get install -y -qq python3-venv python3-dev gcc make + - touch /var/lib/cloud/instance/boot-finished-user diff --git a/hack/qemu-vm.sh b/hack/qemu-vm.sh new file mode 100755 index 00000000..61488dd0 --- /dev/null +++ b/hack/qemu-vm.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Boot a cloud image in QEMU/KVM for integration testing. +# +# Designed to work both locally and in CI (GitHub Actions). The script +# takes a pre-downloaded qcow2 cloud image, injects cloud-init config, +# boots the VM, waits until it is ready, and prints SSH connection +# details. +# +# Usage: +# hack/qemu-vm.sh start [options] — boot the VM +# hack/qemu-vm.sh ssh [options] — open an SSH session to the VM +# hack/qemu-vm.sh stop [options] — kill the VM +# +# Options: +# --image PATH path to the base qcow2 image (required for start) +# --cloud-init PATH cloud-init user-data template (required for start) +# --vm-dir DIR working directory for VM files (default: /tmp/fact-vm) +# --ssh-port PORT host port forwarded to guest 22 (default: 2222) +# --cpu N vCPUs (default: all host CPUs) +# --mem SIZE memory, e.g. 8G (default: 75% of host RAM) +# --host-mount DIR directory to share with the VM via virtiofs at /host +# +# Cloud-init templates: +# The --cloud-init file is a standard #cloud-config YAML with one +# special placeholder: __SSH_PUBKEY__ is replaced with the VM's +# ephemeral SSH public key. See hack/cloud-init/ for examples. +# +# The VM state (image overlay, cloud-init ISO, SSH keys, PID file) lives +# entirely under --vm-dir and can be cleaned up by removing that directory. + +set -euo pipefail + +: "${IMAGE:=}" +: "${CLOUD_INIT:=}" +: "${VM_DIR:=/tmp/fact-vm}" +: "${SSH_PORT:=2222}" +: "${CPU:=$(nproc)}" +: "${MEM:=$(awk '/^MemTotal/{printf "%dG", int($2/1024/1024*0.75)}' /proc/meminfo)}" +: "${HOST_MOUNT:=}" + +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) + +usage() { + sed -n '2,/^$/s/^# \{0,1\}//p' "$0" + exit 1 +} + +log() { printf '==> %s\n' "$*" >&2; } + +EXTRA_ARGS=() + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --image) IMAGE="$2"; shift 2;; + --cloud-init) CLOUD_INIT="$2"; shift 2;; + --vm-dir) VM_DIR="$2"; shift 2;; + --ssh-port) SSH_PORT="$2"; shift 2;; + --cpu) CPU="$2"; shift 2;; + --mem) MEM="$2"; shift 2;; + --host-mount) HOST_MOUNT="$2"; shift 2;; + -h|--help) usage;; + --) shift; EXTRA_ARGS+=("$@"); return;; + *) EXTRA_ARGS+=("$@"); return;; + esac + done +} + +create_overlay() { + local base + base="$(realpath "${IMAGE}")" + local overlay="${VM_DIR}/disk.qcow2" + if [[ -f "${overlay}" ]]; then + log "Overlay already exists, removing stale one" + rm -f "${overlay}" + fi + log "Creating qcow2 overlay" + qemu-img create -f qcow2 -b "${base}" -F qcow2 "${overlay}" 20G >/dev/null +} + +generate_ssh_key() { + local key="${VM_DIR}/id_ed25519" + if [[ -f "${key}" ]]; then + return + fi + log "Generating ephemeral SSH key" + ssh-keygen -t ed25519 -f "${key}" -N "" -q +} + +create_cloud_init_iso() { + local iso="${VM_DIR}/seed.iso" + local pubkey + pubkey="$(cat "${VM_DIR}/id_ed25519.pub")" + + local ci_dir="${VM_DIR}/cloud-init" + mkdir -p "${ci_dir}" + + cat > "${ci_dir}/meta-data" < "${ci_dir}/user-data" + + log "Creating cloud-init seed ISO" + genisoimage -output "${iso}" -volid cidata -joliet -rock \ + "${ci_dir}/user-data" "${ci_dir}/meta-data" 2>/dev/null +} + +find_virtiofsd() { + local bin + bin="$(command -v virtiofsd 2>/dev/null)" && { echo "${bin}"; return; } + for p in /usr/libexec/virtiofsd /usr/lib/virtiofsd; do + [[ -x "${p}" ]] && { echo "${p}"; return; } + done + echo "error: virtiofsd not found" >&2 + return 1 +} + +start_virtiofsd() { + local sock="${VM_DIR}/virtiofsd.sock" + local bin + bin="$(find_virtiofsd)" + log "Starting virtiofsd for ${HOST_MOUNT}" + "${bin}" \ + --socket-path="${sock}" \ + --shared-dir="${HOST_MOUNT}" \ + --cache=always & + echo $! > "${VM_DIR}/virtiofsd.pid" + + local i + for i in $(seq 1 30); do + if [[ -S "${sock}" ]]; then + log "virtiofsd ready after ${i}s" + return 0 + fi + sleep 1 + done + log "virtiofsd socket did not appear after 30s" + return 1 +} + +build_qemu_args() { + # shellcheck disable=SC2054 # commas are inside quoted QEMU option values + local args=( + -nodefaults + -display none + -daemonize + -pidfile "${VM_DIR}/qemu.pid" + -enable-kvm + -cpu host + -smp "${CPU}" + -m "${MEM}" + -drive "file=${VM_DIR}/disk.qcow2,if=virtio,format=qcow2" + -drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw,readonly=on" + -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" + -device virtio-net-pci,netdev=net0 + -serial "file:${VM_DIR}/console.log" + ) + + if [[ -n "${HOST_MOUNT}" ]]; then + args+=( + -object "memory-backend-memfd,id=mem,size=${MEM},share=on" + -numa "node,memdev=mem" + -chardev "socket,id=char0,path=${VM_DIR}/virtiofsd.sock" + -device "vhost-user-fs-pci,chardev=char0,tag=host_mount" + ) + fi + + printf '%s\n' "${args[@]}" +} + +vm_ssh() { + ssh -p "${SSH_PORT}" -i "${VM_DIR}/id_ed25519" "${SSH_OPTS[@]}" root@localhost "$@" +} + +wait_for_ssh() { + log "Waiting for SSH (port ${SSH_PORT})..." + local i + for i in $(seq 1 180); do + if vm_ssh true 2>/dev/null; then + log "SSH ready after ${i}s" + return 0 + fi + sleep 1 + done + log "SSH failed to become ready after 180s" + [[ -f "${VM_DIR}/console.log" ]] && cat "${VM_DIR}/console.log" >&2 + return 1 +} + +wait_for_cloud_init() { + log "Waiting for cloud-init to finish..." + local i + for i in $(seq 1 300); do + if vm_ssh "test -f /var/lib/cloud/instance/boot-finished-user" 2>/dev/null; then + log "cloud-init finished after ${i}s" + return 0 + fi + sleep 1 + done + log "cloud-init did not finish within 300s" + vm_ssh "cat /var/log/cloud-init-output.log" 2>/dev/null >&2 || true + return 1 +} + +cmd_start() { + if [[ -z "${IMAGE}" ]]; then + echo "error: --image is required for start" >&2 + exit 1 + fi + if [[ ! -f "${IMAGE}" ]]; then + echo "error: image not found: ${IMAGE}" >&2 + exit 1 + fi + if [[ -z "${CLOUD_INIT}" ]]; then + echo "error: --cloud-init is required for start" >&2 + exit 1 + fi + if [[ ! -f "${CLOUD_INIT}" ]]; then + echo "error: cloud-init template not found: ${CLOUD_INIT}" >&2 + exit 1 + fi + + mkdir -p "${VM_DIR}" + create_overlay + generate_ssh_key + create_cloud_init_iso + + if [[ -n "${HOST_MOUNT}" ]]; then + start_virtiofsd + fi + + log "Starting QEMU (cpu=${CPU}, mem=${MEM}, ssh_port=${SSH_PORT})" + touch "${VM_DIR}/console.log" + local qemu_args + mapfile -t qemu_args < <(build_qemu_args) + qemu-system-x86_64 "${qemu_args[@]}" + + wait_for_ssh + wait_for_cloud_init + + log "Rebooting for BPF LSM kernel cmdline change..." + vm_ssh "reboot" 2>/dev/null || true + sleep 5 + wait_for_ssh + + log "Verifying BPF LSM is active" + local lsm + lsm="$(vm_ssh "cat /sys/kernel/security/lsm")" + if [[ "${lsm}" != *bpf* ]]; then + log "ERROR: bpf not found in LSM list: ${lsm}" + return 1 + fi + log "LSM list: ${lsm}" + + log "Verifying Docker is running" + vm_ssh "docker info" >/dev/null + + if [[ -n "${HOST_MOUNT}" ]]; then + log "Mounting host filesystem inside VM" + vm_ssh "mkdir -p /host && mount -t virtiofs host_mount /host" + fi + + log "VM is ready" + log " SSH: ssh -p ${SSH_PORT} -i ${VM_DIR}/id_ed25519 ${SSH_OPTS[*]} root@localhost" +} + +cmd_ssh() { + vm_ssh "${EXTRA_ARGS[@]}" +} + +kill_pid_file() { + local pidfile="$1" label="$2" + if [[ -f "${pidfile}" ]]; then + local pid + pid="$(cat "${pidfile}")" + log "Stopping ${label} (pid ${pid})" + kill "${pid}" 2>/dev/null || true + rm -f "${pidfile}" + fi +} + +cmd_stop() { + kill_pid_file "${VM_DIR}/qemu.pid" "VM" + kill_pid_file "${VM_DIR}/virtiofsd.pid" "virtiofsd" +} + +main() { + local cmd="${1:-help}" + shift || true + parse_args "$@" + + case "${cmd}" in + start) cmd_start;; + ssh) cmd_ssh;; + stop) cmd_stop;; + *) usage;; + esac +} + +main "$@" diff --git a/tests/conftest.py b/tests/conftest.py index 7bcfa45e..e3cd8657 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,13 +20,27 @@ pytest_plugins = ['test_editors.commons'] +def _tmp_dir_base(request: pytest.FixtureRequest) -> str: + """ + Directory under which monitored/ignored test directories are + created. + + Defaults to the current working directory, but can be overridden + with --tmp-dir, e.g. when the working directory lives on a + filesystem (such as 9p) that does not report file creation + metadata (FMODE_CREATED) correctly. + """ + tmp_dir = request.config.getoption('--tmp-dir') + assert tmp_dir is None or isinstance(tmp_dir, str) + return tmp_dir or os.getcwd() + + @pytest.fixture -def monitored_dir(): +def monitored_dir(request: pytest.FixtureRequest): """ Create a temporary directory for tests and clean it up afterwards. """ - cwd = os.getcwd() - tmp = mkdtemp(prefix='fact-test-', dir=cwd) + tmp = mkdtemp(prefix='fact-test-', dir=_tmp_dir_base(request)) yield tmp rmtree(tmp) @@ -46,13 +60,12 @@ def test_file(monitored_dir: str): @pytest.fixture -def ignored_dir(): +def ignored_dir(request: pytest.FixtureRequest): """ Create a temporary directory for tests that will not be monitored by fact. After tests are done, the directory is cleaned up. """ - cwd = os.getcwd() - tmp = mkdtemp(prefix='fact-test-', dir=cwd) + tmp = mkdtemp(prefix='fact-test-', dir=_tmp_dir_base(request)) yield tmp rmtree(tmp) @@ -302,6 +315,7 @@ def fact( container.remove() pytest.fail('fact did not finish its initial scan') + sleep(1) yield container # Capture prometheus metrics before stopping the container @@ -343,3 +357,14 @@ def pytest_addoption(parser: pytest.Parser): action='store_true', help='Do not build test containers locally', ) + parser.addoption( + '--tmp-dir', + action='store', + default=None, + help=( + 'Directory in which to create monitored/ignored test ' + 'directories (default: current working directory). Useful ' + 'when the working directory is on a filesystem that does ' + 'not report file creation metadata correctly, e.g. 9p.' + ), + ) diff --git a/tests/containers.py b/tests/containers.py index af515dc2..e387c24b 100644 --- a/tests/containers.py +++ b/tests/containers.py @@ -22,6 +22,17 @@ def read_qa_tag() -> str: ) +def _is_pull_auth_or_missing_error(e: docker.errors.APIError) -> bool: + if e.status_code in (401, 403, 404): + return True + + # Some daemon/registry combinations (e.g. containerd-backed pulls) + # wrap a registry-level 401/404 in a generic 500 Server Error, only + # surfacing the real cause in the explanation text. + explanation = (e.explanation or '').lower() + return 'unauthorized' in explanation or 'not found' in explanation + + def pull_or_build( docker_client: docker.DockerClient, tag: str, @@ -43,7 +54,7 @@ def pull_or_build( if no_local_builds: raise e - if e.status_code != 401 and e.status_code != 404: + if not _is_pull_auth_or_missing_error(e): raise e print(f'Failed to pull image: {e}') print('Attempting to build image from source')