From e6d010a89853bc461b446213fb235fb234761add Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Thu, 17 Sep 2026 12:27:59 +0000 Subject: [PATCH 1/6] NO-ISSUE: validate footperf low-latency setup Run a RPM-host preflight before hardware and oslat measurements, and apply the TuneD profile only after RT kernel setup and the TuneD upgrade complete. --- ...nable-profile.sh => 022-enable-profile.sh} | 0 .../tests/009-verify-low-latency-setup.sh | 126 ++++++++++++++++++ 2 files changed, 126 insertions(+) rename scripts/ci-footprint-and-performance/setup/{020-enable-profile.sh => 022-enable-profile.sh} (100%) create mode 100755 scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh diff --git a/scripts/ci-footprint-and-performance/setup/020-enable-profile.sh b/scripts/ci-footprint-and-performance/setup/022-enable-profile.sh similarity index 100% rename from scripts/ci-footprint-and-performance/setup/020-enable-profile.sh rename to scripts/ci-footprint-and-performance/setup/022-enable-profile.sh diff --git a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh new file mode 100755 index 0000000000..09297a6eab --- /dev/null +++ b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh @@ -0,0 +1,126 @@ +#!/usr/bin/bash + +set -xeuo pipefail + +VARIABLES_FILE=/etc/tuned/microshift-baseline-variables.conf + +fail() { + local -r message=$1 + + printf 'LOW-LATENCY SETUP FAILURE: %s\n' "${message}" >&2 + exit 1 +} + +get_tuned_variable() { + local -r name=$1 + local value + + value=$(awk -F= -v name="${name}" '$1 == name { print substr($0, index($0, "=") + 1); exit }' "${VARIABLES_FILE}") + if [[ -z "${value}" ]]; then + fail "${name} is missing from ${VARIABLES_FILE}; rerun setup/001-configure-wp-lowlat.sh." + fi + + printf '%s\n' "${value}" +} + +require_kernel_argument() { + local -r argument=$1 + local -r cmdline=$2 + + if [[ " ${cmdline} " != *" ${argument} "* ]]; then + fail "missing kernel argument '${argument}' in /proc/cmdline; reapply microshift-baseline and reboot. Current command line: ${cmdline}" + fi +} + +check_cpu_state() { + local -r cpu=$1 + local -r expected_state=$2 + local -r state_path="/sys/devices/system/cpu/cpu${cpu}/online" + local state + + if [[ "${cpu}" -eq 0 ]]; then + if [[ "${expected_state}" != 1 ]]; then + fail "CPU 0 cannot be offlined, but the configured offline CPU set includes it." + fi + return + fi + + if [[ ! -r "${state_path}" ]]; then + fail "CPU ${cpu} is not present at ${state_path}; check the configured CPU sets." + fi + + state=$(<"${state_path}") + if [[ "${state}" != "${expected_state}" ]]; then + fail "CPU ${cpu} is ${state}, expected ${expected_state}; reapply microshift-baseline and reboot." + fi +} + +check_cpu_list_state() { + local -r cpu_list=$1 + local -r expected_state=$2 + local range start end cpu + local IFS=, + local -a ranges + + read -r -a ranges <<< "${cpu_list}" + for range in "${ranges[@]}"; do + range=${range//[[:space:]]/} + if [[ "${range}" == *-* ]]; then + start=${range%-*} + end=${range#*-} + else + start=${range} + end=${range} + fi + + if ! [[ "${start}" =~ ^[0-9]+$ && "${end}" =~ ^[0-9]+$ && "${start}" -le "${end}" ]]; then + fail "invalid CPU range '${range}' in ${VARIABLES_FILE}." + fi + + for ((cpu = start; cpu <= end; cpu++)); do + check_cpu_state "${cpu}" "${expected_state}" + done + done +} + +if [[ ! -r "${VARIABLES_FILE}" ]]; then + fail "${VARIABLES_FILE} is missing; setup/001-configure-wp-lowlat.sh did not complete." +fi + +isolated_cores=$(get_tuned_variable isolated_cores) +hugepages_size=$(get_tuned_variable hugepages_size) +hugepages=$(get_tuned_variable hugepages) +offline_cpu_set=$(get_tuned_variable offline_cpu_set) + +if ! kernel_rt_versions=$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt); then + fail "kernel-rt is not installed; rerun setup/021-install-kernel-rt.sh." +fi +expected_kernel="$(printf '%s\n' "${kernel_rt_versions}" | sort | tail -n 1)+rt" +running_kernel=$(uname -r) +if [[ "${running_kernel}" != "${expected_kernel}" ]]; then + fail "running kernel '${running_kernel}' does not match selected kernel-rt '${expected_kernel}'; reboot into the RT kernel." +fi + +if ! default_kernel=$(sudo grubby --default-kernel); then + fail "could not read the default kernel with grubby; verify the RPM bootloader configuration." +fi +if [[ "${default_kernel}" != "/boot/vmlinuz-${expected_kernel}" ]]; then + fail "default kernel '${default_kernel}' does not match '${expected_kernel}'; rerun setup/021-install-kernel-rt.sh and reboot." +fi + +if ! active_profile=$(sudo tuned-adm active); then + fail "could not determine the active TuneD profile; ensure tuned is running and reapply microshift-baseline." +fi +if [[ "${active_profile}" != "Current active profile: microshift-baseline" ]]; then + fail "active TuneD profile is '${active_profile}', expected microshift-baseline; rerun setup/022-enable-profile.sh and reboot." +fi + +cmdline=$( Date: Thu, 17 Sep 2026 12:40:30 +0000 Subject: [PATCH 2/6] NO-ISSUE: use version sort for kernel-rt Use coreutils version ordering consistently when selecting the installed RT kernel so numeric release components do not select an older build. --- .../ci-footprint-and-performance/setup/021-install-kernel-rt.sh | 2 +- .../tests/009-verify-low-latency-setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh index d8f0ab9ad4..1239fc1829 100755 --- a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh +++ b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh @@ -4,7 +4,7 @@ set -xeuo pipefail sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms sudo dnf install kernel-rt realtime-setup realtime-tests -y -sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel-rt | sort | tail -1)+rt" +sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt | sort -V | tail -n 1)+rt" sudo dnf upgrade tuned -y diff --git a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh index 09297a6eab..b23a3dec70 100755 --- a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh +++ b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh @@ -95,7 +95,7 @@ offline_cpu_set=$(get_tuned_variable offline_cpu_set) if ! kernel_rt_versions=$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt); then fail "kernel-rt is not installed; rerun setup/021-install-kernel-rt.sh." fi -expected_kernel="$(printf '%s\n' "${kernel_rt_versions}" | sort | tail -n 1)+rt" +expected_kernel="$(printf '%s\n' "${kernel_rt_versions}" | sort -V | tail -n 1)+rt" running_kernel=$(uname -r) if [[ "${running_kernel}" != "${expected_kernel}" ]]; then fail "running kernel '${running_kernel}' does not match selected kernel-rt '${expected_kernel}'; reboot into the RT kernel." From ce81d0e51e25abb93900bb87eddf4994ca948b77 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Thu, 17 Sep 2026 13:18:05 +0000 Subject: [PATCH 3/6] NO-ISSUE: redact low-latency kernel cmdline --- .../tests/009-verify-low-latency-setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh index b23a3dec70..e0dacf1133 100755 --- a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh +++ b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh @@ -28,7 +28,7 @@ require_kernel_argument() { local -r cmdline=$2 if [[ " ${cmdline} " != *" ${argument} "* ]]; then - fail "missing kernel argument '${argument}' in /proc/cmdline; reapply microshift-baseline and reboot. Current command line: ${cmdline}" + fail "missing or invalid kernel argument '${argument}' in /proc/cmdline; reapply microshift-baseline and reboot." fi } @@ -115,12 +115,15 @@ if [[ "${active_profile}" != "Current active profile: microshift-baseline" ]]; t fail "active TuneD profile is '${active_profile}', expected microshift-baseline; rerun setup/022-enable-profile.sh and reboot." fi +set +x cmdline=$( Date: Fri, 18 Sep 2026 10:37:36 +0000 Subject: [PATCH 4/6] NO-ISSUE: pin TuneD for footprint CI --- .../setup/021-install-kernel-rt.sh | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh index 1239fc1829..d05a397176 100755 --- a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh +++ b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh @@ -6,7 +6,44 @@ sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms sudo dnf install kernel-rt realtime-setup realtime-tests -y sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt | sort -V | tail -n 1)+rt" -sudo dnf upgrade tuned -y +# TEMPORARY CI MITIGATION: tuned 2.28.0 does not persist the microshift-baseline +# kernel arguments to the RHEL 9.8 BLS entry. Keep all TuneD packages in this +# transaction synchronized; the profile packages require the matching tuned NVR. +readonly TUNED_NVR="2.27.0-2.1.20270724git0eb28ac3.el9fdp" +readonly -a TUNED_PACKAGES=( + "tuned-${TUNED_NVR}" + "tuned-profiles-cpu-partitioning-${TUNED_NVR}" + "tuned-profiles-realtime-${TUNED_NVR}" +) + +# Check that every exact NVR is retained in the currently enabled repositories. +for package in "${TUNED_PACKAGES[@]}"; do + if ! sudo dnf -q repoquery --available --qf '%{name}-%{version}-%{release}.%{arch}' "${package}.noarch" | grep -Fxq "${package}.noarch"; then + echo "ERROR: required TuneD package ${package}.noarch is unavailable in the enabled repositories." >&2 + exit 1 + fi +done + +# Resolve the exact downgrade before changing the host. DNF only erases packages +# when explicitly given --allowerasing, which this transaction deliberately omits. +preflight_output=$(mktemp) +trap 'rm -f "${preflight_output}"' EXIT + +set +e +sudo dnf --assumeno downgrade "${TUNED_PACKAGES[@]}" 2>&1 | tee "${preflight_output}" +preflight_status=${PIPESTATUS[0]} +set -e + +if ! grep -Eq 'Operation aborted|Nothing to do' "${preflight_output}"; then + echo "ERROR: TuneD downgrade preflight failed (dnf exit status ${preflight_status})." >&2 + exit 1 +fi +if grep -Eq '^[[:space:]]*(Removing|Obsoleting):' "${preflight_output}"; then + echo "ERROR: TuneD downgrade preflight would remove or obsolete packages; refusing to continue." >&2 + exit 1 +fi + +sudo dnf downgrade -y "${TUNED_PACKAGES[@]}" # After this point, nothing new will be installed or updated, # so let's list installed packages for debugging purposes. From 032c42c0fbc5f0e0beb59ea281d36ff5a876767d Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Fri, 18 Sep 2026 10:52:53 +0000 Subject: [PATCH 5/6] NO-ISSUE: make TuneD pin arch-specific --- .../setup/021-install-kernel-rt.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh index d05a397176..8b1adc5721 100755 --- a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh +++ b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh @@ -11,21 +11,21 @@ sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{re # transaction synchronized; the profile packages require the matching tuned NVR. readonly TUNED_NVR="2.27.0-2.1.20270724git0eb28ac3.el9fdp" readonly -a TUNED_PACKAGES=( - "tuned-${TUNED_NVR}" - "tuned-profiles-cpu-partitioning-${TUNED_NVR}" - "tuned-profiles-realtime-${TUNED_NVR}" + "tuned-${TUNED_NVR}.noarch" + "tuned-profiles-cpu-partitioning-${TUNED_NVR}.noarch" + "tuned-profiles-realtime-${TUNED_NVR}.noarch" ) # Check that every exact NVR is retained in the currently enabled repositories. for package in "${TUNED_PACKAGES[@]}"; do - if ! sudo dnf -q repoquery --available --qf '%{name}-%{version}-%{release}.%{arch}' "${package}.noarch" | grep -Fxq "${package}.noarch"; then - echo "ERROR: required TuneD package ${package}.noarch is unavailable in the enabled repositories." >&2 + if ! sudo dnf -q repoquery --available --qf '%{name}-%{version}-%{release}.%{arch}' "${package}" | grep -Fxq "${package}"; then + echo "ERROR: required TuneD package ${package} is unavailable in the enabled repositories." >&2 exit 1 fi done -# Resolve the exact downgrade before changing the host. DNF only erases packages -# when explicitly given --allowerasing, which this transaction deliberately omits. +# Resolve the exact downgrade before changing the host. This transaction +# deliberately does not authorize replacing installed packages by erasing them. preflight_output=$(mktemp) trap 'rm -f "${preflight_output}"' EXIT From f0914ffd9d1eed57764f3c82eacedd682d12b1ba Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Fri, 18 Sep 2026 11:15:09 +0000 Subject: [PATCH 6/6] NO-ISSUE: harden TuneD downgrade preflight --- .../setup/021-install-kernel-rt.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh index 8b1adc5721..817c106d3d 100755 --- a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh +++ b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh @@ -34,7 +34,22 @@ sudo dnf --assumeno downgrade "${TUNED_PACKAGES[@]}" 2>&1 | tee "${preflight_out preflight_status=${PIPESTATUS[0]} set -e -if ! grep -Eq 'Operation aborted|Nothing to do' "${preflight_output}"; then +if grep -Eqi '(^|[[:space:]])(Error|Problem):|No match for argument|No matches found|No matching [Pp]ackages|protected package|read-only|read only|permission denied|conflicting requests|dependency conflict|nothing provides|cannot install the best candidate|unable to resolve transaction' "${preflight_output}"; then + echo "ERROR: TuneD downgrade preflight reported a blocking DNF diagnostic; refusing to continue." >&2 + exit 1 +fi +if grep -Eq '^[[:space:]]*Nothing to do\.?[[:space:]]*$' "${preflight_output}"; then + : +# DNF --assumeno refuses a resolved transaction with this exact summary and +# terminal message; other uses of "Operation aborted." are blocking failures. +elif [[ "${preflight_status}" -ne 0 ]] && awk ' + /^[[:space:]]*Transaction Summary[[:space:]]*$/ { transaction_summary = 1 } + transaction_summary && /^[[:space:]]*Downgrade[[:space:]]+/ { downgrade = 1 } + downgrade && /^[[:space:]]*Operation aborted\.[[:space:]]*$/ { refusal_line = NR } + END { exit !(transaction_summary && downgrade && refusal_line == NR) } +' "${preflight_output}"; then + : +else echo "ERROR: TuneD downgrade preflight failed (dnf exit status ${preflight_status})." >&2 exit 1 fi