diff --git a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh index d8f0ab9ad4..817c106d3d 100755 --- a/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh +++ b/scripts/ci-footprint-and-performance/setup/021-install-kernel-rt.sh @@ -4,9 +4,61 @@ set -xeuo pipefail sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms sudo dnf install kernel-rt realtime-setup realtime-tests -y -sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel-rt | sort | tail -1)+rt" +sudo grubby --set-default="/boot/vmlinuz-$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt | sort -V | tail -n 1)+rt" -sudo dnf upgrade tuned -y +# TEMPORARY CI MITIGATION: tuned 2.28.0 does not persist the microshift-baseline +# kernel arguments to the RHEL 9.8 BLS entry. Keep all TuneD packages in this +# transaction synchronized; the profile packages require the matching tuned NVR. +readonly TUNED_NVR="2.27.0-2.1.20270724git0eb28ac3.el9fdp" +readonly -a TUNED_PACKAGES=( + "tuned-${TUNED_NVR}.noarch" + "tuned-profiles-cpu-partitioning-${TUNED_NVR}.noarch" + "tuned-profiles-realtime-${TUNED_NVR}.noarch" +) + +# Check that every exact NVR is retained in the currently enabled repositories. +for package in "${TUNED_PACKAGES[@]}"; do + if ! sudo dnf -q repoquery --available --qf '%{name}-%{version}-%{release}.%{arch}' "${package}" | grep -Fxq "${package}"; then + echo "ERROR: required TuneD package ${package} is unavailable in the enabled repositories." >&2 + exit 1 + fi +done + +# Resolve the exact downgrade before changing the host. This transaction +# deliberately does not authorize replacing installed packages by erasing them. +preflight_output=$(mktemp) +trap 'rm -f "${preflight_output}"' EXIT + +set +e +sudo dnf --assumeno downgrade "${TUNED_PACKAGES[@]}" 2>&1 | tee "${preflight_output}" +preflight_status=${PIPESTATUS[0]} +set -e + +if grep -Eqi '(^|[[:space:]])(Error|Problem):|No match for argument|No matches found|No matching [Pp]ackages|protected package|read-only|read only|permission denied|conflicting requests|dependency conflict|nothing provides|cannot install the best candidate|unable to resolve transaction' "${preflight_output}"; then + echo "ERROR: TuneD downgrade preflight reported a blocking DNF diagnostic; refusing to continue." >&2 + exit 1 +fi +if grep -Eq '^[[:space:]]*Nothing to do\.?[[:space:]]*$' "${preflight_output}"; then + : +# DNF --assumeno refuses a resolved transaction with this exact summary and +# terminal message; other uses of "Operation aborted." are blocking failures. +elif [[ "${preflight_status}" -ne 0 ]] && awk ' + /^[[:space:]]*Transaction Summary[[:space:]]*$/ { transaction_summary = 1 } + transaction_summary && /^[[:space:]]*Downgrade[[:space:]]+/ { downgrade = 1 } + downgrade && /^[[:space:]]*Operation aborted\.[[:space:]]*$/ { refusal_line = NR } + END { exit !(transaction_summary && downgrade && refusal_line == NR) } +' "${preflight_output}"; then + : +else + echo "ERROR: TuneD downgrade preflight failed (dnf exit status ${preflight_status})." >&2 + exit 1 +fi +if grep -Eq '^[[:space:]]*(Removing|Obsoleting):' "${preflight_output}"; then + echo "ERROR: TuneD downgrade preflight would remove or obsolete packages; refusing to continue." >&2 + exit 1 +fi + +sudo dnf downgrade -y "${TUNED_PACKAGES[@]}" # After this point, nothing new will be installed or updated, # so let's list installed packages for debugging purposes. diff --git a/scripts/ci-footprint-and-performance/setup/020-enable-profile.sh b/scripts/ci-footprint-and-performance/setup/022-enable-profile.sh similarity index 100% rename from scripts/ci-footprint-and-performance/setup/020-enable-profile.sh rename to scripts/ci-footprint-and-performance/setup/022-enable-profile.sh diff --git a/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh new file mode 100755 index 0000000000..e0dacf1133 --- /dev/null +++ b/scripts/ci-footprint-and-performance/tests/009-verify-low-latency-setup.sh @@ -0,0 +1,129 @@ +#!/usr/bin/bash + +set -xeuo pipefail + +VARIABLES_FILE=/etc/tuned/microshift-baseline-variables.conf + +fail() { + local -r message=$1 + + printf 'LOW-LATENCY SETUP FAILURE: %s\n' "${message}" >&2 + exit 1 +} + +get_tuned_variable() { + local -r name=$1 + local value + + value=$(awk -F= -v name="${name}" '$1 == name { print substr($0, index($0, "=") + 1); exit }' "${VARIABLES_FILE}") + if [[ -z "${value}" ]]; then + fail "${name} is missing from ${VARIABLES_FILE}; rerun setup/001-configure-wp-lowlat.sh." + fi + + printf '%s\n' "${value}" +} + +require_kernel_argument() { + local -r argument=$1 + local -r cmdline=$2 + + if [[ " ${cmdline} " != *" ${argument} "* ]]; then + fail "missing or invalid kernel argument '${argument}' in /proc/cmdline; reapply microshift-baseline and reboot." + fi +} + +check_cpu_state() { + local -r cpu=$1 + local -r expected_state=$2 + local -r state_path="/sys/devices/system/cpu/cpu${cpu}/online" + local state + + if [[ "${cpu}" -eq 0 ]]; then + if [[ "${expected_state}" != 1 ]]; then + fail "CPU 0 cannot be offlined, but the configured offline CPU set includes it." + fi + return + fi + + if [[ ! -r "${state_path}" ]]; then + fail "CPU ${cpu} is not present at ${state_path}; check the configured CPU sets." + fi + + state=$(<"${state_path}") + if [[ "${state}" != "${expected_state}" ]]; then + fail "CPU ${cpu} is ${state}, expected ${expected_state}; reapply microshift-baseline and reboot." + fi +} + +check_cpu_list_state() { + local -r cpu_list=$1 + local -r expected_state=$2 + local range start end cpu + local IFS=, + local -a ranges + + read -r -a ranges <<< "${cpu_list}" + for range in "${ranges[@]}"; do + range=${range//[[:space:]]/} + if [[ "${range}" == *-* ]]; then + start=${range%-*} + end=${range#*-} + else + start=${range} + end=${range} + fi + + if ! [[ "${start}" =~ ^[0-9]+$ && "${end}" =~ ^[0-9]+$ && "${start}" -le "${end}" ]]; then + fail "invalid CPU range '${range}' in ${VARIABLES_FILE}." + fi + + for ((cpu = start; cpu <= end; cpu++)); do + check_cpu_state "${cpu}" "${expected_state}" + done + done +} + +if [[ ! -r "${VARIABLES_FILE}" ]]; then + fail "${VARIABLES_FILE} is missing; setup/001-configure-wp-lowlat.sh did not complete." +fi + +isolated_cores=$(get_tuned_variable isolated_cores) +hugepages_size=$(get_tuned_variable hugepages_size) +hugepages=$(get_tuned_variable hugepages) +offline_cpu_set=$(get_tuned_variable offline_cpu_set) + +if ! kernel_rt_versions=$(rpm -q --queryformat '%{version}-%{release}.%{arch}\n' kernel-rt); then + fail "kernel-rt is not installed; rerun setup/021-install-kernel-rt.sh." +fi +expected_kernel="$(printf '%s\n' "${kernel_rt_versions}" | sort -V | tail -n 1)+rt" +running_kernel=$(uname -r) +if [[ "${running_kernel}" != "${expected_kernel}" ]]; then + fail "running kernel '${running_kernel}' does not match selected kernel-rt '${expected_kernel}'; reboot into the RT kernel." +fi + +if ! default_kernel=$(sudo grubby --default-kernel); then + fail "could not read the default kernel with grubby; verify the RPM bootloader configuration." +fi +if [[ "${default_kernel}" != "/boot/vmlinuz-${expected_kernel}" ]]; then + fail "default kernel '${default_kernel}' does not match '${expected_kernel}'; rerun setup/021-install-kernel-rt.sh and reboot." +fi + +if ! active_profile=$(sudo tuned-adm active); then + fail "could not determine the active TuneD profile; ensure tuned is running and reapply microshift-baseline." +fi +if [[ "${active_profile}" != "Current active profile: microshift-baseline" ]]; then + fail "active TuneD profile is '${active_profile}', expected microshift-baseline; rerun setup/022-enable-profile.sh and reboot." +fi + +set +x +cmdline=$(