DNM: Add OCP upgrade stage for OSP 18

  Port upgrade functionality from openshift-ir-plugin to support sequential
  OCP upgrades on OSP 18. Supports versions 4.16-4.23, automatically builds
  upgrade paths skipping unsupported versions like 4.17.

  Includes version-specific workarounds for 4.16 consoleplugins and 4.19
  k8s API acknowledgment. Collects must-gather on failure.
Change-Id: I8c0b804cb91774fb5bb5fccf47cdb6015d46d771
Signed-off-by: Daniel Lawton <dlawton@redhat.com>

Author: dlaw4608

collection/stages/roles/upgrade/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# Target version to upgrade to (e.g., "4.21")
+# This should be set in job definitions
+upgrade_target_version: ""
+
+# Optional: specific build name to upgrade to (e.g., "4.19.3")
+# If empty, will use latest from channel
+upgrade_build_name: ""
+
+# Optional: release channel (candidate, fast, stable, eus)
+# Used if upgrade_build_name is empty
+upgrade_channel: "candidate"
+
+# Wait parameters for health checks
+wait_retries: 90
+wait_delay: 120
+
+# OSP 18 supported OCP versions
+# Note: 4.17 is NOT supported on OSP 18
+osp18_supported_versions:
+  - "4.16"
+  - "4.18"
+  - "4.19"
+  - "4.20"
+  - "4.21"
+  - "4.22"
+  - "4.23"

collection/stages/roles/upgrade/meta/main.yml

@@ -0,0 +1,3 @@
+---
+collections:
+  - shiftstack.tools

collection/stages/roles/upgrade/tasks/main.yml

@@ -0,0 +1,51 @@
+---
+# Main entry point - orchestrates sequential OCP upgrades
+# This role upgrades from the current cluster version to upgrade_target_version
+# following OSP 18 supported version path (skipping unsupported versions like 4.17)
+
+- name: Validate required variables
+  ansible.builtin.assert:
+    that:
+      - upgrade_target_version is defined
+      - upgrade_target_version | length > 0
+    fail_msg: "upgrade_target_version must be defined (e.g., '4.21')"
+
+- name: Discover current OCP version
+  ansible.builtin.include_role:
+    name: shiftstack.tools.tools_get_deploy_info
+    tasks_from: discover_ocp_version.yml
+
+- name: Display current cluster version
+  ansible.builtin.debug:
+    msg: "Current OCP version: {{ discovered_openshift_release }}"
+
+- name: Build upgrade path from current version to target (only OSP 18 supported versions)
+  ansible.builtin.set_fact:
+    upgrade_path: "{{ osp18_supported_versions |
+                      select('version', discovered_openshift_release, '>') |
+                      select('version', upgrade_target_version, '<=') |
+                      list }}"
+
+- name: Validate upgrade path is not empty
+  ansible.builtin.assert:
+    that:
+      - upgrade_path | length > 0
+    fail_msg: |
+      No upgrade path found from {{ discovered_openshift_release }} to {{ upgrade_target_version }}.
+      OSP 18 supported versions: {{ osp18_supported_versions | join(', ') }}
+    success_msg: "Upgrade path validated: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}"
+
+- name: Display upgrade path
+  ansible.builtin.debug:
+    msg: "Upgrade path: {{ discovered_openshift_release }} → {{ upgrade_path | join(' → ') }}"
+
+- name: Execute sequential upgrades
+  ansible.builtin.include_tasks: single_upgrade.yml
+  loop: "{{ upgrade_path }}"
+  loop_control:
+    loop_var: target_release
+    label: "Upgrade to {{ target_release }}"
+
+- name: Display upgrade completion
+  ansible.builtin.debug:
+    msg: "Successfully upgraded cluster from {{ discovered_openshift_release }} to {{ upgrade_target_version }}"

collection/stages/roles/upgrade/tasks/single_upgrade.yml

@@ -0,0 +1,228 @@
+---
+# Single upgrade step: upgrade from current version to target_release
+# This file is included in a loop from main.yml with target_release variable
+
+- name: Check cluster health before upgrading to {{ target_release }}
+  ansible.builtin.include_role:
+    name: shiftstack.tools.tools_cluster_checks
+  vars:
+    actions:
+      - wait_until_cluster_is_healthy
+      - wait_until_nodes_ready
+      - wait_until_no_unschedulable_nodes
+    wait_retries: 1
+    wait_delay: 0
+
+- name: Get the ClusterVersion object before upgrade
+  ansible.builtin.shell: |
+    oc get clusterversion version -o json
+  register: cluster_version_json_before
+  changed_when: false
+
+- name: Parse ClusterVersion before upgrade
+  ansible.builtin.set_fact:
+    cluster_version_before_upgrade: "{{ cluster_version_json_before.stdout | from_json }}"
+
+- name: Get current cluster version from history
+  ansible.builtin.set_fact:
+    current_version: "{{ cluster_version_before_upgrade.status.history[0].version }}"
+
+- name: Check the version to upgrade to is >= the current version
+  ansible.builtin.assert:
+    that:
+      - target_release is version(current_version | regex_replace('^(\\d+\\.\\d+).*', '\\1'), '>=')
+    fail_msg: "The version to upgrade to ({{ target_release }}) must be >= the current version ({{ current_version }})"
+
+- name: Get the OCP release build name to upgrade to
+  ansible.builtin.include_role:
+    name: shiftstack.tools.tools_get_openshift_release
+    tasks_from: get_openshift_release_build_name.yml
+  vars:
+    openshift_release: "{{ target_release }}"
+    openshift_build_name: "{{ upgrade_build_name }}"
+
+- name: Print upgrade versions
+  ansible.builtin.debug:
+    msg: "Upgrading FROM: '{{ current_version }}' TO: '{{ openshift_release_build_name }}'"
+
+- name: Mark the OpenShift version to upgrade to
+  ansible.builtin.debug:
+    msg: "Build mark: upgrade_to={{ openshift_release_build_name }}"
+
+# Version-specific workarounds for OSP 18
+- name: Set patch data for consoleplugins (4.16 only)
+  ansible.builtin.set_fact:
+    console_patch:
+      storedVersions:
+        - v1
+  when: openshift_release_build_name is regex('^4\\.16\\..*')
+
+- name: Set consoleplugins to v1 for 4.16 upgrade
+  ansible.builtin.shell: |
+    oc patch crd consoleplugins.console.openshift.io --subresource='status' --type="merge" -p '{{ console_patch | to_json }}'
+  when: openshift_release_build_name is regex('^4\\.16\\..*')
+  changed_when: true
+
+- name: Set ack data for 4.19 upgrade handholding
+  ansible.builtin.set_fact:
+    ack419_patch:
+      data:
+        ack-4.18-kube-1.32-api-removals-in-4.19: "true"
+  when: openshift_release_build_name is regex('^4\\.19\\..*')
+
+- name: Manual ack for 4.19 upgrade (k8s 1.32 API removals)
+  ansible.builtin.shell: |
+    oc -n openshift-config patch cm admin-acks --type="merge" -p '{{ ack419_patch | to_json }}'
+  when: openshift_release_build_name is regex('^4\\.19\\..*')
+  changed_when: true
+
+# Get the digest ID from the release image
+- name: Create secret.json for OCP release info
+  ansible.builtin.copy:
+    content: "{{ ocp_pull_secret }}"
+    dest: /tmp/secret.json
+    mode: '0600'
+
+- name: Get the OCP release digest
+  ansible.builtin.shell: |
+    oc adm release info registry.ci.openshift.org/ocp/release:{{ openshift_release_build_name }} --registry-config=/tmp/secret.json | grep Digest | awk '{print $2}'
+  register: release_digest
+  changed_when: false
+
+- name: Display release digest
+  ansible.builtin.debug:
+    msg: "{{ openshift_release_build_name }} release digest: {{ release_digest.stdout }}"
+
+- name: Run openshift upgrade command
+  ansible.builtin.shell: |
+    oc adm upgrade --to-image=registry.ci.openshift.org/ocp/release@{{ release_digest.stdout }} --allow-explicit-upgrade --force
+  register: upgrade
+  changed_when: true
+
+- name: Display upgrade command output
+  ansible.builtin.debug:
+    var: upgrade
+
+- name: Check the upgrade command returns ok
+  ansible.builtin.assert:
+    that:
+      - upgrade.rc == 0
+    fail_msg: "oc adm upgrade command returned {{ upgrade.rc }} and stderr {{ upgrade.stderr_lines }}"
+
+- name: Set upgrade start datetime
+  ansible.builtin.set_fact:
+    start_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
+
+- name: Wait until the upgrade starts (Progressing=True)
+  ansible.builtin.shell: |
+    oc get clusterversion version -o json | jq '.status.conditions[] | select(.type=="Progressing" and .status=="True")'
+  register: progressing_check
+  until: progressing_check.stdout | length > 0
+  retries: 20
+  delay: 15
+  changed_when: false
+
+# Main upgrade validation block with must-gather rescue
+- name: Block for upgrade checks with must-gather on failure
+  block:
+    - name: Wait until the upgrade is completed (cluster is healthy)
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - wait_until_cluster_is_healthy
+        wait_retries: 90
+        wait_delay: 120
+
+    - name: Wait until cluster nodes are ready after OCP upgrade
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - wait_until_nodes_ready
+        wait_retries: 80
+        wait_delay: 30
+
+    - name: Wait until there are no unschedulable nodes after OCP upgrade
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - wait_until_no_unschedulable_nodes
+        wait_retries: 80
+        wait_delay: 30
+
+    - name: Set upgrade finish datetime
+      ansible.builtin.set_fact:
+        finish_time: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
+
+    - name: Set the upgrade execution time in minutes
+      ansible.builtin.set_fact:
+        upgrade_time: "{{ ( ((finish_time | to_datetime('%Y-%m-%dT%H:%M:%S')) - (start_time | to_datetime('%Y-%m-%dT%H:%M:%S'))).total_seconds() / 60 ) | int }}"
+
+    - name: Mark the upgrade execution time
+      ansible.builtin.debug:
+        msg: "Build mark: upgrade_minutes={{ upgrade_time }}"
+
+    - name: Check there are no pods in CrashLoopBackOff
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - check_no_crashloopbackoff
+
+    - name: Wait until the running cluster pods are ready after OCP upgrade
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - wait_until_pods_ready
+        wait_retries: 20
+        wait_delay: 30
+
+    - name: Get the ClusterVersion object after upgrade
+      ansible.builtin.shell: |
+        oc get clusterversion version -o json
+      register: cluster_version_json_after
+      changed_when: false
+
+    - name: Parse ClusterVersion after upgrade
+      ansible.builtin.set_fact:
+        cluster_version_after: "{{ cluster_version_json_after.stdout | from_json }}"
+
+    - name: Check the upgraded cluster version is the desired one
+      ansible.builtin.assert:
+        that:
+          - cluster_version_after.status.history[0].version == openshift_release_build_name
+        fail_msg: "Cluster upgraded to '{{ cluster_version_after.status.history[0].version }}' instead of '{{ openshift_release_build_name }}'"
+        success_msg: "Cluster successfully upgraded to '{{ cluster_version_after.status.history[0].version }}'"
+
+    - name: Check cluster operators are ready after the OCP upgrade
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_cluster_checks
+      vars:
+        actions:
+          - check_cluster_operators
+        release_name: "{{ openshift_release_build_name | default('') }}"
+
+  rescue:
+    - name: Run must-gather for failed upgrade
+      ansible.builtin.include_role:
+        name: shiftstack.tools.tools_must-gather
+      vars:
+        must_gather_dir_suffix: "must-gather-upgrade-{{ target_release }}"
+
+    - name: Fail inside rescue block
+      ansible.builtin.fail:
+        msg: "The cluster is not healthy after the upgrade to {{ target_release }}. Check must-gather artifacts."
+
+# Post-upgrade tasks
+- name: Get the installer and oc client binaries for {{ openshift_release_build_name }} release
+  ansible.builtin.include_role:
+    name: shiftstack.tools.tools_get_openshift_release
+    tasks_from: get_openshift_release_binaries.yml
+  vars:
+    binaries:
+      - installer
+      - client
+    release_name: "{{ openshift_release_build_name | default('') }}"

jobs_definitions/upgrade_4.16_to_4.18_osp18.yaml

@@ -0,0 +1,16 @@
+---
+# Example job: Single-hop upgrade on OSP 18
+# Upgrades from 4.16 to 4.18 (skipping 4.17 which is not supported on OSP 18)
+openshift_release: 4.16
+installation_type: ipi
+stages:
+  - cleanup
+  - prepare
+  - install
+  - post
+  - verification
+  - upgrade
+
+# Target version for upgrade stage
+upgrade_target_version: "4.18"
+upgrade_channel: candidate

jobs_definitions/upgrade_4.16_to_4.21_osp18.yaml

@@ -0,0 +1,20 @@
+---
+# Example job: Full OSP 18 upgrade chain
+# Sequential upgrade: 4.16 → 4.18 → 4.19 → 4.20 → 4.21
+# Note: OSP 18 supports 4.16, 4.18, 4.19, 4.20, 4.21 (NOT 4.17)
+openshift_release: 4.16
+installation_type: ipi
+stages:
+  - cleanup
+  - prepare
+  - install
+  - post
+  - verification
+  - upgrade
+
+# Target version for upgrade stage
+upgrade_target_version: "4.21"
+upgrade_channel: candidate
+
+# Optional: Enable availability zones for testing CPMS updates
+# provision_az_enable: true

jobs_definitions/upgrade_4.19_to_4.21_osp18.yaml

@@ -0,0 +1,16 @@
+---
+# Example job: Upgrade from 4.19 to 4.21 on OSP 18
+# Tests the 4.19 ACK workaround when upgrading TO 4.19
+openshift_release: 4.19
+installation_type: ipi
+stages:
+  - cleanup
+  - prepare
+  - install
+  - post
+  - verification
+  - upgrade
+
+# Target version for upgrade stage
+upgrade_target_version: "4.21"
+upgrade_channel: candidate

jobs_definitions/upgrade_4.22_to_4.23_osp18.yaml

@@ -0,0 +1,16 @@
+---
+# Example job: Upgrade from 4.22 to 4.23 on OSP 18
+# Single-hop upgrade for testing latest versions
+openshift_release: 4.22
+installation_type: ipi
+stages:
+  - cleanup
+  - prepare
+  - install
+  - post
+  - verification
+  - upgrade
+
+# Target version for upgrade stage
+upgrade_target_version: "4.23"
+upgrade_channel: candidate

playbooks/ocp_testing.yaml

@@ -84,6 +84,10 @@
   ansible.builtin.import_playbook: plays/verification.yaml
   when: "'verification' in stages"
 
+- name: Upgrade OpenShift cluster
+  ansible.builtin.import_playbook: plays/upgrade.yaml
+  when: "'upgrade' in stages"
+
 - name: Day2ops OpenShift stage
   ansible.builtin.import_playbook: plays/day2ops.yaml
   when: "'day2ops' in stages"