Add Availability Zones (AZ) scenario support

Port the Nova + Cinder Availability Zones scenario from the legacy
openshift-ir-plugin / rhos-infrared automation into the shiftstack-qa
stages. The AZ job installs OCP IPI on OpenStack with one Nova aggregate
per compute host (one AZ each), OCP masters/workers distributed evenly
across the AZs, and rootVolumes placed on per-AZ Cinder backends. Post-
install verification asserts the OCP nodes and their root volumes landed
in the expected Nova and Cinder AZs.

Stages/roles:
- prepare: configure_az.yml creates host aggregates + per-AZ volume types,
  configure_az_ceph_crush.yml pins each per-AZ pool to its host-local OSDs.
- install: discovers Nova/Cinder AZs (tools_cluster_checks/discover_az.yml)
  and renders zones + per-AZ rootVolume types/zones into install-config.
- verification: check_azs.yml verifies Nova and Cinder AZ placement.
- day2ops: scaleup_worker and cinder_registry (AZ-aware registry storage).
- cleanup: remove_az.yml tears down aggregates and AZ volume types, gated
  by az_remove_on_cleanup.
- jobs_definitions/availability_zones_4.22_nightly.yaml wires it together.

All AZ behaviour is gated behind az_enable / openshift_root_volumes
(default false), so non-AZ jobs are unaffected.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: I45bdf61b3bc8e270c950e2e07590a736a9cd81ce
Signed-off-by: Daniel Lawton <dlawton@redhat.com>

Author: dlaw4608

collection/stages/roles/cleanup/defaults/main.yml

@@ -4,3 +4,6 @@
 # Set the OpenShift installation type to None
 existing_ocp_installation_type: ""
 force_cleanup: false
+az_remove_on_cleanup: false
+az_volume_type_prefix: "fastpool-"
+az_volume_type_count: 3

collection/stages/roles/cleanup/tasks/main.yml

@@ -68,6 +68,10 @@
             - name: Discover the OpenShift installation type
               ansible.builtin.include_tasks: detect_ocp_installation.yml
 
+            - name: Remove availability zone configuration from OpenStack
+              ansible.builtin.include_tasks: remove_az.yml
+              when: az_remove_on_cleanup | default(false) | bool
+
             - name: Cleanup IPv6 secondary network resources if they exist
               ansible.builtin.include_tasks: cleanup_ipv6_secondary.yml
               when: ocp_deployment_topology.secondary_ip_protocol | default('') == 'ipv6'

collection/stages/roles/cleanup/tasks/remove_az.yml

@@ -0,0 +1,46 @@
+---
+- name: Discover existing host aggregates
+  ansible.builtin.command:
+    cmd: openstack aggregate list -c Name -f value
+  environment:
+    OS_CLOUD: "{{ admin_cloud }}"
+  register: az_existing_aggregates
+  changed_when: false
+
+- name: Remove hosts from existing aggregates
+  openstack.cloud.host_aggregate:
+    cloud: "{{ admin_cloud }}"
+    state: present
+    name: "{{ item }}"
+    hosts: []
+  loop: "{{ az_existing_aggregates.stdout_lines }}"
+  when: az_existing_aggregates.stdout_lines | length > 0
+
+- name: Delete existing host aggregates
+  openstack.cloud.host_aggregate:
+    cloud: "{{ admin_cloud }}"
+    state: absent
+    name: "{{ item }}"
+  loop: "{{ az_existing_aggregates.stdout_lines }}"
+  when: az_existing_aggregates.stdout_lines | length > 0
+
+- name: Show nova host AZ mapping after aggregate cleanup
+  ansible.builtin.command:
+    cmd: openstack host list -c Zone -c "Host Name" -f table
+  environment:
+    OS_CLOUD: "{{ admin_cloud }}"
+  register: az_host_list_after_cleanup
+  changed_when: false
+
+- name: Print nova host AZ mapping after aggregate cleanup
+  ansible.builtin.debug:
+    var: az_host_list_after_cleanup.stdout_lines
+
+- name: Delete AZ volume types
+  ansible.builtin.command:
+    cmd: openstack volume type delete {{ az_volume_type_prefix }}{{ item }}
+  environment:
+    OS_CLOUD: "{{ admin_cloud }}"
+  loop: "{{ range(0, az_volume_type_count | int) | list }}"
+  failed_when: false
+  changed_when: true

collection/stages/roles/day2ops/defaults/main.yml

@@ -2,6 +2,7 @@
 # defaults file for day2ops
 day2ops_steps: []
 day2ops_report_filename: shiftstack-qa-day2ops-results.xml
+registry_test_project: registry-test
 
 # Application Credentials rotation
 app_credential_name: "AppCreds-{{ user_cloud }}"

collection/stages/roles/day2ops/tasks/procedures/cinder_registry.yml

@@ -0,0 +1,257 @@
+---
+- name: Get existing workers
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: v1
+    kind: Node
+    label_selectors:
+      - node-role.kubernetes.io/worker
+  register: workers
+
+- name: Get first worker name and its Cinder zone label
+  ansible.builtin.set_fact:
+    worker_name: "{{ workers.resources[0].metadata.labels['kubernetes.io/hostname'] }}"
+    nova_az_for_registry: "{{ workers.resources[0].metadata.labels['topology.cinder.csi.openstack.org/zone'] }}"
+
+- name: Discover Cinder AZ for the worker root volume
+  block:
+    - name: Query worker root volume AZ
+      ansible.builtin.shell: |
+        set -o pipefail
+        openstack volume show \
+          "$(openstack volume list -c ID -c Name -f value | grep "{{ worker_name }}" | cut -d' ' -f2)" \
+          -c availability_zone -f value
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+      register: registry_worker_volume_az
+      failed_when: false
+      changed_when: false
+
+    - name: Set Cinder AZ for registry PVC
+      ansible.builtin.set_fact:
+        cinder_az_for_registry: "{{ (registry_worker_volume_az.stdout | trim == '') | ternary('nova', registry_worker_volume_az.stdout | trim) }}"
+
+- name: Remove existing topology-aware storage class
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: absent
+    api_version: storage.k8s.io/v1
+    kind: StorageClass
+    name: topology-aware-image-registry
+
+- name: Create a storage class for integrated registry
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    definition:
+      apiVersion: storage.k8s.io/v1
+      kind: StorageClass
+      metadata:
+        name: topology-aware-image-registry
+      provisioner: cinder.csi.openstack.org
+      parameters:
+        availability: "{{ cinder_az_for_registry }}"
+      volumeBindingMode: WaitForFirstConsumer
+
+- name: Create PVC for integrated registry
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    definition:
+      apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
+        name: pvc-registry
+        namespace: openshift-image-registry
+        annotations:
+          imageregistry.openshift.io: "true"
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 1Gi
+        storageClassName: topology-aware-image-registry
+
+- name: Trigger registry reconfiguration
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    definition:
+      apiVersion: imageregistry.operator.openshift.io/v1
+      kind: Config
+      metadata:
+        name: cluster
+      spec:
+        defaultRoute: true
+        disableRedirect: false
+        rolloutStrategy: Recreate
+        replicas: 1
+        storage:
+          managementState: Managed
+          pvc:
+            claim: pvc-registry
+          swift: null
+        status: null
+    state: present
+
+- name: Wait for registry pod on Running phase
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    namespace: openshift-image-registry
+    label_selectors:
+      - docker-registry=default
+  register: registry_pod
+  retries: 60
+  delay: 10
+  until:
+    - registry_pod.resources[0].spec.volumes[0].persistentVolumeClaim is defined
+    - registry_pod.resources[0].spec.volumes[0].persistentVolumeClaim.claimName == "pvc-registry"
+    - registry_pod.resources[0].status is defined
+    - registry_pod.resources[0].status.phase == "Running"
+
+- name: Get integrated registry route
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: route.openshift.io/v1
+    kind: Route
+    namespace: openshift-image-registry
+    name: default-route
+  register: registry_routes
+
+- name: Set integrated registry URL
+  ansible.builtin.set_fact:
+    registry_url: "{{ registry_routes.resources[0].spec.host }}"
+
+- name: Add registry domain resolution to /etc/hosts
+  block:
+    - name: Resolve apps ingress IP from /etc/hosts
+      ansible.builtin.shell: |
+        set -o pipefail
+        awk '{if ($2 ~ "apps.{{ ocp_cluster_name }}") print $1}' /etc/hosts | uniq
+      register: hosts_apps_ip
+      changed_when: false
+
+    - name: Set apps ingress IP
+      ansible.builtin.set_fact:
+        apps_fip_ip: "{{ hosts_apps_ip.stdout }}"
+
+    - name: Add registry floating IP to /etc/hosts
+      ansible.builtin.lineinfile:
+        path: /etc/hosts
+        line: "{{ apps_fip_ip }} {{ registry_url }}"
+      become: true
+
+- name: Delete the registry test project if it exists
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: absent
+    api_version: project.openshift.io/v1
+    kind: Project
+    name: "{{ registry_test_project }}"
+    wait: true
+    wait_timeout: 60
+
+- name: Create the registry test project
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    api_version: project.openshift.io/v1
+    kind: Project
+    name: "{{ registry_test_project }}"
+
+- name: Add registry permissions to admin user
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc project {{ registry_test_project }}
+    oc policy add-role-to-user registry-viewer admin
+    oc policy add-role-to-user registry-editor admin
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: true
+
+- name: Get builder service account token for OCP before 4.11
+  when: openshift_release_build_name is version('4.11', '<')
+  block:
+    - name: Get builder token
+      ansible.builtin.command: oc serviceaccounts get-token builder
+      environment:
+        KUBECONFIG: "{{ kubeconfig }}"
+      register: registry_token_result
+      retries: 5
+      delay: 60
+      until: registry_token_result is succeeded
+      changed_when: false
+
+    - name: Set registry token
+      ansible.builtin.set_fact:
+        registry_token: "{{ registry_token_result.stdout }}"
+
+- name: Get builder service account token for OCP 4.11 and later
+  when: openshift_release_build_name is version('4.11', '>=')
+  block:
+    - name: Create builder token
+      ansible.builtin.command: oc create token builder
+      environment:
+        KUBECONFIG: "{{ kubeconfig }}"
+      register: registry_token_result
+      retries: 5
+      delay: 60
+      until: registry_token_result is succeeded
+      changed_when: false
+
+    - name: Set registry token
+      ansible.builtin.set_fact:
+        registry_token: "{{ registry_token_result.stdout }}"
+
+- name: Push demo image to integrated registry
+  ansible.builtin.shell: |
+    set -o pipefail
+    sudo podman login -u admin -p {{ registry_token }} --tls-verify=false {{ registry_url }} && \
+    sudo podman pull quay.io/kuryr/demo && \
+    sudo podman tag quay.io/kuryr/demo {{ registry_url }}/{{ registry_test_project }}/demo && \
+    sudo podman push {{ registry_url }}/{{ registry_test_project }}/demo --tls-verify=false
+  register: registry_push_result
+  retries: 10
+  delay: 30
+  until: not registry_push_result.failed
+  changed_when: true
+
+- name: Create a pod using integrated registry
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig }}"
+    state: present
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: demo
+        namespace: "{{ registry_test_project }}"
+      spec:
+        replicas: 1
+        selector:
+          matchLabels:
+            app: demo
+        template:
+          metadata:
+            labels:
+              app: demo
+          spec:
+            containers:
+              - name: demo
+                image: image-registry.openshift-image-registry.svc:5000/{{ registry_test_project }}/demo
+                ports:
+                  - containerPort: 80
+
+- name: Wait for demo pod to reach Running phase
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    kind: Pod
+    label_selectors:
+      - app=demo
+  register: demo_pod
+  retries: 5
+  delay: 10
+  until:
+    - demo_pod.resources[0].status is defined
+    - demo_pod.resources[0].status.phase == "Running"

collection/stages/roles/day2ops/tasks/procedures/scaleup_worker.yml

@@ -0,0 +1,52 @@
+---
+- name: Get the worker machineset
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: machine.openshift.io/v1beta1
+    kind: MachineSet
+    namespace: openshift-machine-api
+    label_selectors:
+      - machine.openshift.io/cluster-api-machine-type=worker
+  register: ms_worker
+
+- name: Set the expected workers replicas
+  ansible.builtin.set_fact:
+    worker_spec_replicas: "{{ ms_worker.resources[0].spec.replicas }}"
+
+- name: Check that the number of worker replicas are as expected
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: machine.openshift.io/v1beta1
+    kind: MachineSet
+    namespace: openshift-machine-api
+    name: "{{ ms_worker.resources[0].metadata.name }}"
+  register: workers
+  until: workers | json_query('resources[0].status.readyReplicas') == worker_spec_replicas | int
+  retries: 40
+  delay: 30
+
+- name: Set the number of new workers
+  ansible.builtin.set_fact:
+    new_num_workers: "{{ ms_worker.resources[0].spec.replicas | int + 1 }}"
+
+- name: Scale workers
+  ansible.builtin.command:
+    cmd: >-
+      oc scale --replicas={{ new_num_workers }}
+      machineset/{{ ms_worker.resources[0].metadata.name }}
+      -n openshift-machine-api
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: true
+
+- name: Wait for all the workers to be up
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig }}"
+    api_version: machine.openshift.io/v1beta1
+    kind: MachineSet
+    namespace: openshift-machine-api
+    name: "{{ ms_worker.resources[0].metadata.name }}"
+  register: new_workers
+  until: new_workers | json_query('resources[0].status.readyReplicas') | int == new_num_workers | int
+  retries: 40
+  delay: 30

collection/stages/roles/install/tasks/main.yml

@@ -22,6 +22,12 @@
     state: directory
     mode: u=rwx,g=rw,o=r
 
+- name: Discover Nova and Cinder AZs for install-config generation
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: discover_az.yml
+  when: az_enable | default(false) | bool
+
 - name: Prepare for UPI installation
   ansible.builtin.include_role:
     name: tools_upi