diff --git a/01_install_requirements.sh b/01_install_requirements.sh index 3a6aa0f97..ce99dd6a8 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -110,8 +110,15 @@ case $DISTRO in ;; "rhel10"|"centos10") sudo dnf -y install python3-pip - if sudo subscription-manager identity > /dev/null 2>&1; then - sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true + if [[ $DISTRO == "centos10" ]]; then + sudo dnf config-manager --set-enabled crb + sudo dnf -y install epel-release + elif [[ $DISTRO == "rhel10" ]]; then + if sudo subscription-manager identity > /dev/null 2>&1; then + sudo subscription-manager repos --enable "codeready-builder-for-rhel-10-$(arch)-rpms" || true + fi + # EPEL provides tayga (needed for NAT64); mirror the EL9 rhel handling. + sudo dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm fi sudo ln -s /usr/bin/python3 /usr/bin/python || true PYTHON_DEVEL="python3-devel" @@ -207,6 +214,19 @@ if [[ "${NODES_PLATFORM:-}" == "baremetal" ]] ; then sudo dnf -y install ipmitool fi +# Install NAT64 dependencies if enabled +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + echo "Installing NAT64 dependencies (TAYGA, unbound)..." + # TAYGA provides the NAT64 translation; unbound provides DNS64 synthesis. + # unbound is used rather than CoreDNS because its built-in dns64 module + # reliably synthesizes AAAA records (the CoreDNS dns64 plugin build did not). + sudo dnf -y install tayga unbound + # We run our own DNS64 unbound instance on a dedicated port; make sure the + # stock unbound.service does not also grab port 53 and clash with the host + # NetworkManager resolver. + sudo systemctl disable --now unbound.service 2>/dev/null || true +fi + retry_with_timeout 5 60 "curl -L $OPENSHIFT_CLIENT_TOOLS_URL | sudo tar -U -C /usr/local/bin -xzf -" sudo chmod +x /usr/local/bin/oc oc version --client -o json diff --git a/02_configure_host.sh b/02_configure_host.sh index a23dc9997..9db9de145 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -4,6 +4,7 @@ set -euxo pipefail source logging.sh source common.sh source network.sh +source nat64.sh source utils.sh source validation.sh source oc_mirror.sh @@ -100,6 +101,11 @@ if [[ "${NODES_PLATFORM}" == "baremetal" ]]; then # Add a /etc/hosts entry for $LOCAL_REGISTRY_DNS_NAME sudo sed -i "/${LOCAL_REGISTRY_DNS_NAME}/d" /etc/hosts echo "${PROVISIONING_HOST_EXTERNAL_IP} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts + # Under NAT64 the cluster is IPv6-only; also publish an AAAA record so nodes can + # reach host services (e.g. the image-registry NFS export) over the host's IPv6 + # baremetal address. The sed cleanup above matches by name, so it removes both + # lines on re-run (idempotent). + [ -n "${PROVISIONING_HOST_EXTERNAL_IP_V6:-}" ] && echo "${PROVISIONING_HOST_EXTERNAL_IP_V6} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts # When MIRROR_IMAGES is configured, the local registry must be running # before 04_setup_ironic.sh attempts to mirror release images into it. @@ -293,6 +299,13 @@ if [ "${NUM_EXTRA_WORKERS}" -ne 0 ] || [ "${NUM_ARM_WORKERS}" -ne 0 ]; then fi fi +# For NAT64 (IPv6-only cluster on an IPv4-only host) the BMC emulator must be +# reached over IPv6 by the in-cluster Ironic pods; rewrite the node BMC addresses +# to the host's IPv6 baremetal address. +if [[ "${ENABLE_NAT64}" == "true" ]]; then + nat64_fixup_bmc_addresses +fi + # shellcheck disable=SC2034 ZONE="\nZONE=libvirt" @@ -474,6 +487,16 @@ if [ "$EXT_IF" ]; then sudo $IPTABLES -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT fi +# When NAT64 is enabled, ensure IPv6 forwarding rules are set for the bridge +if [[ "${ENABLE_NAT64}" == "true" ]]; then + # Check-then-add so re-running does not accumulate duplicate rules; cleanup_nat64 + # removes these with the matching -D commands. + sudo ip6tables -C FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || \ + sudo ip6tables -A FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT + sudo ip6tables -C FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || \ + sudo ip6tables -A FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT +fi + # Switch NetworkManager to internal DNS if [ "$MANAGE_BR_BRIDGE" == "y" ]; then switch_to_internal_dns @@ -482,6 +505,11 @@ fi # Add a /etc/hosts entry for $LOCAL_REGISTRY_DNS_NAME sudo sed -i "/${LOCAL_REGISTRY_DNS_NAME}/d" /etc/hosts echo "${PROVISIONING_HOST_EXTERNAL_IP} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts +# Under NAT64 the cluster is IPv6-only; also publish an AAAA record so nodes can +# reach host services (e.g. the image-registry NFS export) over the host's IPv6 +# baremetal address. The sed cleanup above matches by name, so it removes both +# lines on re-run (idempotent). +[ -n "${PROVISIONING_HOST_EXTERNAL_IP_V6:-}" ] && echo "${PROVISIONING_HOST_EXTERNAL_IP_V6} ${LOCAL_REGISTRY_DNS_NAME}" | sudo tee -a /etc/hosts if use_registry "podman"; then # Remove any previous file, or podman login panics when reading the @@ -536,6 +564,16 @@ fi sudo virsh net-list | grep "${PROVISIONING_NETWORK_NAME}" || sudo virsh net-start "${PROVISIONING_NETWORK_NAME}" sudo virsh net-list | grep "${BAREMETAL_NETWORK_NAME}" || sudo virsh net-start "${BAREMETAL_NETWORK_NAME}" +# Configure NAT64/DNS64 if enabled +if [[ "${ENABLE_NAT64}" == "true" ]]; then + configure_nat64_bridge_ipv6 + configure_tayga + configure_dns64 + # Make the sushy BMC cert valid for the IPv6 baremetal address so the + # IPv6-only in-cluster Ironic can reach the BMC. Must run before step 05 + # embeds this cert into the install-config trust bundle. + nat64_fixup_sushy_cert +fi # Setup a single nfs export for image registry if [ "${PERSISTENT_IMAGEREG}" == true ] ; then diff --git a/common.sh b/common.sh index c7049c3d5..831f4b111 100644 --- a/common.sh +++ b/common.sh @@ -632,6 +632,56 @@ if [[ ! -z ${AGENT_E2E_TEST_SCENARIO} ]]; then fi fi +# Validate NAT64 configuration. This must run after the agent scenario parsing +# above, which derives IP_STACK from AGENT_E2E_TEST_SCENARIO; validating earlier +# would only ever see the default IP_STACK and silently pass for v4/dual-stack +# agent scenarios. +if [[ "${ENABLE_NAT64:-false}" == "true" ]]; then + if [[ "${IP_STACK:-v6}" != "v6" ]]; then + error "ENABLE_NAT64=true requires IP_STACK=v6 (got IP_STACK=${IP_STACK:-v6})" + exit 1 + fi + if [[ "${HOST_IP_STACK:-${IP_STACK:-v6}}" != "v4" ]]; then + error "ENABLE_NAT64=true requires HOST_IP_STACK=v4 (got HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK:-v6}})" + exit 1 + fi + # The IPv6-only in-cluster Ironic reaches redfish (sushy) over the host's native + # IPv6 address, and reaches ipmi (vbmc, IPv4-only) via NAT64 translation of its + # IPv4 address (see nat64_fixup_bmc_addresses). Allow redfish, redfish-virtualmedia, + # ipmi and mixed; idrac/others are not emulated here, so still reject them. + case "${BMC_DRIVER}" in + redfish|redfish-virtualmedia|ipmi|mixed) ;; + *) + error "ENABLE_NAT64=true supports BMC_DRIVER redfish, redfish-virtualmedia, ipmi or mixed, got BMC_DRIVER=${BMC_DRIVER}" + exit 1 + ;; + esac + # ipmi BMCs are reached through NAT64 by embedding their RFC1918 IPv4 address in + # NAT64_PREFIX. RFC 6052 forbids using the well-known prefix 64:ff9b::/96 for + # non-global (RFC1918) IPv4, so mixed/ipmi need a ULA prefix. Default one here + # (exported before network.sh applies its own 64:ff9b default), and reject an + # explicit well-known prefix rather than failing opaquely at BMC registration. + if [[ "${BMC_DRIVER}" == "mixed" || "${BMC_DRIVER}" == "ipmi" ]]; then + export NAT64_PREFIX=${NAT64_PREFIX:-"fd00:64::/96"} + if [[ "${NAT64_PREFIX}" == 64:ff9b::* ]]; then + error "ENABLE_NAT64=true with BMC_DRIVER=${BMC_DRIVER} cannot use the well-known NAT64 prefix ${NAT64_PREFIX} to reach the RFC1918 BMC network; set a ULA prefix, e.g. NAT64_PREFIX=fd00:64::/96" + exit 1 + fi + # nat64_embed_v4 embeds the BMC's IPv4 in the last 32 bits, which is only the + # correct RFC 6052 layout for a /96 whose text ends in "::". Reject anything + # else here rather than silently producing an unreachable BMC address. + if [[ ! "${NAT64_PREFIX}" =~ ::/96$ ]]; then + error "ENABLE_NAT64=true with BMC_DRIVER=${BMC_DRIVER} requires a /96 NAT64_PREFIX ending in '::' (e.g. fd00:64::/96) to embed the ipmi BMC IPv4 address, got NAT64_PREFIX=${NAT64_PREFIX}" + exit 1 + fi + fi + # NAT64 provides external registry access, so image mirroring is skipped and no + # local registry is created. Default MIRROR_IMAGES to false here (before the + # registry-override decision later in this file) so the installer is not pointed + # at a registry that was never stood up. + export MIRROR_IMAGES=${MIRROR_IMAGES:-false} +fi + if [[ ! -z ${AGENT_E2E_TEST_BOOT_MODE} ]]; then case "$AGENT_E2E_TEST_BOOT_MODE" in "ISO" | "PXE" | "DISKIMAGE" | "ISCSI"| "ISO_NO_REGISTRY") diff --git a/config_example.sh b/config_example.sh index 16a546f78..564b288a4 100755 --- a/config_example.sh +++ b/config_example.sh @@ -441,6 +441,62 @@ set -x #export EXTERNAL_SUBNET_V4="192.168.111.0/24" #export EXTERNAL_SUBNET_V6="fd2e:6f44:5dd8:c956::/120" +# ENABLE_NAT64 - +# Enable NAT64/DNS64 to allow IPv6-only clusters (IP_STACK=v6) to run on +# IPv4-only hosts (HOST_IP_STACK=v4). Uses TAYGA for NAT64 translation and +# unbound for DNS64 synthesis, enabling cluster VMs to reach external IPv4 +# resources (e.g. container registries) via synthesized IPv6 addresses. +# Requires: IP_STACK=v6 and HOST_IP_STACK=v4. Supported BMC_DRIVERs: redfish, +# redfish-virtualmedia, ipmi and mixed. redfish (sushy) is reached over the host's +# native IPv6 address; ipmi (vbmc, IPv4-only) is reached through NAT64 by embedding +# its IPv4 address in NAT64_PREFIX. Because that embedding targets the RFC1918 +# baremetal network, BMC_DRIVER=mixed/ipmi requires a ULA NAT64_PREFIX (defaulted to +# fd00:64::/96) -- the well-known 64:ff9b::/96 cannot carry RFC1918 (RFC 6052). +# Default: false +# +#export ENABLE_NAT64=true + +# NAT64_PREFIX - +# IPv6 prefix used for NAT64 address translation. Packets sent to addresses +# in this prefix are translated to IPv4 by TAYGA. +# Default: "64:ff9b::/96" (redfish drivers), or "fd00:64::/96" when +# BMC_DRIVER=mixed/ipmi (those reach the RFC1918 BMC network through NAT64). +# Note: The well-known prefix 64:ff9b::/96 cannot reach RFC1918 addresses. +# Use a ULA prefix (e.g. "fd00:64::/96") if you need to reach private IPv4. +# Only the /96 length is supported for BMC address embedding. +# +#export NAT64_PREFIX="64:ff9b::/96" + +# NAT64_V4_POOL - +# IPv4 address pool used by TAYGA for dynamic NAT64 mappings. +# Default: "192.168.255.0/24" +# +#export NAT64_V4_POOL="192.168.255.0/24" + +# NAT64_V4_ADDR - +# TAYGA's own IPv4 address on the NAT64 TUN interface. +# Default: "192.168.255.1" +# +#export NAT64_V4_ADDR="192.168.255.1" + +# NAT64_V6_ADDR - +# TAYGA's own IPv6 address (source of the ICMPv6 errors it generates, e.g. +# for PMTUD). Defaults to the 3rd address of EXTERNAL_SUBNET_V6, which is +# on-link on the baremetal bridge. If you carve TAYGA its own subnet, set this +# to an address outside any locally-attached subnet and route it to the NAT64 +# TUN device so error replies are delivered symmetrically. +# Default: nth_ip(EXTERNAL_SUBNET_V6, 3) +# +#export NAT64_V6_ADDR="fd2e:6f44:5dd8:c956::3" + +# NAT64_DNS64_UPSTREAM - +# Space-separated upstream resolver(s) the DNS64 unbound instance forwards to. +# By default these are auto-discovered from the host's resolver config; set this +# explicitly when the host has no usable upstream in resolv.conf (otherwise DNS64 +# falls back to 8.8.8.8, which is usually unreachable on a firewalled NAT64 host). +# +#export NAT64_DNS64_UPSTREAM="10.0.0.53" + # ENABLE_BOOTSTRAP_STATIC_IP - # Configure a static IP for the bootstrap VM external NIC # (Currently this just expects a non-empty value, the IP is fixed to .9) diff --git a/host_cleanup.sh b/host_cleanup.sh index 047379338..921ff1130 100755 --- a/host_cleanup.sh +++ b/host_cleanup.sh @@ -3,6 +3,8 @@ set -x source logging.sh source common.sh +source network.sh +source nat64.sh source utils.sh source validation.sh @@ -32,6 +34,12 @@ ansible-playbook \ -i "${VM_SETUP_PATH}/inventory.ini" \ -b -vvv "${VM_SETUP_PATH}/teardown-playbook.yml" +# Always clean up NAT64/DNS64. cleanup_nat64 is fully idempotent (every step is a +# no-op when nothing is present), so run it unconditionally rather than gating on +# the current ENABLE_NAT64 value: otherwise unsetting ENABLE_NAT64 before teardown +# would strand the unbound service, dnsmasq drop-in, TUN device, routes and rules. +cleanup_nat64 + sudo rm -rf "/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME}.conf" /etc/yum.repos.d/delorean* sudo rm -rf /etc/NetworkManager/conf.d/dnsmasq.conf sudo rm -rf /etc/NetworkManager/dnsmasq.d/upstream.conf diff --git a/nat64.sh b/nat64.sh new file mode 100755 index 000000000..78631e545 --- /dev/null +++ b/nat64.sh @@ -0,0 +1,473 @@ +#!/bin/bash +# NAT64/DNS64 helper functions for enabling IPv6-only clusters +# on IPv4-only hosts using TAYGA (NAT64) and unbound (DNS64). + +export NAT64_TAYGA_CONF="/etc/tayga.conf" +export NAT64_TAYGA_DATA_DIR="/var/db/tayga" +export NAT64_TAYGA_SERVICE="nat64-tayga" +export NAT64_TUN_INTERFACE="nat64" +export NAT64_DNS64_PORT="5353" +export NAT64_UNBOUND_CONF="/etc/unbound/unbound-dns64.conf" +export NAT64_UNBOUND_SERVICE="unbound-dns64" +export NAT64_DNSMASQ_CONF="/etc/NetworkManager/dnsmasq.d/nat64-dns64.conf" + +function configure_nat64_bridge_ipv6() { + local bridge_name="${BAREMETAL_NETWORK_NAME}" + local ipv6_addr + ipv6_addr=$(nth_ip "$EXTERNAL_SUBNET_V6" 1) + + echo "Configuring IPv6 on bridge ${bridge_name} for NAT64..." + + # Enable IPv6 forwarding + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + sudo sysctl -w net.ipv6.conf.default.forwarding=1 + + # Add IPv6 address to the baremetal bridge + local prefix_len + prefix_len=$(echo "${EXTERNAL_SUBNET_V6}" | cut -d'/' -f2) + sudo ip -6 addr add "${ipv6_addr}/${prefix_len}" dev "${bridge_name}" 2>/dev/null || true + + echo "IPv6 address ${ipv6_addr}/${prefix_len} configured on ${bridge_name}" +} + +function configure_tayga() { + echo "Configuring TAYGA NAT64 daemon..." + + # Create TAYGA data directory + sudo mkdir -p "${NAT64_TAYGA_DATA_DIR}" + + # Write TAYGA configuration + sudo tee "${NAT64_TAYGA_CONF}" > /dev/null < /dev/null </dev/null || /usr/sbin/iptables -t nat -A POSTROUTING -s ${NAT64_V4_POOL} -j MASQUERADE' +ExecStart=/usr/sbin/tayga --nodetach -c ${NAT64_TAYGA_CONF} +# Put the freshly (re)created TUN into firewalld's trusted zone. Traffic to an ipmi +# BMC (BMC_DRIVER=mixed/ipmi) is translated by TAYGA and re-enters the host locally +# on this interface destined for the vbmc port on the baremetal address; without a +# permissive zone firewalld drops it (the vbmc ports are only opened in the libvirt +# zone, and an unassigned interface falls into the default 'public' zone), so the +# in-cluster/bootstrap Ironic cannot reach ipmi BMCs. The '-' prefix keeps this a +# no-op when firewalld is not running, and re-runs on every (re)start since the TUN +# is recreated each time. +ExecStartPost=-/usr/bin/firewall-cmd --zone=trusted --change-interface=${NAT64_TUN_INTERFACE} +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + + sudo systemctl daemon-reload + sudo systemctl enable "${NAT64_TAYGA_SERVICE}.service" + # restart (not just start) so a re-run picks up any config/unit changes. + sudo systemctl restart "${NAT64_TAYGA_SERVICE}.service" + + # Persist the trusted-zone binding so it survives a firewalld reload/reboot too + # (the ExecStartPost above only sets the runtime binding on each tayga start). + # Guarded: harmless no-op when firewalld is absent. + if command -v firewall-cmd >/dev/null 2>&1 && sudo firewall-cmd --state >/dev/null 2>&1; then + sudo firewall-cmd --permanent --zone=trusted --change-interface="${NAT64_TUN_INTERFACE}" 2>/dev/null || true + sudo firewall-cmd --zone=trusted --change-interface="${NAT64_TUN_INTERFACE}" 2>/dev/null || true + fi + + echo "TAYGA NAT64 daemon started (prefix=${NAT64_PREFIX}, pool=${NAT64_V4_POOL})" +} + +function configure_dns64() { + echo "Configuring unbound for DNS64..." + + # Remove any legacy CoreDNS DNS64 service from before the switch to unbound so + # it does not hold the DNS64 port. + _nat64_remove_legacy_coredns + + # Determine the upstream resolvers DNS64 forwards to. An explicit + # NAT64_DNS64_UPSTREAM (space-separated) always wins. Otherwise discover the + # host's real resolvers: NAT64 hosts frequently cannot reach public resolvers + # (e.g. 8.8.8.8) through their firewall, so we forward to whatever the host + # itself uses. When NetworkManager runs in dnsmasq mode /etc/resolv.conf points + # at a loopback stub and the real upstream servers live in no-stub-resolv.conf. + # The `|| true` matters under set -e: awk exits non-zero when the file is + # absent (e.g. no-stub-resolv.conf only exists when NM is in dnsmasq mode), + # and 2>/dev/null hides the message but not the status, so a bare assignment + # would abort the whole host-configure run. Fall through to the next source. + local upstreams="${NAT64_DNS64_UPSTREAM:-}" + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /run/NetworkManager/no-stub-resolv.conf 2>/dev/null || true) + fi + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + upstreams=$(awk '/^nameserver/ && $2 != "127.0.0.1" && $2 != "::1" {print $2}' /etc/resolv.conf 2>/dev/null || true) + fi + if [[ -z "${upstreams//[[:space:]]/}" ]]; then + # Fall back to a public resolver so DNS64 still comes up, but warn loudly: + # on a firewalled NAT64 host 8.8.8.8 is usually unreachable and every DNS64 + # lookup will time out. Set NAT64_DNS64_UPSTREAM to fix this deliberately. + echo "WARNING: could not discover any host upstream resolver for DNS64;" >&2 + echo "WARNING: falling back to 8.8.8.8, which is often unreachable on a" >&2 + echo "WARNING: firewalled NAT64 host. Set NAT64_DNS64_UPSTREAM to override." >&2 + upstreams=8.8.8.8 + fi + echo "DNS64 upstream resolvers: ${upstreams}" + + # NOTE: this unbound instance is a non-validating DNS64 forwarder. dns64-synthall + # rewrites AAAA answers into ${NAT64_PREFIX}, which is fundamentally incompatible + # with DNSSEC AAAA validation, and it implicitly trusts the host's upstream + # resolvers (discovered above). That trust model matches the rest of dev-scripts + # (a lab/CI tool on a controlled network); do not treat this resolver as a + # security boundary. + + # Write the unbound DNS64 config. dns64-synthall makes unbound synthesize an + # AAAA in ${NAT64_PREFIX} for EVERY name, even ones that already have a native + # AAAA: the host has no native IPv6 egress, so all IPv6 must be routed through + # NAT64/TAYGA. (The CoreDNS dns64 plugin was used previously but its build did + # not synthesize at all, so IPv6-only nodes received unreachable native AAAA.) + sudo mkdir -p "$(dirname "${NAT64_UNBOUND_CONF}")" + sudo tee "${NAT64_UNBOUND_CONF}" > /dev/null < /dev/null < /dev/null </dev/null 2>&1 || return 0 + + echo "Pointing libvirt network ${net} dnsmasq at DNS64 resolver..." + sudo virsh net-dumpxml "${net}" > "${xml}" + + # Idempotent: only inject the forwarding options if not already present. + if ! grep -q "server=127.0.0.1#${NAT64_DNS64_PORT}" "${xml}"; then + sudo sed -i "/<\/dnsmasq:options>/i\\ + \\ + " "${xml}" + fi + + # net-info (above) also succeeds for a defined-but-inactive network, for which + # net-destroy exits non-zero ("network is not active"); tolerate that under + # set -e so an inactive extra network does not abort the run. + sudo virsh net-destroy "${net}" 2>/dev/null || true + sudo virsh net-undefine "${net}" + sudo virsh net-define "${xml}" + # net-undefine clears the autostart flag metal3-dev-env set; restore it so the + # network still comes up after a host reboot. + sudo virsh net-autostart "${net}" + sudo virsh net-start "${net}" + + # net-destroy drops the bridge; restore a dummy for carrier and addr_gen_mode=0 + # so the network's IPv6 address comes up before the VMs provide carrier (needed + # for IPv6 on EL9). + local dmy + dmy=$(_nat64_dummy_ifname "${net}") + sudo ip link add name "${dmy}" up master "${net}" type dummy 2>/dev/null || true + echo 0 | sudo dd of="/proc/sys/net/ipv6/conf/${net}/addr_gen_mode" 2>/dev/null || true +} + +# Embed an IPv4 address inside a NAT64 /96 prefix (RFC 6052), e.g. +# nat64_embed_v4 "fd00:64::/96" 192.168.111.1 -> fd00:64::c0a8:6f01. Only the /96 +# form is supported (the documented default); with /96 the IPv4 simply occupies the +# last 32 bits, i.e. the two hex groups appended after the prefix's trailing "::". +# Other prefix lengths interleave the v4 octets differently and are not used by +# dev-scripts. +function nat64_embed_v4() { + local prefix="${1%/*}" # strip the /96 mask -> e.g. fd00:64:: + local o1 o2 o3 o4 + IFS='.' read -r o1 o2 o3 o4 <<<"$2" + # Pack the four octets into two 16-bit groups and print them without leading + # zeros, matching IPv6's canonical hextet form (e.g. 10.0.0.255 -> a00:ff). + # 10# forces base-10 so an octet written with a leading zero (e.g. 011) is not + # mis-parsed as octal (and 08/09 don't abort as "invalid octal"). + printf '%s%x:%x\n' "${prefix}" "$(( (10#$o1 << 8) | 10#$o2 ))" "$(( (10#$o3 << 8) | 10#$o4 ))" +} + +# When running an IPv6-only cluster via NAT64, the in-cluster Ironic runs as +# IPv6-only pods (hostNetwork on v6-only nodes) and cannot reach the BMC emulators +# at their IPv4 baremetal address. Rewrite the generated node BMC addresses so both +# the bootstrap Ironic and the pivoted in-cluster Ironic can control the nodes: +# - redfish (sushy) also listens on IPv6, so point it at the host's native IPv6 +# baremetal address (paired with nat64_fixup_sushy_cert for its TLS SAN). +# - ipmi (vbmc) is IPv4-only, so point it at the NAT64-synthesized form of its +# IPv4 address; TAYGA translates that back to IPv4 (needs a ULA NAT64_PREFIX, +# enforced in common.sh, since the well-known prefix cannot carry RFC1918). +# This makes BMC_DRIVER=mixed (and ipmi) work, not just the redfish drivers. +function nat64_fixup_bmc_addresses() { + local v4host v6host embed + v4host=$(nth_ip "${EXTERNAL_SUBNET_V4}" 1) + v6host=$(nth_ip "${EXTERNAL_SUBNET_V6}" 1) + if [[ -z "${v4host}" || -z "${v6host}" ]]; then + echo "nat64_fixup_bmc_addresses: missing v4/v6 baremetal host address, skipping" + return 0 + fi + # NAT64-embedded form of the IPv4 baremetal address, used for ipmi BMCs. Unused + # (and harmless) when every node is redfish, so it is always computed. + embed=$(nat64_embed_v4 "${NAT64_PREFIX}" "${v4host}") + echo "Rewriting node BMC addresses for NAT64: redfish -> [${v6host}], ipmi -> [${embed}]..." + local f tmp + for f in "${NODES_FILE}" "${NODES_FILE}.orig" "${EXTRA_NODES_FILE:-}" "${ARM_NODES_FILE:-}"; do + [[ -n "${f}" && -f "${f}" ]] || continue + tmp="${f}.nat64" + # Per-node, driver-aware literal (non-regex) host replacement on the "//host:" + # token. The address? == null guard skips nodes without a BMC address (e.g. a + # manually-edited inventory) instead of aborting jq on a null/string divide. + # ipmi nodes get the NAT64-embedded address; everything else (redfish, + # redfish-virtualmedia) gets the native IPv6 host. On success copy back through + # the original file (preserving its mode/owner rather than replacing it with a + # fresh-umask temp via mv); always remove the temp so a jq failure does not + # strand a partial ".nat64" file. + if jq --arg old "//${v4host}:" --arg v6 "//[${v6host}]:" --arg embed "//[${embed}]:" \ + '.nodes[]? |= ( + if (.driver_info.address? == null) then . + else .driver_info.address = + (if .driver == "ipmi" + then (.driver_info.address / $old | join($embed)) + else (.driver_info.address / $old | join($v6)) end) + end)' \ + "${f}" > "${tmp}"; then + # Overwrite in place with sudo: ${f} may be root-owned (written by an + # earlier privileged step) while its directory is user-writable, so a + # plain ">" redirect fails with EACCES. cp onto the existing file also + # keeps its original mode/owner rather than replacing it with a + # fresh-umask temp (which a plain "mv" would have done). + sudo cp "${tmp}" "${f}" + fi + rm -f "${tmp}" + done +} + +# Regenerate the sushy-tools BMC emulator TLS certificate so it is valid for the +# host's IPv6 baremetal address in addition to its IPv4 one. metal3-dev-env only +# puts the IPv4 baremetal address in the cert's SAN, but for NAT64 the in-cluster +# Ironic pods are IPv6-only and must reach the BMC over IPv6; without the IPv6 SAN +# they reject the redfish connection with an "IP address mismatch" TLS error. +# (On OCP >= 4.22 dev-scripts no longer emits disableCertificateVerification, so +# certificate verification is always on and the cert MUST carry the IPv6 SAN.) +# +# This must run before 05_create_install_config.sh, which embeds this cert into the +# install-config trust bundle (see ocp_install_env.sh). The existing key is reused +# and the sushy-tools container is restarted so it serves the new cert. Idempotent +# and a no-op when sushy or the IPv6 address is not present. +function nat64_fixup_sushy_cert() { + local sushy_dir="${WORKING_DIR}/virtualbmc/sushy-tools" + local cert="${sushy_dir}/cert.pem" + local key="${sushy_dir}/key.pem" + local v4host v6host + v4host=$(nth_ip "${EXTERNAL_SUBNET_V4}" 1) + v6host=$(nth_ip "${EXTERNAL_SUBNET_V6}" 1) + # Probe for the cert/key under sudo: ${WORKING_DIR}/virtualbmc is root-owned and + # root-only (drwxr-x---), so the invoking user cannot traverse it and a plain + # "[[ -f ]]" would report the files missing even when they exist, silently + # skipping the fixup (leaving sushy serving an IPv4-only cert that the IPv6-only + # in-cluster Ironic rejects). The rest of this function already uses sudo. + if ! sudo test -f "${cert}" || ! sudo test -f "${key}" || [[ -z "${v6host}" ]]; then + echo "nat64_fixup_sushy_cert: sushy cert/key or IPv6 host address missing, skipping" + return 0 + fi + + # Already valid for the IPv6 address? Nothing to do. Use openssl's own SAN + # matching (-checkip) rather than grepping the text dump: openssl renders IPv6 + # SANs in uncompressed form, so a substring match for the compressed address + # never matched and the cert was needlessly regenerated on every run. + if sudo openssl x509 -in "${cert}" -noout -checkip "${v6host}" >/dev/null 2>&1; then + echo "sushy BMC cert already valid for ${v6host}" + return 0 + fi + + echo "Regenerating sushy BMC cert with SANs IP:${v4host}, IP:${v6host} for NAT64..." + local tmp + tmp=$(mktemp -d) + cat > "${tmp}/san.cnf" </dev/null || true + + echo "sushy BMC cert regenerated for ${v4host} and ${v6host}" +} + +# Remove the legacy CoreDNS DNS64 service (superseded by unbound). No-op if absent. +function _nat64_remove_legacy_coredns() { + if sudo systemctl is-active --quiet coredns-nat64.service 2>/dev/null; then + sudo systemctl stop coredns-nat64.service + fi + sudo systemctl disable coredns-nat64.service 2>/dev/null || true + sudo rm -f /etc/systemd/system/coredns-nat64.service + # NOTE: deliberately do NOT "rm -rf /etc/coredns" here. This helper runs on + # every configure_dns64 call, and /etc/coredns may hold an unrelated host-owned + # CoreDNS configuration. Stopping/removing our own service is enough to free the + # DNS64 port. + sudo systemctl daemon-reload +} + +function cleanup_nat64() { + echo "Cleaning up NAT64/DNS64 configuration..." + + # Remove any legacy CoreDNS DNS64 service from before the switch to unbound + _nat64_remove_legacy_coredns + + # Stop and disable the DNS64 unbound instance + if sudo systemctl is-active --quiet "${NAT64_UNBOUND_SERVICE}.service" 2>/dev/null; then + sudo systemctl stop "${NAT64_UNBOUND_SERVICE}.service" + fi + sudo systemctl disable "${NAT64_UNBOUND_SERVICE}.service" 2>/dev/null || true + sudo rm -f "/etc/systemd/system/${NAT64_UNBOUND_SERVICE}.service" + sudo systemctl daemon-reload + + # Remove unbound DNS64 configuration + sudo rm -f "${NAT64_UNBOUND_CONF}" + + # Remove the per-network DNS64 dummy carrier interfaces + local net + for net in "${BAREMETAL_NETWORK_NAME}" ${EXTRA_NETWORK_NAMES:-}; do + sudo ip link del "$(_nat64_dummy_ifname "${net}")" 2>/dev/null || true + done + # Remove the legacy baremetal dummy name used by earlier revisions + sudo ip link del bm-ipv6-dummy 2>/dev/null || true + + # Remove dnsmasq DNS64 forwarding config + sudo rm -f "${NAT64_DNSMASQ_CONF}" + if systemctl is-active --quiet NetworkManager; then + sudo systemctl reload NetworkManager + fi + + # Stop and disable the TAYGA systemd service + if sudo systemctl is-active --quiet "${NAT64_TAYGA_SERVICE}.service" 2>/dev/null; then + sudo systemctl stop "${NAT64_TAYGA_SERVICE}.service" + fi + sudo systemctl disable "${NAT64_TAYGA_SERVICE}.service" 2>/dev/null || true + sudo rm -f "/etc/systemd/system/${NAT64_TAYGA_SERVICE}.service" + sudo systemctl daemon-reload + # Kill any stray bare tayga daemon FIRST, then tear down the TUN device: tayga + # holds the TUN open, so --rmtun before the process is gone leaves it orphaned. + # Match by exact process name (the cmdline is "/usr/sbin/tayga ...", so an + # "^tayga" -f pattern never matched). Delete the device explicitly as a backstop + # in case --rmtun fails (e.g. the tayga binary is already gone). + sudo pkill -x tayga 2>/dev/null || true + sudo tayga --rmtun 2>/dev/null || true + sudo ip link del "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + + # Drop the permanent firewalld trusted-zone binding for the TUN (added in + # configure_tayga so translated ipmi BMC traffic is not dropped). The runtime + # binding disappears with the interface; remove the persisted one too. + if command -v firewall-cmd >/dev/null 2>&1 && sudo firewall-cmd --state >/dev/null 2>&1; then + sudo firewall-cmd --permanent --zone=trusted --remove-interface="${NAT64_TUN_INTERFACE}" 2>/dev/null || true + sudo firewall-cmd --reload 2>/dev/null || true + fi + + # Remove NAT64 routes + sudo ip route del "${NAT64_V4_POOL}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + sudo ip -6 route del "${NAT64_PREFIX}" dev "${NAT64_TUN_INTERFACE}" 2>/dev/null || true + + # Remove iptables masquerade rule + sudo iptables -t nat -D POSTROUTING -s "${NAT64_V4_POOL}" -j MASQUERADE 2>/dev/null || true + + # Remove the IPv6 FORWARD rules added by 02_configure_host.sh for NAT64 + sudo ip6tables -D FORWARD --in-interface "${BAREMETAL_NETWORK_NAME}" -j ACCEPT 2>/dev/null || true + sudo ip6tables -D FORWARD --out-interface "${BAREMETAL_NETWORK_NAME}" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true + + # Clean up TAYGA data and config + sudo rm -rf "${NAT64_TAYGA_DATA_DIR}" + sudo rm -f "${NAT64_TAYGA_CONF}" + + echo "NAT64/DNS64 cleanup complete" +} diff --git a/network.sh b/network.sh index 97204e389..7ee7afc2c 100755 --- a/network.sh +++ b/network.sh @@ -32,6 +32,12 @@ export PATH_CONF_DNSMASQ="/etc/NetworkManager/dnsmasq.d/openshift-${CLUSTER_NAME export IP_STACK=${IP_STACK:-"v6"} export HOST_IP_STACK=${HOST_IP_STACK:-${IP_STACK}} +# NAT64/DNS64 configuration for IPv6-only clusters on IPv4 hosts +export ENABLE_NAT64=${ENABLE_NAT64:-false} +export NAT64_PREFIX=${NAT64_PREFIX:-"64:ff9b::/96"} +export NAT64_V4_POOL=${NAT64_V4_POOL:-"192.168.255.0/24"} +export NAT64_V4_ADDR=${NAT64_V4_ADDR:-"192.168.255.1"} + # Record the pre-defaulting NETWORK_TYPE export ORIG_NETWORK_TYPE=${NETWORK_TYPE:-""} @@ -96,7 +102,12 @@ if [[ "$HOST_IP_STACK" = "v4" ]] then export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-"172.22.0.0/24"} export EXTERNAL_SUBNET_V4=${EXTERNAL_SUBNET_V4:-"192.168.111.0/24"} - export EXTERNAL_SUBNET_V6="" + if [[ "${ENABLE_NAT64}" == "true" ]]; then + # NAT64: bridge needs IPv6 for cluster VMs even though host is IPv4-only + export EXTERNAL_SUBNET_V6=${EXTERNAL_SUBNET_V6:-"fd2e:6f44:5dd8:c956::/120"} + else + export EXTERNAL_SUBNET_V6="" + fi elif [[ "$HOST_IP_STACK" = "v6" ]]; then export PROVISIONING_NETWORK=${PROVISIONING_NETWORK:-"fd00:1101::0/64"} export EXTERNAL_SUBNET_V4="" @@ -114,6 +125,18 @@ else exit 1 fi +if [[ "${ENABLE_NAT64}" == "true" ]]; then + export NAT64_V6_ADDR=${NAT64_V6_ADDR:-$(nth_ip "$EXTERNAL_SUBNET_V6" 3)} + # Under NAT64 the host is IPv4-only externally (PROVISIONING_HOST_EXTERNAL_IP is + # the IPv4 baremetal address), but it also carries an IPv6 baremetal address on + # the bridge (see nat64.sh). Export it so host services that IPv6-only cluster + # nodes must reach (e.g. the image-registry NFS export) can be published over it. + # Not overridable: it must stay in lockstep with the same address nat64.sh puts + # on the bridge and into the sushy cert SAN (also nth_ip "$EXTERNAL_SUBNET_V6" 1); + # an independent override here would publish an AAAA the bridge never answers. + export PROVISIONING_HOST_EXTERNAL_IP_V6=$(nth_ip "$EXTERNAL_SUBNET_V6" 1) +fi + function openshift_sdn_deprecated() { # OpenShiftSDN is deprecated in 4.15 and later printf '4.15\n%s\n' "$(openshift_version)" | sort -V -C @@ -140,7 +163,8 @@ elif [[ "$IP_STACK" = "v6" ]]; then export SERVICE_SUBNET_V4="" export SERVICE_SUBNET_V6=${SERVICE_SUBNET_V6:-"fd02::/112"} export NETWORK_TYPE=${NETWORK_TYPE:-"OVNKubernetes"} - if [[ ${AGENT_E2E_TEST_BOOT_MODE} != "ISO_NO_REGISTRY" ]]; then + if [[ "${ENABLE_NAT64}" != "true" ]] && [[ ${AGENT_E2E_TEST_BOOT_MODE} != "ISO_NO_REGISTRY" ]]; then + # NAT64 provides external registry access, so mirroring is not required export MIRROR_IMAGES=${MIRROR_IMAGES:-true} fi elif [[ "$IP_STACK" = "v4v6" || "$IP_STACK" = "v6v4" ]]; then @@ -256,7 +280,7 @@ function get_vips() { # Returns: # None # - if [[ -n "${EXTERNAL_SUBNET_V4}" ]]; then + if [[ -n "${EXTERNAL_SUBNET_V4}" ]] && [[ "${IP_STACK}" != "v6" ]]; then API_VIPS_V4=$(dig +noall +answer "api.${CLUSTER_DOMAIN}" @"$(network_ip "${BAREMETAL_NETWORK_NAME}" v4)" | awk '{print $NF}') if [ -z "$EXTERNAL_LOADBALANCER" ]; then INGRESS_VIPS_V4=$(nth_ip "$EXTERNAL_SUBNET_V4" 4) @@ -265,7 +289,7 @@ function get_vips() { fi fi - if [[ -n "${EXTERNAL_SUBNET_V6}" ]]; then + if [[ -n "${EXTERNAL_SUBNET_V6}" ]] && [[ "${IP_STACK}" != "v4" ]]; then API_VIPS_V6=$(dig -t AAAA +noall +answer "api.${CLUSTER_DOMAIN}" @"$(network_ip "${BAREMETAL_NETWORK_NAME}" v6)" | awk '{print $NF}') if [ -z "$EXTERNAL_LOADBALANCER" ]; then INGRESS_VIPS_V6=$(nth_ip "$EXTERNAL_SUBNET_V6" 4)