From b798c718a260398e8c21fbef080568a6ca345718 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Fri, 4 Sep 2026 14:57:37 +0000 Subject: [PATCH] Boilerplate: Update to a8a3172411f3f2b8848f64333843e028ef4b3ed1 Conventions: - openshift/golang-osd-e2e: Update --- https://github.com/openshift/boilerplate/compare/a0e42e58ed1d65bb75a848c595b34ae5553296eb...a8a3172411f3f2b8848f64333843e028ef4b3ed1 commit: 39b903e8c5db27cee7bb5b2dcce9ff4b5a07da5e author: red-hat-konflux[bot] chore(deps): update konflux references Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: c678988174f35841a156cc2bba88d043b84cea9f author: red-hat-konflux[bot] chore(deps): update registry.access.redhat.com/ubi8/ubi-minimal:latest docker digest to d9beb74 Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> commit: 1542cb9a6f953091ea36868f18ef42542103dba7 author: Bo Meng Add shared CodeRabbit configuration commit: 76056e0c32e5423cbc9e163c2053cd36b7435961 author: Dustin Row Update roxctl-scan task bundle to fix null jq error Updates the roxctl-scan task bundle SHA to include the fix for KONFLUX-15651, where the proccess-output step fails with "Cannot iterate over null" on scratch-based images. Fix: konflux-ci/konflux-test#906 commit: e857a1ac44cb260f0df165bc3cdc77524fcdca48 author: Dustin Row gangway-bridge: tighten POLL_OVERSHOOT and remove redundant 429 sleep POLL_OVERSHOOT now uses max(POLL_INTERVAL, 300) + 30 instead of the hardcoded 300+30+300, so the budget reflects whichever delay is larger. The extra sleep in the 429 branch is removed; the loop leading sleep already provides the backoff on the next iteration. commit: bcf83f16fe1a00029f45af9c3ac0584203090991 author: Dustin Row gangway-bridge: fix REQUIRED_DEADLINE to include INITIAL_DELAY and larger POLL_OVERSHOOT commit: 7f5eecc7db7d567023b2015c71f2786b91d77c19 author: Dustin Row gangway-bridge: longer poll interval and retry backoff cap Double default POLL_INTERVAL from 60s to 120s to reduce the baseline polling rate. With multiple concurrent jobs the polling alone can consume the 9 req/min Gangway rate limit budget. Raise the inter-retry backoff cap from 480s to 900s so later retry attempts back off more aggressively when contention is high. commit: d2a4c7a5454223b2169392c645f108e364bb57be author: Dustin Row gangway-bridge: add INITIAL_DELAY and poll 429 backoff Gangway rate-limits at 9 req/min per source IP with nodelay burst of 5. When multiple operators deploy in the same SAPM pipeline run their gangway-bridge jobs all start simultaneously and saturate the shared quota, causing trigger attempts to exhaust all retries and fail. Add INITIAL_DELAY parameter (default 0s) so callers can stagger concurrent jobs by setting different delays per target in the saas file. Also fix the status-poll loop to back off exponentially (doubling up to 300s) on 429 responses instead of silently retrying at the normal POLL_INTERVAL, which was burning rate limit budget during polling and competing with trigger retries from other concurrent jobs. commit: 03d7a11d4b501360c57e0d4aec00f0f19d556044 author: Chai Bot gangway-bridge: back off on 429 during status polling --- boilerplate/_data/last-boilerplate-commit | 2 +- .../gangway-bridge-template.yml | 48 +++++++++++++++---- test/e2e/gangway-bridge-template.yml | 48 +++++++++++++++---- 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/boilerplate/_data/last-boilerplate-commit b/boilerplate/_data/last-boilerplate-commit index af0bb090..d198b673 100644 --- a/boilerplate/_data/last-boilerplate-commit +++ b/boilerplate/_data/last-boilerplate-commit @@ -1 +1 @@ -a0e42e58ed1d65bb75a848c595b34ae5553296eb +a8a3172411f3f2b8848f64333843e028ef4b3ed1 diff --git a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml index eba8951a..d1a925a4 100644 --- a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml +++ b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml @@ -8,7 +8,7 @@ parameters: required: true description: Prow periodic job name to trigger via Gangway - name: POLL_INTERVAL - value: "60" + value: "120" description: Seconds between status polls - name: TIMEOUT value: "7200" @@ -17,8 +17,11 @@ parameters: value: "5" description: Number of times to retry the Prow job on failure before reporting failure - name: ACTIVE_DEADLINE - value: "50400" + value: "54000" description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: INITIAL_DELAY + value: "0" + description: Seconds to sleep before the first Gangway call; stagger concurrent jobs to avoid shared rate limit saturation - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -53,15 +56,22 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } - # Backoff sum: base 30s doubling each retry = 30*(2^N-1), plus 15s max jitter + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + + # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) - # Each attempt may overshoot TIMEOUT by up to POLL_INTERVAL + status-request - # max-time (30s) on the last poll cycle - POLL_OVERSHOOT=$(( POLL_INTERVAL + 30 )) + # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + + # status-request max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt TRIGGER_OVERHEAD=$(( 60 + 600 )) - REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM )) + # INITIAL_DELAY is a one-time cost at job startup, not per attempt + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM + INITIAL_DELAY )) if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" exit 1 @@ -114,9 +124,25 @@ objects: log "Prow logs: ${PROW_URL}" END=$((SECONDS + ${TIMEOUT})) + local poll_backoff="${POLL_INTERVAL}" while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + sleep "$poll_backoff" + local poll_file="/dev/shm/gw_poll.$$" + local poll_code + poll_code=$(curl -sSL --max-time 30 \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -o "$poll_file" -w '%{http_code}' \ + "${GW}/${ID}" 2>/dev/null) || poll_code=000 + if [[ "$poll_code" == "429" ]]; then + rm -f "$poll_file" + poll_backoff=$(( poll_backoff * 2 )) + [[ $poll_backoff -gt 300 ]] && poll_backoff=300 + log "Rate limited polling status (429) — backing off ${poll_backoff}s" + continue + fi + poll_backoff="${POLL_INTERVAL}" + S=$(jq -r .job_status "$poll_file" 2>/dev/null) || S=UNKNOWN + rm -f "$poll_file" log "${S} ($((SECONDS))s)" case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; @@ -143,7 +169,7 @@ objects: RATE_LIMITED_WAITED=0 else BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) - [[ $BACKOFF -gt 480 ]] && BACKOFF=480 + [[ $BACKOFF -gt 900 ]] && BACKOFF=900 JITTER=$(( RANDOM % 16 )) DELAY=$(( BACKOFF + JITTER )) log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." @@ -166,6 +192,8 @@ objects: value: ${JOB_ENVS} - name: MAX_RETRIES value: ${MAX_RETRIES} + - name: INITIAL_DELAY + value: ${INITIAL_DELAY} - name: ACTIVE_DEADLINE value: ${ACTIVE_DEADLINE} resources: diff --git a/test/e2e/gangway-bridge-template.yml b/test/e2e/gangway-bridge-template.yml index eba8951a..d1a925a4 100644 --- a/test/e2e/gangway-bridge-template.yml +++ b/test/e2e/gangway-bridge-template.yml @@ -8,7 +8,7 @@ parameters: required: true description: Prow periodic job name to trigger via Gangway - name: POLL_INTERVAL - value: "60" + value: "120" description: Seconds between status polls - name: TIMEOUT value: "7200" @@ -17,8 +17,11 @@ parameters: value: "5" description: Number of times to retry the Prow job on failure before reporting failure - name: ACTIVE_DEADLINE - value: "50400" + value: "54000" description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: INITIAL_DELAY + value: "0" + description: Seconds to sleep before the first Gangway call; stagger concurrent jobs to avoid shared rate limit saturation - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -53,15 +56,22 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + [[ "${INITIAL_DELAY}" =~ ^[0-9]+$ ]] || { log "ERROR: INITIAL_DELAY must be a non-negative integer"; exit 1; } - # Backoff sum: base 30s doubling each retry = 30*(2^N-1), plus 15s max jitter + if [[ "${INITIAL_DELAY}" -gt 0 ]]; then + log "Waiting ${INITIAL_DELAY}s before first Gangway call (INITIAL_DELAY)..." + sleep "${INITIAL_DELAY}" + fi + + # Backoff sum: base 30s doubling each retry, capped at 900s, plus 15s max jitter MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) - # Each attempt may overshoot TIMEOUT by up to POLL_INTERVAL + status-request - # max-time (30s) on the last poll cycle - POLL_OVERSHOOT=$(( POLL_INTERVAL + 30 )) + # Each attempt may overshoot TIMEOUT by up to max(POLL_INTERVAL, 300s max backoff) + + # status-request max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( (POLL_INTERVAL > 300 ? POLL_INTERVAL : 300) + 30 )) # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt TRIGGER_OVERHEAD=$(( 60 + 600 )) - REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM )) + # INITIAL_DELAY is a one-time cost at job startup, not per attempt + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM + INITIAL_DELAY )) if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" exit 1 @@ -114,9 +124,25 @@ objects: log "Prow logs: ${PROW_URL}" END=$((SECONDS + ${TIMEOUT})) + local poll_backoff="${POLL_INTERVAL}" while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + sleep "$poll_backoff" + local poll_file="/dev/shm/gw_poll.$$" + local poll_code + poll_code=$(curl -sSL --max-time 30 \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -o "$poll_file" -w '%{http_code}' \ + "${GW}/${ID}" 2>/dev/null) || poll_code=000 + if [[ "$poll_code" == "429" ]]; then + rm -f "$poll_file" + poll_backoff=$(( poll_backoff * 2 )) + [[ $poll_backoff -gt 300 ]] && poll_backoff=300 + log "Rate limited polling status (429) — backing off ${poll_backoff}s" + continue + fi + poll_backoff="${POLL_INTERVAL}" + S=$(jq -r .job_status "$poll_file" 2>/dev/null) || S=UNKNOWN + rm -f "$poll_file" log "${S} ($((SECONDS))s)" case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; @@ -143,7 +169,7 @@ objects: RATE_LIMITED_WAITED=0 else BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) - [[ $BACKOFF -gt 480 ]] && BACKOFF=480 + [[ $BACKOFF -gt 900 ]] && BACKOFF=900 JITTER=$(( RANDOM % 16 )) DELAY=$(( BACKOFF + JITTER )) log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." @@ -166,6 +192,8 @@ objects: value: ${JOB_ENVS} - name: MAX_RETRIES value: ${MAX_RETRIES} + - name: INITIAL_DELAY + value: ${INITIAL_DELAY} - name: ACTIVE_DEADLINE value: ${ACTIVE_DEADLINE} resources: