diff --git a/boilerplate/_data/backing-image-tag b/boilerplate/_data/backing-image-tag index ca21d244..284dd19e 100644 --- a/boilerplate/_data/backing-image-tag +++ b/boilerplate/_data/backing-image-tag @@ -1 +1 @@ -image-v8.3.6 +image-v8.4.3 diff --git a/boilerplate/_data/last-boilerplate-commit b/boilerplate/_data/last-boilerplate-commit index 3e14fd49..af0bb090 100644 --- a/boilerplate/_data/last-boilerplate-commit +++ b/boilerplate/_data/last-boilerplate-commit @@ -1 +1 @@ -8fb7c801f68dc7e06e8d2ae138c2a98f0b234b56 +a0e42e58ed1d65bb75a848c595b34ae5553296eb diff --git a/boilerplate/_lib/subscriber-propose-update b/boilerplate/_lib/subscriber-propose-update index f3b06ef2..4ac51256 100755 --- a/boilerplate/_lib/subscriber-propose-update +++ b/boilerplate/_lib/subscriber-propose-update @@ -25,7 +25,7 @@ Quirks and Limitations: - Is still slightly interactive, because 'gh pr create' likes to ask questions about your origin and upstream. EOF - exit -1 + exit 1 } source $REPO_ROOT/boilerplate/_lib/subscriber.sh @@ -34,47 +34,101 @@ source $REPO_ROOT/boilerplate/_lib/subscriber.sh [[ $# -eq 0 ]] && usage TMPD=$(mktemp -d) +echo $TMPD; trap "rm -fr $TMPD" EXIT +run_step() { + local title=$1 + local log_file="$TMPD/$title.log" + log_file=$(tr '[:upper:]' '[:lower:]' <<< "$log_file") + log_file=$(tr ' ' '-' <<< "$log_file") + shift + + if [[ $1 != "--" ]]; then + echo "ERR: expected '--' but got '$1'" + exit 1 + fi + shift + echo -n "$title... " + + if ! "$@" > "$log_file" 2>&1; then + echo " FAILED" + echo "!!!" + echo "!!! Boilerplate update failed for $subscriber" + echo "!!!" + echo "" + cat "$log_file" + exit 1 + fi + echo " DONE" +} + +sync_main() { + local main_branch=$1 + shift + + git pull upstream $main_branch + git push origin $main_branch +} + +git_clean_and_push() { + local branch=$1 + shift + + git push --delete origin $branch || true + git push -u origin $branch +} + propose_update() { local subscriber=$1 local proj=${subscriber#*/} - if [[ -z "$DRY_RUN" ]]; then - echo "DRY RUN: Would propose update for $subscriber" - return 0 - fi - ( # Clone my fork of the subscriber repo cd $TMPD # This # - uses the existing fork if one exists # - sets 'origin' and 'upstream' remotes - gh repo fork $subscriber --clone=true --remote=true + # only clones the default branch to save disk space and time + + run_step "Creating fork" -- gh repo fork $subscriber --clone=true --default-branch-only cd $proj - # Current branch is 'master' or 'main' - cur_branch=$(current_branch .) - # Make sure our origin is synced with upstream, so our update - # commit is based off of the latest code. - # WARNING: This changes your fork! - git pull upstream $cur_branch - git push origin $cur_branch - - # Create the update commit - make boilerplate-update - make boilerplate-commit - - # And create the PR - # TODO: This is interactive. How do we tell gh "Yes, please use - # upstream as upstream and origin as origin?" - gh pr create -f + # Current branch is 'master' or 'main' or 'trunk' + main_branch=$(current_branch .) + run_step "Syncing Fork" -- sync_main $main_branch + # run_step "Pushing fork" -- git push origin $main_branch + + # Create the update commit - only cat logs if something goes wrong. + run_step "Updating boilerplate" -- make boilerplate-update + run_step "Committing boilerplate update" -- make boilerplate-commit + + boilerplate_branch=$(git rev-parse --abbrev-ref HEAD) + # By pushing to the origin boilerplate branch explicitly before opening a PR, + # we make don't get prompted for the branch to push to. + # If we still find that it's giving us an interactive prompt, we can otherwise + # use `gh api` to create the PR programmatically. + if [[ "$boilerplate_branch" == "$main_branch" ]]; then + echo "CRITICAL ERROR: boilerplate branch '$boilerplate_branch' is the same as main branch '$main_branch'" + echo "If you see this, something has gone terribly wrong" + echo "Skipping" + exit 20 + fi + run_step "pushing update" -- git_clean_and_push $boilerplate_branch + + gh pr create --repo $subscriber -f $DRY_RUN_FLAG ) } bp_master=$(git rev-parse master) +DRY_RUN_FLAG="" +if [[ -z "$DRY_RUN" ]]; then + echo "DRY RUN: ENABLED" + DRY_RUN_FLAG="--dry-run" +fi + + for subscriber in $(subscriber_args "$@"); do # Does this one need an update? @@ -89,14 +143,45 @@ for subscriber in $(subscriber_args "$@"); do continue fi - # Is there already a PR proposed for this level? - existing_pr=$(gh pr list --repo $subscriber | grep -P ":boilerplate-\S+-$bp_master\s") + # Is there already a PR proposed for this commit? + pr_list=$(gh pr list --repo $subscriber --json headRefName,url,number | jq -r '. | map(select(.headRefName | startswith("boilerplate-update--")))') + existing_pr=$(jq -r ".[] | select(.headRefName == \"boilerplate-update--$bp_master\")" <<< "$pr_list") if [[ -n "$existing_pr" ]]; then - echo "Subscriber '$subscriber' already has an open PR:" - echo "https://github.com/$subscriber/pull/$existing_pr" + echo "Subscriber '$subscriber' already has an open PR for this boilerplate commit:" + jq -r .url <<< "$existing_pr" continue fi # Pull the trigger - propose_update "$subscriber" + if ! propose_update "$subscriber"; then + echo "Error: failed to propose update for '$subscriber'" + continue + fi + + new_pr="XXXX" + # Get the new PR URL + # only run if not dry-run - otherwise the new_pr var will be empty + if [[ -n $DRY_RUN ]]; then + new_pr=$(gh pr list --repo $subscriber --json headRefName,number | jq -r ".[] | select(.headRefName == \"boilerplate-update--$bp_master\") | .number") + if [[ -z "$new_pr" ]]; then + echo "error: unable to find new PR for boilerplate update '$bp_master' on subscriber '$subscriber'" + continue + fi + fi + + # Add comments to existing PRs to say they're superseded by this new one + if [[ -n "$pr_list" ]]; then + prs=$(jq -r '. | map(.number) | @tsv' <<< "$pr_list") + echo "Closing old PRs: $prs" + for pr in $prs; do + if [[ -z $DRY_RUN ]]; then + echo "Dry run - would close $pr with comment:" + echo " \"Superseded by #$new_pr.\"" + continue + fi + + gh pr close --repo $subscriber --comment "Superseded by #$new_pr." $pr + done + fi + done diff --git a/boilerplate/openshift/golang-osd-e2e/OWNERS b/boilerplate/openshift/golang-osd-e2e/OWNERS index c0c694ae..0287d7e1 100644 --- a/boilerplate/openshift/golang-osd-e2e/OWNERS +++ b/boilerplate/openshift/golang-osd-e2e/OWNERS @@ -1,4 +1,4 @@ reviewers: -- srep-infra-cicd +- rosa-staff-engineers approvers: -- srep-infra-cicd +- rosa-staff-engineers diff --git a/boilerplate/openshift/golang-osd-e2e/README.md b/boilerplate/openshift/golang-osd-e2e/README.md index cb747a68..cc7ba3d1 100644 --- a/boilerplate/openshift/golang-osd-e2e/README.md +++ b/boilerplate/openshift/golang-osd-e2e/README.md @@ -31,8 +31,48 @@ following: |------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `e2e-binary-build` | Compiles ginkgo tests under test/e2e and creates the ginkgo binary. | | `e2e-image-build-push` | Builds e2e image and pushes to operator's quay repo. Image name is defaulted to -test-harness. Quay repository must be created beforehand. | +| `e2e-local` | Builds the e2e binary and runs it against a cluster via KUBECONFIG or backplane. Supports focused tests via GINKGO_FOCUS. | #### E2E Local Testing -Please follow [this README](https://github.com/openshift/ops-sop/blob/master/v4/howto/osde2e/operator-test-harnesses.md#using-ginkgo) to run your e2e tests locally +Run e2e tests locally against a managed cluster without waiting for the full Prow CI pipeline. +**Prerequisites:** +- `ocm` CLI logged into the appropriate environment (`ocm login --use-auth-code --url staging`) +- Access to a managed cluster (via KUBECONFIG or backplane) +- Go toolchain installed + +**Option 1: Using backplane (recommended)** + +```bash +# Run all tests against a cluster by ID +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq + +# Run focused tests +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq GINKGO_FOCUS="is installed" + +# Run tests with a label filter +make e2e-local CLUSTER_ID=2rmlgv5dbdp2285n85o7h3aaa6pafkpq GINKGO_LABEL_FILTER="!slow" +``` + +**Option 2: Using an existing KUBECONFIG** + +```bash +# Set KUBECONFIG to your cluster's kubeconfig +export KUBECONFIG=/path/to/kubeconfig + +# Run all tests +make e2e-local + +# Run a single test +make e2e-local GINKGO_FOCUS="reconciles required resources" +``` + +**Output:** +- Test results print to stdout with verbose Ginkgo output +- JUnit XML report saved to `e2e-local-junit.xml` + +**Tips:** +- Use lease clusters for testing (check `#rosa-prow-info` for available clusters) +- The operator must be deployed on the target cluster (via PKO or OLM) +- If tests fail with "not found" errors, verify the operator is running: `oc get deployment -n ` diff --git a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml new file mode 100644 index 00000000..eba8951a --- /dev/null +++ b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml @@ -0,0 +1,185 @@ +# THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: gangway-bridge-e2e +parameters: + - name: JOB_NAME + required: true + description: Prow periodic job name to trigger via Gangway + - name: POLL_INTERVAL + value: "60" + description: Seconds between status polls + - name: TIMEOUT + value: "7200" + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "5" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "50400" + description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: JOB_ENVS + value: "" + description: Comma-separated KEY=VALUE pairs passed to the Prow job + - name: JOBID + generate: expression + from: "[0-9a-z]{7}" + - name: IMAGE_TAG + value: '' + required: true +objects: + - apiVersion: batch/v1 + kind: Job + metadata: + name: gangway-bridge-${IMAGE_TAG}-${JOBID} + spec: + backoffLimit: 0 + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} + template: + spec: + automountServiceAccountToken: false + restartPolicy: Never + containers: + - name: gangway-bridge + image: quay.io/openshift/origin-tools:latest + command: + - /bin/bash + - -ceu + - | + GW="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com/v1/executions" + log() { echo "$(date +%H:%M:%S) $*" >&2; } + + [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } + [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + + # Backoff sum: base 30s doubling each retry = 30*(2^N-1), plus 15s max jitter + MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + # Each attempt may overshoot TIMEOUT by up to POLL_INTERVAL + status-request + # max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( POLL_INTERVAL + 30 )) + # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt + TRIGGER_OVERHEAD=$(( 60 + 600 )) + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi + + BODY='{"job_execution_type":"1"}' + if [[ -n "${JOB_ENVS:-}" ]]; then + ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs // input | split(",")[] | split("=") | {(.[0]): .[1:] | join("=")}] | add' <<< "${JOB_ENVS}") + BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') + fi + + RATE_LIMITED_WAITED=0 + trigger_and_poll() { + local resp_file="/dev/shm/gw_resp.$$" header_file="/dev/shm/gw_hdr.$$" + trap 'rm -f "$resp_file" "$header_file"' RETURN + HTTP_CODE=$(curl -sSL --max-time 60 -X POST \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${BODY}" \ + -o "$resp_file" -D "$header_file" \ + -w '%{http_code}' "${GW}/${JOB_NAME}" 2>/dev/null) || HTTP_CODE=000 + + # Handle 429 — parse Retry-After header (capped at 600s) + if [[ "$HTTP_CODE" == "429" ]]; then + local retry_after + retry_after=$(grep -i '^retry-after:' "$header_file" | awk '{print $2}' | tr -d '\r') + if [[ "$retry_after" =~ ^[0-9]+$ ]] && [[ "$retry_after" -gt 0 ]] && [[ "$retry_after" -le 600 ]]; then + log "Rate limited (429) — sleeping ${retry_after}s (Retry-After)" + sleep "$retry_after" + RATE_LIMITED_WAITED=1 + else + log "Rate limited (429) — no valid Retry-After header" + fi + return 1 # falls through to outer retry with backoff + fi + + # Fail on non-2xx + if [[ "$HTTP_CODE" -lt 200 || "$HTTP_CODE" -ge 300 ]]; then + log "Failed to trigger ${JOB_NAME} (HTTP ${HTTP_CODE})" + return 1 + fi + + RESP=$(cat "$resp_file") + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + while [[ $SECONDS -lt $END ]]; do + sleep "${POLL_INTERVAL}" + S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } + + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + if [[ $RATE_LIMITED_WAITED -eq 1 ]]; then + log "Skipping backoff (already waited for Retry-After)" + RATE_LIMITED_WAITED=0 + else + BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + [[ $BACKOFF -gt 480 ]] && BACKOFF=480 + JITTER=$(( RANDOM % 16 )) + DELAY=$(( BACKOFF + JITTER )) + log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." + sleep "$DELAY" + fi + done + env: + - name: JOB_NAME + value: ${JOB_NAME} + - name: GANGWAY_TOKEN + valueFrom: + secretKeyRef: + name: gangway-api-token + key: token + - name: POLL_INTERVAL + value: ${POLL_INTERVAL} + - name: TIMEOUT + value: ${TIMEOUT} + - name: JOB_ENVS + value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "100m" + memory: "128Mi" + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault diff --git a/boilerplate/openshift/golang-osd-e2e/standard.mk b/boilerplate/openshift/golang-osd-e2e/standard.mk index 7d051307..22f94aae 100644 --- a/boilerplate/openshift/golang-osd-e2e/standard.mk +++ b/boilerplate/openshift/golang-osd-e2e/standard.mk @@ -64,6 +64,26 @@ e2e-binary-build: go mod tidy go test ./test/e2e -v -c --tags=osde2e -o e2e.test +# Run e2e tests locally against a cluster accessible via KUBECONFIG. +# Usage: +# make e2e-local # run all tests +# make e2e-local GINKGO_FOCUS="test name" # run matching tests +# make e2e-local CLUSTER_ID= # use backplane for cluster access +# +# Requires: KUBECONFIG set, or CLUSTER_ID + ocm login for backplane access. +.PHONY: e2e-local +e2e-local: e2e-binary-build + @if [ -n "$(CLUSTER_ID)" ] && [ -z "$(KUBECONFIG)" ]; then \ + echo "Logging into cluster $(CLUSTER_ID) via backplane..."; \ + ocm backplane login $(CLUSTER_ID); \ + fi + @echo "Running e2e tests against $${KUBECONFIG:-backplane cluster}..." + DISABLE_JUNIT_REPORT=true ./e2e.test \ + --ginkgo.v \ + --ginkgo.junit-report=e2e-local-junit.xml \ + $(if $(GINKGO_FOCUS),--ginkgo.focus="$(GINKGO_FOCUS)") \ + $(if $(GINKGO_LABEL_FILTER),--ginkgo.label-filter="$(GINKGO_LABEL_FILTER)") + # push e2e image tagged as latest and as repo commit hash .PHONY: e2e-image-build-push e2e-image-build-push: container-engine-login diff --git a/boilerplate/openshift/golang-osd-e2e/update b/boilerplate/openshift/golang-osd-e2e/update index 7d2c25ad..fbd86675 100755 --- a/boilerplate/openshift/golang-osd-e2e/update +++ b/boilerplate/openshift/golang-osd-e2e/update @@ -12,8 +12,13 @@ source $CONVENTION_ROOT/_lib/common.sh REPO_ROOT=$(git rev-parse --show-toplevel) OPERATOR_NAME=$(sed -n 's/.*OperatorName .*=.*"\([^"]*\)".*/\1/p' "${REPO_ROOT}/config/config.go") +GO_MODULE_PATH=$(awk '/^module / { print $2; exit }' "${REPO_ROOT}/go.mod") E2E_SUITE_DIRECTORY=$REPO_ROOT/test/e2e +if [[ -z "${GO_MODULE_PATH}" ]]; then + err "Could not read module path from ${REPO_ROOT}/go.mod" +fi + # Update operator name in templates OPERATOR_UNDERSCORE_NAME=${OPERATOR_NAME//-/_} OPERATOR_PROPER_NAME=$(echo "$OPERATOR_NAME" | sed 's/-/ /g' | awk '{for(i=1;i<=NF;i++){ $i=toupper(substr($i,1,1)) substr($i,2) }}1') @@ -21,7 +26,7 @@ OPERATOR_NAME_CAMEL_CASE=${OPERATOR_PROPER_NAME// /} mkdir -p "${E2E_SUITE_DIRECTORY}" -E2E_SUITE_BUILDER_IMAGE=registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.25-openshift-4.21 +E2E_SUITE_BUILDER_IMAGE=registry.ci.openshift.org/openshift/release:rhel-9-release-golang-1.26-openshift-4.22 if [[ -n ${KONFLUX_BUILDS} ]]; then E2E_SUITE_BUILDER_IMAGE="brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_1.26" fi @@ -30,7 +35,7 @@ echo "syncing ${E2E_SUITE_DIRECTORY}/Dockerfile" tee "${E2E_SUITE_DIRECTORY}/Dockerfile" < /(path-to)/kubeconfig -5. Run test suite using - +5. Run test suite using + DISABLE_JUNIT_REPORT=true KUBECONFIG=/(path-to)/kubeconfig ./(path-to)/bin/ginkgo --tags=osde2e -v test/e2e EOF sed -e "s/\${OPERATOR_NAME}/${OPERATOR_NAME}/" $(dirname $0)/e2e-template.yml >"${E2E_SUITE_DIRECTORY}/e2e-template.yml" +cp $(dirname $0)/gangway-bridge-template.yml "${E2E_SUITE_DIRECTORY}/gangway-bridge-template.yml" + # todo: remove after file is renamed in ALL consumer repos if [ -f "${E2E_SUITE_DIRECTORY}/test-harness-template.yml" ]; then rm -f "${E2E_SUITE_DIRECTORY}/test-harness-template.yml" diff --git a/boilerplate/update b/boilerplate/update index c04389bd..c3f0cc40 100755 --- a/boilerplate/update +++ b/boilerplate/update @@ -165,6 +165,9 @@ if [[ -z "$LATEST_IMAGE_TAG" ]]; then export LATEST_IMAGE_TAG=$(cd $BP_CLONE; git describe --tags --abbrev=0 --match image-v*) fi +# The boilerplate commit hash. Export for convention `update` scripts. +export BOILERPLATE_COMMIT=$(cd ${BP_CLONE} && git rev-parse HEAD) + # Prepare the "nexus makefile include". NEXUS_MK="${CONVENTION_ROOT}/generated-includes.mk" cat <<'EOF'>"${NEXUS_MK}" diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index efb317c3..b83a20cc 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -1,6 +1,6 @@ # THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_9_1.26 as builder -WORKDIR /go/src/github.com/openshift/validation-webhook/ +WORKDIR /go/src/github.com/openshift/managed-cluster-validating-webhooks/ COPY . . RUN CGO_ENABLED=0 GOFLAGS="-mod=mod" go test ./test/e2e -v -c --tags=osde2e -o /e2e.test diff --git a/test/e2e/README.md b/test/e2e/README.md index f6ddc260..16231b89 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -1,14 +1,13 @@ ## Locally running e2e test suite -When updating your operator it's beneficial to add e2e tests for new functionality AND ensure existing functionality is not breaking using e2e tests. -To do this, following steps are recommended +When updating your operator, add e2e tests for new functionality and ensure existing functionality continues to work. The following steps are recommended: -1. Run "make e2e-binary-build" to make sure e2e tests build +1. Run "make e2e-binary-build" to make sure e2e tests build 2. Deploy your new version of operator in a test cluster 3. Run "go install github.com/onsi/ginkgo/ginkgo@latest" -4. Get kubeadmin credentials from your cluster using +4. Get kubeadmin credentials from your cluster using ocm get /api/clusters_mgmt/v1/clusters/(cluster-id)/credentials | jq -r .kubeconfig > /(path-to)/kubeconfig -5. Run test suite using - +5. Run test suite using + DISABLE_JUNIT_REPORT=true KUBECONFIG=/(path-to)/kubeconfig ./(path-to)/bin/ginkgo --tags=osde2e -v test/e2e diff --git a/test/e2e/gangway-bridge-template.yml b/test/e2e/gangway-bridge-template.yml new file mode 100644 index 00000000..eba8951a --- /dev/null +++ b/test/e2e/gangway-bridge-template.yml @@ -0,0 +1,185 @@ +# THIS FILE IS GENERATED BY BOILERPLATE. DO NOT EDIT. +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: gangway-bridge-e2e +parameters: + - name: JOB_NAME + required: true + description: Prow periodic job name to trigger via Gangway + - name: POLL_INTERVAL + value: "60" + description: Seconds between status polls + - name: TIMEOUT + value: "7200" + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "5" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "50400" + description: Kubernetes Job deadline in seconds (must exceed all attempts plus backoff delays) + - name: JOB_ENVS + value: "" + description: Comma-separated KEY=VALUE pairs passed to the Prow job + - name: JOBID + generate: expression + from: "[0-9a-z]{7}" + - name: IMAGE_TAG + value: '' + required: true +objects: + - apiVersion: batch/v1 + kind: Job + metadata: + name: gangway-bridge-${IMAGE_TAG}-${JOBID} + spec: + backoffLimit: 0 + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} + template: + spec: + automountServiceAccountToken: false + restartPolicy: Never + containers: + - name: gangway-bridge + image: quay.io/openshift/origin-tools:latest + command: + - /bin/bash + - -ceu + - | + GW="https://gangway-ci.apps.ci.l2s4.p1.openshiftapps.com/v1/executions" + log() { echo "$(date +%H:%M:%S) $*" >&2; } + + [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } + [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + + # Backoff sum: base 30s doubling each retry = 30*(2^N-1), plus 15s max jitter + MAX_BACKOFF_SUM=$(( 30 * ((1 << MAX_RETRIES) - 1) + MAX_RETRIES * 15 )) + # Each attempt may overshoot TIMEOUT by up to POLL_INTERVAL + status-request + # max-time (30s) on the last poll cycle + POLL_OVERSHOOT=$(( POLL_INTERVAL + 30 )) + # Trigger POST max-time (60s) + worst-case Retry-After (600s) per attempt + TRIGGER_OVERHEAD=$(( 60 + 600 )) + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * (TIMEOUT + POLL_OVERSHOOT + TRIGGER_OVERHEAD) + MAX_BACKOFF_SUM )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi + + BODY='{"job_execution_type":"1"}' + if [[ -n "${JOB_ENVS:-}" ]]; then + ENVS=$(echo "${JOB_ENVS}" | jq -Rn '[inputs // input | split(",")[] | split("=") | {(.[0]): .[1:] | join("=")}] | add' <<< "${JOB_ENVS}") + BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') + fi + + RATE_LIMITED_WAITED=0 + trigger_and_poll() { + local resp_file="/dev/shm/gw_resp.$$" header_file="/dev/shm/gw_hdr.$$" + trap 'rm -f "$resp_file" "$header_file"' RETURN + HTTP_CODE=$(curl -sSL --max-time 60 -X POST \ + -H "Authorization: Bearer ${GANGWAY_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "${BODY}" \ + -o "$resp_file" -D "$header_file" \ + -w '%{http_code}' "${GW}/${JOB_NAME}" 2>/dev/null) || HTTP_CODE=000 + + # Handle 429 — parse Retry-After header (capped at 600s) + if [[ "$HTTP_CODE" == "429" ]]; then + local retry_after + retry_after=$(grep -i '^retry-after:' "$header_file" | awk '{print $2}' | tr -d '\r') + if [[ "$retry_after" =~ ^[0-9]+$ ]] && [[ "$retry_after" -gt 0 ]] && [[ "$retry_after" -le 600 ]]; then + log "Rate limited (429) — sleeping ${retry_after}s (Retry-After)" + sleep "$retry_after" + RATE_LIMITED_WAITED=1 + else + log "Rate limited (429) — no valid Retry-After header" + fi + return 1 # falls through to outer retry with backoff + fi + + # Fail on non-2xx + if [[ "$HTTP_CODE" -lt 200 || "$HTTP_CODE" -ge 300 ]]; then + log "Failed to trigger ${JOB_NAME} (HTTP ${HTTP_CODE})" + return 1 + fi + + RESP=$(cat "$resp_file") + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + while [[ $SECONDS -lt $END ]]; do + sleep "${POLL_INTERVAL}" + S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } + + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + if [[ $RATE_LIMITED_WAITED -eq 1 ]]; then + log "Skipping backoff (already waited for Retry-After)" + RATE_LIMITED_WAITED=0 + else + BACKOFF=$(( 30 * (1 << (ATTEMPT - 1)) )) + [[ $BACKOFF -gt 480 ]] && BACKOFF=480 + JITTER=$(( RANDOM % 16 )) + DELAY=$(( BACKOFF + JITTER )) + log "Retrying in ${DELAY}s (backoff=${BACKOFF}s, jitter=${JITTER}s)..." + sleep "$DELAY" + fi + done + env: + - name: JOB_NAME + value: ${JOB_NAME} + - name: GANGWAY_TOKEN + valueFrom: + secretKeyRef: + name: gangway-api-token + key: token + - name: POLL_INTERVAL + value: ${POLL_INTERVAL} + - name: TIMEOUT + value: ${TIMEOUT} + - name: JOB_ENVS + value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "100m" + memory: "128Mi" + securityContext: + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault diff --git a/test/e2e/validation_webhook_tests.go b/test/e2e/validation_webhook_tests.go index dea1c322..6a8a8e72 100644 --- a/test/e2e/validation_webhook_tests.go +++ b/test/e2e/validation_webhook_tests.go @@ -50,7 +50,7 @@ var _ = Describe("Managed Cluster Validating Webhooks", Ordered, func() { const ( namespaceName = "openshift-validation-webhook" serviceName = "validation-webhook" - daemonsetName = "validation-webhook" + deploymentName = "validation-webhook" configMapName = "webhook-cert" secretName = "webhook-cert" testNsName = "osde2e-temp-ns" @@ -131,14 +131,12 @@ var _ = Describe("Managed Cluster Validating Webhooks", Ordered, func() { err = client.Get(ctx, serviceName, namespaceName, &v1.Service{}) Expect(err).ToNot(HaveOccurred()) - By("checking the daemonset exists") - ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: daemonsetName, Namespace: namespaceName}} - err = wait.For(conditions.New(client.Resources).ResourceMatch(ds, func(object k8s.Object) bool { - d := object.(*appsv1.DaemonSet) - desiredNumScheduled := d.Status.DesiredNumberScheduled - return d.Status.CurrentNumberScheduled == desiredNumScheduled && - d.Status.NumberReady == desiredNumScheduled && - d.Status.NumberAvailable == desiredNumScheduled + By("checking the deployment exists and is ready") + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deploymentName, Namespace: namespaceName}} + err = wait.For(conditions.New(client.Resources).ResourceMatch(dep, func(object k8s.Object) bool { + d := object.(*appsv1.Deployment) + return d.Status.ReadyReplicas > 0 && + d.Status.ReadyReplicas == d.Status.Replicas })) Expect(err).ToNot(HaveOccurred()) })