From f58a6e372952e2e79ac4486a6c37d377871a6e5f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 11:55:31 -0700 Subject: [PATCH 01/41] fix(acs-ci): use absolute gcloud path --- tasks/acs-ci-nightly/workflow/preflight.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 8762ae4..ea07b6a 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -50,8 +50,8 @@ agent: fi echo - echo "GCS: gcloud storage ls gs://$GCS_BUCKET" - if timeout 30s gcloud storage ls "gs://$GCS_BUCKET" \ + echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" + if timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ >/dev/null 2>"$gcs_error"; then echo "GCS: PASS" else From 235a584dcfa38e250cef8092e93d312904e01b0f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 12:05:51 -0700 Subject: [PATCH 02/41] fix(acs-ci): pass provider token to gcloud --- tasks/acs-ci-nightly/openshell/README.md | 5 +++++ tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 3 +++ tasks/acs-ci-nightly/workflow/preflight.yaml | 6 +++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/openshell/README.md b/tasks/acs-ci-nightly/openshell/README.md index afe33c4..03c069d 100644 --- a/tasks/acs-ci-nightly/openshell/README.md +++ b/tasks/acs-ci-nightly/openshell/README.md @@ -18,6 +18,11 @@ points Google Cloud CLI tools at OpenShell's combined CA bundle so the sandbox verifies the proxy certificate without disabling TLS. The image keeps standalone `gsutil` only as a temporary compatibility fallback. +The Google Cloud CLI is invoked with +`CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"` so it uses the +OpenShell-managed short-lived provider token rather than a local gcloud +account or service-account key. + The `github_git` policy is intentionally unauthenticated and read-only because the StackRox repositories used by this task are public. The Atlassian and Prow GCS provider instances remain gateway-owned; repository source being public diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index eb68b83..def8b5a 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -12,6 +12,9 @@ Run only the CI failure analysis portion of the ACS triage agent. `gh auth login` or push to them. 4. Find failures from the last 24 hours in the Prow nightly jobs under `gs://${GCS_BUCKET:-test-platform-results}/logs/` using `gcloud storage`. + Prefix each gcloud command with + `CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"`; this uses the + OpenShell-managed provider token without requiring `gcloud auth login`. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index ea07b6a..7e3b238 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -51,7 +51,11 @@ agent: echo echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" - if timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ + if [ -z "${GCP_SA_ACCESS_TOKEN:-}" ]; then + echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" + status=1 + elif CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ + timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ >/dev/null 2>"$gcs_error"; then echo "GCS: PASS" else From 78b76ab37d242856be5ef4a7b88d0df8eb59f6c6 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 12:10:03 -0700 Subject: [PATCH 03/41] fix(acs-ci): avoid workflow interpolation in token check --- tasks/acs-ci-nightly/workflow/preflight.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 7e3b238..3c44553 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -51,7 +51,7 @@ agent: echo echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" - if [ -z "${GCP_SA_ACCESS_TOKEN:-}" ]; then + if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" status=1 elif CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ From 5cc6dc4865b25b51eebfd256a707409fa2ae998a Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 12:23:41 -0700 Subject: [PATCH 04/41] docs(acs-ci): describe bounded GCS discovery --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index def8b5a..aaabef5 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -11,8 +11,9 @@ Run only the CI failure analysis portion of the ACS triage agent. `git clone` or `git fetch` for these public repositories; do not run `gh auth login` or push to them. 4. Find failures from the last 24 hours in the Prow nightly jobs under - `gs://${GCS_BUCKET:-test-platform-results}/logs/` using `gcloud storage`. - Prefix each gcloud command with + `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use the bounded GCS JSON + prefix query in the coordinator instructions for discovery, then use + `gcloud storage` for object reads. Prefix each gcloud command with `CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"`; this uses the OpenShell-managed provider token without requiring `gcloud auth login`. 5. Spawn the repository's CI analysis agents as instructed and wait for their From 0d494eff118bdbd18b69bcd28949a08ff75c72b1 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 13:21:09 -0700 Subject: [PATCH 05/41] feat(acs-ci): configure triage lookback window --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 13 +++++++------ tasks/acs-ci-nightly/workflow/harness.yaml | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index aaabef5..d3e2849 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -10,12 +10,13 @@ Run only the CI failure analysis portion of the ACS triage agent. repositories only when needed for the analysis. Use unauthenticated HTTPS `git clone` or `git fetch` for these public repositories; do not run `gh auth login` or push to them. -4. Find failures from the last 24 hours in the Prow nightly jobs under - `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use the bounded GCS JSON - prefix query in the coordinator instructions for discovery, then use - `gcloud storage` for object reads. Prefix each gcloud command with - `CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"`; this uses the - OpenShell-managed provider token without requiring `gcloud auth login`. +4. Find failures from the configured lookback window in the Prow nightly jobs + under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use + `TRIAGE_LOOKBACK_DAYS` (default `1`) as the number of days to include. Use + the bounded GCS JSON prefix query in the coordinator instructions for + discovery, then use `gcloud storage` for object reads. Prefix each gcloud + command with `CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"`; this uses + the OpenShell-managed provider token without requiring `gcloud auth login`. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index 4c08206..b4a4dc1 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -21,6 +21,7 @@ sandbox: CONFLUENCE_URL: https://redhat.atlassian.net/wiki READ_ONLY_MODE: "true" TRIAGE_RUN_URL: ${TRIAGE_RUN_URL} + TRIAGE_LOOKBACK_DAYS: ${TRIAGE_LOOKBACK_DAYS} GCS_BUCKET: test-platform-results-public CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE: /etc/openshell-tls/ca-bundle.pem CURL_CA_BUNDLE: /etc/openshell-tls/ca-bundle.pem From 5bc1a4f94edbd85e462d82230f6e7d30ebd3e1d2 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 13:57:00 -0700 Subject: [PATCH 06/41] test(acs-ci): probe nightly GCS objects in preflight --- tasks/acs-ci-nightly/workflow/preflight.yaml | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 3c44553..071e260 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -23,6 +23,8 @@ agent: mkdir -p /sandbox/artifacts log=/sandbox/artifacts/preflight.log gcs_error=/tmp/preflight-gcs-error + gcs_prefix_error=/tmp/preflight-gcs-prefix-error + gcs_object_error=/tmp/preflight-gcs-object-error jira_error=/tmp/preflight-jira-error status=0 @@ -64,6 +66,61 @@ agent: sed -n '1,20p' "$gcs_error" status=1 fi + + echo + echo "GCS object probe: latest-build.txt and finished.json" + if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then + echo "GCS object probe: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" + status=1 + else + nightly_response=$(curl --silent --show-error --fail \ + --connect-timeout 10 --max-time 30 \ + --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ + --get \ + --data-urlencode 'prefix=logs/branch-ci-stackrox-stackrox-nightlies-' \ + --data-urlencode 'delimiter=/' \ + --data-urlencode 'maxResults=1' \ + "https://storage.googleapis.com/storage/v1/b/$GCS_BUCKET/o" \ + 2>"$gcs_prefix_error") + prefix_status=$? + if [ "$prefix_status" -ne 0 ]; then + echo "GCS object probe: FAIL (nightly prefix listing, exit $prefix_status)" + sed -n '1,20p' "$gcs_prefix_error" + status=1 + else + nightly_prefix=$(printf '%s' "$nightly_response" | jq -r '.prefixes[0] // empty') + if [ -z "$nightly_prefix" ]; then + echo "GCS object probe: FAIL (no nightly prefixes returned)" + status=1 + else + nightly_job="gs://$GCS_BUCKET/${nightly_prefix%/}" + build=$(CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ + timeout 30s /opt/google-cloud-sdk/bin/gcloud storage cat \ + "$nightly_job/latest-build.txt" 2>"$gcs_object_error") + build_status=$? + build=$(printf '%s' "$build" | tr -d '\r\n') + if [ "$build_status" -ne 0 ] || [ -z "$build" ]; then + echo "GCS object probe: FAIL (latest-build.txt, exit $build_status)" + sed -n '1,20p' "$gcs_object_error" + status=1 + else + finished=$(CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ + timeout 30s /opt/google-cloud-sdk/bin/gcloud storage cat \ + "$nightly_job/$build/finished.json" 2>"$gcs_object_error") + finished_status=$? + if [ "$finished_status" -ne 0 ] || [ -z "$finished" ]; then + echo "GCS object probe: FAIL (finished.json, exit $finished_status)" + sed -n '1,20p' "$gcs_object_error" + status=1 + else + timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // "unknown"') + result=$(printf '%s' "$finished" | jq -r '.result // "unknown"') + echo "GCS object probe: PASS (job=$nightly_job build=$build timestamp=$timestamp result=$result)" + fi + fi + fi + fi + fi } >"$log" 2>&1 cat "$log" From 7fb89431e564d9f52b361ebf290398cead741a62 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 13:59:02 -0700 Subject: [PATCH 07/41] fix(acs-ci): avoid workflow interpolation in probe --- tasks/acs-ci-nightly/workflow/preflight.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 071e260..ea2b4f8 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -93,7 +93,8 @@ agent: echo "GCS object probe: FAIL (no nightly prefixes returned)" status=1 else - nightly_job="gs://$GCS_BUCKET/${nightly_prefix%/}" + nightly_prefix=$(printf '%s' "$nightly_prefix" | sed 's:/*$::') + nightly_job="gs://$GCS_BUCKET/$nightly_prefix" build=$(CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ timeout 30s /opt/google-cloud-sdk/bin/gcloud storage cat \ "$nightly_job/latest-build.txt" 2>"$gcs_object_error") From 4e0352518a523feb7e8a64e856294346a371d61c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 14:03:25 -0700 Subject: [PATCH 08/41] test(acs-ci): probe GCS objects with JSON API --- tasks/acs-ci-nightly/workflow/preflight.yaml | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index ea2b4f8..32800e4 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -95,22 +95,28 @@ agent: else nightly_prefix=$(printf '%s' "$nightly_prefix" | sed 's:/*$::') nightly_job="gs://$GCS_BUCKET/$nightly_prefix" - build=$(CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ - timeout 30s /opt/google-cloud-sdk/bin/gcloud storage cat \ - "$nightly_job/latest-build.txt" 2>"$gcs_object_error") + build_path=$(printf '%s/latest-build.txt' "$nightly_prefix" | jq -sRr @uri) + build=$(curl --silent --show-error --fail \ + --connect-timeout 10 --max-time 30 \ + --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ + "https://storage.googleapis.com/download/storage/v1/b/$GCS_BUCKET/o/$build_path?alt=media" \ + 2>"$gcs_object_error") build_status=$? build=$(printf '%s' "$build" | tr -d '\r\n') if [ "$build_status" -ne 0 ] || [ -z "$build" ]; then - echo "GCS object probe: FAIL (latest-build.txt, exit $build_status)" + echo "GCS object probe: FAIL (latest-build.txt via JSON API, exit $build_status)" sed -n '1,20p' "$gcs_object_error" status=1 else - finished=$(CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ - timeout 30s /opt/google-cloud-sdk/bin/gcloud storage cat \ - "$nightly_job/$build/finished.json" 2>"$gcs_object_error") + finished_path=$(printf '%s/%s/finished.json' "$nightly_prefix" "$build" | jq -sRr @uri) + finished=$(curl --silent --show-error --fail \ + --connect-timeout 10 --max-time 30 \ + --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ + "https://storage.googleapis.com/download/storage/v1/b/$GCS_BUCKET/o/$finished_path?alt=media" \ + 2>"$gcs_object_error") finished_status=$? if [ "$finished_status" -ne 0 ] || [ -z "$finished" ]; then - echo "GCS object probe: FAIL (finished.json, exit $finished_status)" + echo "GCS object probe: FAIL (finished.json via JSON API, exit $finished_status)" sed -n '1,20p' "$gcs_object_error" status=1 else From 90482c4364e89ece01bfe9520953cb7f45e83f68 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 14:08:21 -0700 Subject: [PATCH 09/41] fix(acs-ci): read nightly build results from GCS --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 8 ++-- tasks/acs-ci-nightly/workflow/preflight.yaml | 39 +++++++++++++------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index d3e2849..c7ad43a 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -14,9 +14,11 @@ Run only the CI failure analysis portion of the ACS triage agent. under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use `TRIAGE_LOOKBACK_DAYS` (default `1`) as the number of days to include. Use the bounded GCS JSON prefix query in the coordinator instructions for - discovery, then use `gcloud storage` for object reads. Prefix each gcloud - command with `CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"`; this uses - the OpenShell-managed provider token without requiring `gcloud auth login`. + discovery, enumerate each job's build-directory prefixes, and read the + root `/finished.json` object through the authenticated GCS JSON media + API. The public results bucket does not provide a `latest-build.txt` marker, + and its OpenShell `gcloud storage cat` path is not reliable for these + objects. Use `GCP_SA_ACCESS_TOKEN` as the bearer token without logging it. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 32800e4..73a824e 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -68,7 +68,7 @@ agent: fi echo - echo "GCS object probe: latest-build.txt and finished.json" + echo "GCS object probe: newest build directory and finished.json" if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then echo "GCS object probe: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" status=1 @@ -92,20 +92,30 @@ agent: if [ -z "$nightly_prefix" ]; then echo "GCS object probe: FAIL (no nightly prefixes returned)" status=1 + else + nightly_prefix=$(printf '%s' "$nightly_prefix" | sed 's:/*$::') + nightly_job="gs://$GCS_BUCKET/$nightly_prefix" + build_response=$(curl --silent --show-error --fail \ + --connect-timeout 10 --max-time 30 \ + --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ + --get \ + --data-urlencode "prefix=$nightly_prefix/" \ + --data-urlencode 'delimiter=/' \ + --data-urlencode 'maxResults=1000' \ + "https://storage.googleapis.com/storage/v1/b/$GCS_BUCKET/o" \ + 2>"$gcs_prefix_error") + build_list_status=$? + if [ "$build_list_status" -ne 0 ]; then + echo "GCS object probe: FAIL (build prefix listing, exit $build_list_status)" + sed -n '1,20p' "$gcs_prefix_error" + status=1 else - nightly_prefix=$(printf '%s' "$nightly_prefix" | sed 's:/*$::') - nightly_job="gs://$GCS_BUCKET/$nightly_prefix" - build_path=$(printf '%s/latest-build.txt' "$nightly_prefix" | jq -sRr @uri) - build=$(curl --silent --show-error --fail \ - --connect-timeout 10 --max-time 30 \ - --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ - "https://storage.googleapis.com/download/storage/v1/b/$GCS_BUCKET/o/$build_path?alt=media" \ - 2>"$gcs_object_error") - build_status=$? - build=$(printf '%s' "$build" | tr -d '\r\n') - if [ "$build_status" -ne 0 ] || [ -z "$build" ]; then - echo "GCS object probe: FAIL (latest-build.txt via JSON API, exit $build_status)" - sed -n '1,20p' "$gcs_object_error" + build=$(printf '%s' "$build_response" | jq -r \ + --arg prefix "$nightly_prefix/" \ + '.prefixes[]? | select(startswith($prefix)) | ltrimstr($prefix) | rtrimstr("/")' \ + | sort -n | tail -n 1) + if [ -z "$build" ]; then + echo "GCS object probe: FAIL (no build prefixes returned)" status=1 else finished_path=$(printf '%s/%s/finished.json' "$nightly_prefix" "$build" | jq -sRr @uri) @@ -126,6 +136,7 @@ agent: fi fi fi + fi fi fi } >"$log" 2>&1 From 627bbdb8000e63c24811c61b20ea405a11b19af6 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 14:12:06 -0700 Subject: [PATCH 10/41] fix(acs-ci): use direct GCS object endpoint --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 9 +++++---- tasks/acs-ci-nightly/workflow/preflight.yaml | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index c7ad43a..a75e062 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -15,10 +15,11 @@ Run only the CI failure analysis portion of the ACS triage agent. `TRIAGE_LOOKBACK_DAYS` (default `1`) as the number of days to include. Use the bounded GCS JSON prefix query in the coordinator instructions for discovery, enumerate each job's build-directory prefixes, and read the - root `/finished.json` object through the authenticated GCS JSON media - API. The public results bucket does not provide a `latest-build.txt` marker, - and its OpenShell `gcloud storage cat` path is not reliable for these - objects. Use `GCP_SA_ACCESS_TOKEN` as the bearer token without logging it. + root `/finished.json` object through the authenticated direct GCS + object endpoint. The public results bucket does not provide a + `latest-build.txt` marker, and the OpenShell JSON media/gcloud object-read + routes are not reliable for these objects. Use `GCP_SA_ACCESS_TOKEN` as the + bearer token without logging it. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 73a824e..515b4f2 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -118,15 +118,15 @@ agent: echo "GCS object probe: FAIL (no build prefixes returned)" status=1 else - finished_path=$(printf '%s/%s/finished.json' "$nightly_prefix" "$build" | jq -sRr @uri) + finished_object=$(printf '%s/%s/finished.json' "$nightly_prefix" "$build") finished=$(curl --silent --show-error --fail \ --connect-timeout 10 --max-time 30 \ --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ - "https://storage.googleapis.com/download/storage/v1/b/$GCS_BUCKET/o/$finished_path?alt=media" \ + "https://storage.googleapis.com/$GCS_BUCKET/$finished_object" \ 2>"$gcs_object_error") finished_status=$? if [ "$finished_status" -ne 0 ] || [ -z "$finished" ]; then - echo "GCS object probe: FAIL (finished.json via JSON API, exit $finished_status)" + echo "GCS object probe: FAIL (finished.json via direct object endpoint, exit $finished_status)" sed -n '1,20p' "$gcs_object_error" status=1 else From 0afa08e497d3394befbc2996c56ea27f22b21c01 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 14:43:37 -0700 Subject: [PATCH 11/41] cleanup(acs-ci): remove temporary object probe --- tasks/acs-ci-nightly/README.md | 5 +- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 5 +- tasks/acs-ci-nightly/workflow/preflight.yaml | 74 -------------------- 3 files changed, 5 insertions(+), 79 deletions(-) diff --git a/tasks/acs-ci-nightly/README.md b/tasks/acs-ci-nightly/README.md index e6f6e94..bd60c8b 100644 --- a/tasks/acs-ci-nightly/README.md +++ b/tasks/acs-ci-nightly/README.md @@ -26,8 +26,9 @@ validated. the caller's output directory. The output is optional so partial diagnostics can still be retained when analysis fails. - Preflight: the trusted caller may run `workflow/preflight.yaml` in the same - workspace before the main task. It verifies read-only GCS and Jira access - from inside the sandbox without exposing response bodies or credentials. + workspace before the main task. It verifies read-only GCS bucket-listing and + Jira access from inside the sandbox without exposing response bodies or + credentials. - Cleanup: the sandbox and host-side source staging are removed after outputs are downloaded. Downloaded artifacts and any external reads remain with the caller. diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index a75e062..68527ca 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -17,9 +17,8 @@ Run only the CI failure analysis portion of the ACS triage agent. discovery, enumerate each job's build-directory prefixes, and read the root `/finished.json` object through the authenticated direct GCS object endpoint. The public results bucket does not provide a - `latest-build.txt` marker, and the OpenShell JSON media/gcloud object-read - routes are not reliable for these objects. Use `GCP_SA_ACCESS_TOKEN` as the - bearer token without logging it. + `latest-build.txt` marker. Use `GCP_SA_ACCESS_TOKEN` as the bearer token + without logging it. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 515b4f2..42f1f77 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -23,8 +23,6 @@ agent: mkdir -p /sandbox/artifacts log=/sandbox/artifacts/preflight.log gcs_error=/tmp/preflight-gcs-error - gcs_prefix_error=/tmp/preflight-gcs-prefix-error - gcs_object_error=/tmp/preflight-gcs-object-error jira_error=/tmp/preflight-jira-error status=0 @@ -67,78 +65,6 @@ agent: status=1 fi - echo - echo "GCS object probe: newest build directory and finished.json" - if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then - echo "GCS object probe: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" - status=1 - else - nightly_response=$(curl --silent --show-error --fail \ - --connect-timeout 10 --max-time 30 \ - --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ - --get \ - --data-urlencode 'prefix=logs/branch-ci-stackrox-stackrox-nightlies-' \ - --data-urlencode 'delimiter=/' \ - --data-urlencode 'maxResults=1' \ - "https://storage.googleapis.com/storage/v1/b/$GCS_BUCKET/o" \ - 2>"$gcs_prefix_error") - prefix_status=$? - if [ "$prefix_status" -ne 0 ]; then - echo "GCS object probe: FAIL (nightly prefix listing, exit $prefix_status)" - sed -n '1,20p' "$gcs_prefix_error" - status=1 - else - nightly_prefix=$(printf '%s' "$nightly_response" | jq -r '.prefixes[0] // empty') - if [ -z "$nightly_prefix" ]; then - echo "GCS object probe: FAIL (no nightly prefixes returned)" - status=1 - else - nightly_prefix=$(printf '%s' "$nightly_prefix" | sed 's:/*$::') - nightly_job="gs://$GCS_BUCKET/$nightly_prefix" - build_response=$(curl --silent --show-error --fail \ - --connect-timeout 10 --max-time 30 \ - --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ - --get \ - --data-urlencode "prefix=$nightly_prefix/" \ - --data-urlencode 'delimiter=/' \ - --data-urlencode 'maxResults=1000' \ - "https://storage.googleapis.com/storage/v1/b/$GCS_BUCKET/o" \ - 2>"$gcs_prefix_error") - build_list_status=$? - if [ "$build_list_status" -ne 0 ]; then - echo "GCS object probe: FAIL (build prefix listing, exit $build_list_status)" - sed -n '1,20p' "$gcs_prefix_error" - status=1 - else - build=$(printf '%s' "$build_response" | jq -r \ - --arg prefix "$nightly_prefix/" \ - '.prefixes[]? | select(startswith($prefix)) | ltrimstr($prefix) | rtrimstr("/")' \ - | sort -n | tail -n 1) - if [ -z "$build" ]; then - echo "GCS object probe: FAIL (no build prefixes returned)" - status=1 - else - finished_object=$(printf '%s/%s/finished.json' "$nightly_prefix" "$build") - finished=$(curl --silent --show-error --fail \ - --connect-timeout 10 --max-time 30 \ - --header "Authorization: Bearer $GCP_SA_ACCESS_TOKEN" \ - "https://storage.googleapis.com/$GCS_BUCKET/$finished_object" \ - 2>"$gcs_object_error") - finished_status=$? - if [ "$finished_status" -ne 0 ] || [ -z "$finished" ]; then - echo "GCS object probe: FAIL (finished.json via direct object endpoint, exit $finished_status)" - sed -n '1,20p' "$gcs_object_error" - status=1 - else - timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // "unknown"') - result=$(printf '%s' "$finished" | jq -r '.result // "unknown"') - echo "GCS object probe: PASS (job=$nightly_job build=$build timestamp=$timestamp result=$result)" - fi - fi - fi - fi - fi - fi } >"$log" 2>&1 cat "$log" From ec68895b6804188c1da3086e8727b4f06a00627d Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 14:51:06 -0700 Subject: [PATCH 12/41] fix(acs-ci): remove obsolete write permission --- images/stackrox/sandbox-stackrox-ci/settings.json | 1 - 1 file changed, 1 deletion(-) diff --git a/images/stackrox/sandbox-stackrox-ci/settings.json b/images/stackrox/sandbox-stackrox-ci/settings.json index 3433177..e0e4562 100644 --- a/images/stackrox/sandbox-stackrox-ci/settings.json +++ b/images/stackrox/sandbox-stackrox-ci/settings.json @@ -17,7 +17,6 @@ "Bash(timeout *)", "Bash(while *)", "Read", - "Write(artifacts/*)", "Edit(artifacts/*)", "Glob", "Grep", From 54d05930c78d0fefc93b952bbaee26057d2d386b Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 15:02:29 -0700 Subject: [PATCH 13/41] fix(acs-ci): configure gcloud storage auth once --- tasks/acs-ci-nightly/openshell/README.md | 9 +++++---- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 8 +++----- tasks/acs-ci-nightly/workflow/harness.yaml | 6 ++++++ tasks/acs-ci-nightly/workflow/preflight.yaml | 13 ++++++++++--- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tasks/acs-ci-nightly/openshell/README.md b/tasks/acs-ci-nightly/openshell/README.md index 03c069d..bf58f07 100644 --- a/tasks/acs-ci-nightly/openshell/README.md +++ b/tasks/acs-ci-nightly/openshell/README.md @@ -18,10 +18,11 @@ points Google Cloud CLI tools at OpenShell's combined CA bundle so the sandbox verifies the proxy certificate without disabling TLS. The image keeps standalone `gsutil` only as a temporary compatibility fallback. -The Google Cloud CLI is invoked with -`CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN"` so it uses the -OpenShell-managed short-lived provider token rather than a local gcloud -account or service-account key. +The task payload sets `BASH_ENV` to a runtime-only shell fragment that maps +the provider-injected `GCP_SA_ACCESS_TOKEN` to gcloud's +`CLOUDSDK_AUTH_ACCESS_TOKEN`. The token value is never interpolated into the +workflow or payload, and gcloud therefore uses the OpenShell-managed +short-lived provider token rather than a local account or service-account key. The `github_git` policy is intentionally unauthenticated and read-only because the StackRox repositories used by this task are public. The Atlassian and Prow diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index 68527ca..51ad101 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -14,11 +14,9 @@ Run only the CI failure analysis portion of the ACS triage agent. under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use `TRIAGE_LOOKBACK_DAYS` (default `1`) as the number of days to include. Use the bounded GCS JSON prefix query in the coordinator instructions for - discovery, enumerate each job's build-directory prefixes, and read the - root `/finished.json` object through the authenticated direct GCS - object endpoint. The public results bucket does not provide a - `latest-build.txt` marker. Use `GCP_SA_ACCESS_TOKEN` as the bearer token - without logging it. + top-level job discovery, then use `gcloud storage` to enumerate exact + job/build prefixes and read the root `/finished.json` object. Do not + infer a job's result from a `latest-build.txt` marker. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index b4a4dc1..db25d6b 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -23,6 +23,7 @@ sandbox: TRIAGE_RUN_URL: ${TRIAGE_RUN_URL} TRIAGE_LOOKBACK_DAYS: ${TRIAGE_LOOKBACK_DAYS} GCS_BUCKET: test-platform-results-public + BASH_ENV: /sandbox/.bash_env CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE: /etc/openshell-tls/ca-bundle.pem CURL_CA_BUNDLE: /etc/openshell-tls/ca-bundle.pem @@ -34,6 +35,11 @@ source: payloads: - source: CI-NIGHTLY.md destination: /sandbox/CI-NIGHTLY.md + - content: | + if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then + export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" + fi + destination: /sandbox/.bash_env agent: type: claude diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 42f1f77..949f645 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -11,11 +11,12 @@ sandbox: env: JIRA_URL: https://redhat.atlassian.net GCS_BUCKET: test-platform-results-public + BASH_ENV: /sandbox/.bash_env CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE: /etc/openshell-tls/ca-bundle.pem CURL_CA_BUNDLE: /etc/openshell-tls/ca-bundle.pem agent: - type: /bin/sh + type: /bin/bash args: - -c - | @@ -54,8 +55,7 @@ agent: if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" status=1 - elif CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" \ - timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ + elif timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ >/dev/null 2>"$gcs_error"; then echo "GCS: PASS" else @@ -70,6 +70,13 @@ agent: cat "$log" exit "$status" +payloads: + - content: | + if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then + export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" + fi + destination: /sandbox/.bash_env + outputs: - source: /sandbox/artifacts/preflight.log destination: preflight.log From 373442ebbcbf50da8fe97141f1330b34a7e9d3fe Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 15:03:59 -0700 Subject: [PATCH 14/41] chore(acs-ci): keep default triage window --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 10 +++++----- tasks/acs-ci-nightly/workflow/harness.yaml | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index 51ad101..a7c430e 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -12,11 +12,11 @@ Run only the CI failure analysis portion of the ACS triage agent. `gh auth login` or push to them. 4. Find failures from the configured lookback window in the Prow nightly jobs under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use - `TRIAGE_LOOKBACK_DAYS` (default `1`) as the number of days to include. Use - the bounded GCS JSON prefix query in the coordinator instructions for - top-level job discovery, then use `gcloud storage` to enumerate exact - job/build prefixes and read the root `/finished.json` object. Do not - infer a job's result from a `latest-build.txt` marker. + the existing 24-hour triage window. Use the bounded GCS JSON prefix query + in the coordinator instructions for top-level job discovery, then use + `gcloud storage` to enumerate exact job/build prefixes and read the root + `/finished.json` object. Do not infer a job's result from a + `latest-build.txt` marker. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index db25d6b..3ab0976 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -21,7 +21,6 @@ sandbox: CONFLUENCE_URL: https://redhat.atlassian.net/wiki READ_ONLY_MODE: "true" TRIAGE_RUN_URL: ${TRIAGE_RUN_URL} - TRIAGE_LOOKBACK_DAYS: ${TRIAGE_LOOKBACK_DAYS} GCS_BUCKET: test-platform-results-public BASH_ENV: /sandbox/.bash_env CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE: /etc/openshell-tls/ca-bundle.pem From 1b8efb60374d109fa4176a4faee64303358dc5a8 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 15:24:58 -0700 Subject: [PATCH 15/41] docs(acs-ci): use gcloud for nightly discovery --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index a7c430e..f2e8024 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -12,11 +12,10 @@ Run only the CI failure analysis portion of the ACS triage agent. `gh auth login` or push to them. 4. Find failures from the configured lookback window in the Prow nightly jobs under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use - the existing 24-hour triage window. Use the bounded GCS JSON prefix query - in the coordinator instructions for top-level job discovery, then use - `gcloud storage` to enumerate exact job/build prefixes and read the root - `/finished.json` object. Do not infer a job's result from a - `latest-build.txt` marker. + the existing 24-hour triage window. Use `gcloud storage` to discover + nightly `/finished.json` objects, then enumerate exact job/build + prefixes and read the root result object. Do not infer a job's result from + a `latest-build.txt` marker. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using From 09fd43bb65a1b5924fef5d70d984c37b92cc6616 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 15:40:47 -0700 Subject: [PATCH 16/41] docs(acs-ci): document bounded job discovery --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index f2e8024..a7c430e 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -12,10 +12,11 @@ Run only the CI failure analysis portion of the ACS triage agent. `gh auth login` or push to them. 4. Find failures from the configured lookback window in the Prow nightly jobs under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use - the existing 24-hour triage window. Use `gcloud storage` to discover - nightly `/finished.json` objects, then enumerate exact job/build - prefixes and read the root result object. Do not infer a job's result from - a `latest-build.txt` marker. + the existing 24-hour triage window. Use the bounded GCS JSON prefix query + in the coordinator instructions for top-level job discovery, then use + `gcloud storage` to enumerate exact job/build prefixes and read the root + `/finished.json` object. Do not infer a job's result from a + `latest-build.txt` marker. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using From e3ea8159672fff661d8aeeae4d7b0e73c0952d48 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 16:34:49 -0700 Subject: [PATCH 17/41] fix(acs-ci): add trusted gsutil compatibility wrapper --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 11 +-- tasks/acs-ci-nightly/workflow/gsutil | 91 +++++++++++++++++++++ tasks/acs-ci-nightly/workflow/harness.yaml | 3 + 3 files changed, 98 insertions(+), 7 deletions(-) create mode 100755 tasks/acs-ci-nightly/workflow/gsutil diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index a7c430e..50abb2b 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -10,13 +10,10 @@ Run only the CI failure analysis portion of the ACS triage agent. repositories only when needed for the analysis. Use unauthenticated HTTPS `git clone` or `git fetch` for these public repositories; do not run `gh auth login` or push to them. -4. Find failures from the configured lookback window in the Prow nightly jobs - under `gs://${GCS_BUCKET:-test-platform-results}/logs/`. Use - the existing 24-hour triage window. Use the bounded GCS JSON prefix query - in the coordinator instructions for top-level job discovery, then use - `gcloud storage` to enumerate exact job/build prefixes and read the root - `/finished.json` object. Do not infer a job's result from a - `latest-build.txt` marker. +4. Follow the repository's existing `ci-coordinator.md` instructions for the + Prow failure lookup. The trusted task provides a `gsutil` compatibility + command backed by the configured bucket and `gcloud storage`; do not replace + it with a recursive bucket or wildcard listing. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil new file mode 100755 index 0000000..68d2739 --- /dev/null +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Compatibility shim for the existing ACS triage instructions. Exact reads +# use the image's gcloud binary. The top-level logs listing is bounded because +# recursively listing the bucket is too large for this task. +gcloud_bin=/opt/google-cloud-sdk/bin/gcloud +configured_bucket="${GCS_BUCKET:-test-platform-results-public}" + +rewrite_uri() { + local uri=$1 + case "$uri" in + gs://test-platform-results|gs://test-platform-results/*) + printf 'gs://%s%s\n' "$configured_bucket" "${uri#gs://test-platform-results}" + ;; + *) + printf '%s\n' "$uri" + ;; + esac +} + +list_nightly_jobs() { + local token=${GCP_SA_ACCESS_TOKEN:-} + local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" + local page_token="" + + if [[ -z "$token" ]]; then + echo "gsutil compatibility wrapper: GCP_SA_ACCESS_TOKEN is missing" >&2 + return 1 + fi + + while :; do + local response + local -a page_args=() + if [[ -n "$page_token" ]]; then + page_args=(--data-urlencode "pageToken=$page_token") + fi + + response=$(timeout 90s curl --silent --show-error --fail \ + --connect-timeout 10 --max-time 60 \ + --header "Authorization: Bearer $token" \ + --get \ + --data-urlencode "prefix=logs/branch-ci-stackrox-stackrox-nightlies-" \ + --data-urlencode "delimiter=/" \ + --data-urlencode "maxResults=1000" \ + "${page_args[@]}" "$api") + + printf '%s' "$response" \ + | jq -r --arg bucket "$configured_bucket" \ + '.prefixes[]? | "gs://" + $bucket + "/" + .' + + page_token=$(printf '%s' "$response" | jq -r '.nextPageToken // empty') + [[ -z "$page_token" ]] && break + done +} + +command=${1:-} +shift || true + +case "$command" in + ls) + if [[ $# -ne 1 ]]; then + echo "gsutil compatibility wrapper supports: gsutil ls URI" >&2 + exit 2 + fi + uri=$(rewrite_uri "$1") + if [[ "$uri" == "gs://${configured_bucket}/logs/" ]]; then + list_nightly_jobs + else + if [[ -n "${GCP_SA_ACCESS_TOKEN:-}" && -z "${CLOUDSDK_AUTH_ACCESS_TOKEN:-}" ]]; then + export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" + fi + exec timeout 90s "$gcloud_bin" storage ls "$uri" + fi + ;; + cat) + if [[ $# -ne 1 ]]; then + echo "gsutil compatibility wrapper supports: gsutil cat URI" >&2 + exit 2 + fi + uri=$(rewrite_uri "$1") + if [[ -n "${GCP_SA_ACCESS_TOKEN:-}" && -z "${CLOUDSDK_AUTH_ACCESS_TOKEN:-}" ]]; then + export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" + fi + exec timeout 90s "$gcloud_bin" storage cat "$uri" + ;; + *) + echo "gsutil compatibility wrapper supports only ls and cat" >&2 + exit 2 + ;; +esac diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index 3ab0976..c56d539 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -34,7 +34,10 @@ source: payloads: - source: CI-NIGHTLY.md destination: /sandbox/CI-NIGHTLY.md + - source: gsutil + destination: /sandbox/tools/gsutil - content: | + export PATH="/sandbox/tools:$PATH" if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi From e6de088d1a7552bdd5d23ee233b5d52c193e4e7c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 16:51:44 -0700 Subject: [PATCH 18/41] fix(acs-ci): export gsutil shim to agent shells --- tasks/acs-ci-nightly/workflow/harness.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index c56d539..432f07b 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -38,6 +38,8 @@ payloads: destination: /sandbox/tools/gsutil - content: | export PATH="/sandbox/tools:$PATH" + gsutil() { /bin/bash /sandbox/tools/gsutil "$@"; } + export -f gsutil if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi From c9621f860aff53db878e572fe1a1b1ba841f3fa7 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 16:58:53 -0700 Subject: [PATCH 19/41] fix(acs-ci): resolve latest build in GCS wrapper --- tasks/acs-ci-nightly/workflow/gsutil | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 68d2739..f0e1341 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -79,6 +79,24 @@ case "$command" in exit 2 fi uri=$(rewrite_uri "$1") + if [[ "$uri" == gs://*/logs/*/latest-build.txt ]]; then + job_prefix="${uri%/latest-build.txt}/" + build=$(timeout 90s "$gcloud_bin" storage ls "$job_prefix" \ + | awk -v prefix="$job_prefix" 'index($0, prefix) == 1 { + rest = substr($0, length(prefix) + 1) + if (rest ~ /^[0-9]+\/$/) { + sub(/\/$/, "", rest) + print rest + } + }' \ + | sort -n | tail -n 1) + if [[ -z "$build" ]]; then + echo "gsutil compatibility wrapper: no numeric build for $job_prefix" >&2 + exit 1 + fi + printf '%s\n' "$build" + exit 0 + fi if [[ -n "${GCP_SA_ACCESS_TOKEN:-}" && -z "${CLOUDSDK_AUTH_ACCESS_TOKEN:-}" ]]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi From 6838280660fdbe51e05b325b6e2d503dff306d44 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:10:50 -0700 Subject: [PATCH 20/41] fix(acs-ci): precompute authoritative failure manifest --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 4 ++- tasks/acs-ci-nightly/workflow/gsutil | 38 +++++++++++++++++++++ tasks/acs-ci-nightly/workflow/harness.yaml | 9 +++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index 50abb2b..801fa4c 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -13,7 +13,9 @@ Run only the CI failure analysis portion of the ACS triage agent. 4. Follow the repository's existing `ci-coordinator.md` instructions for the Prow failure lookup. The trusted task provides a `gsutil` compatibility command backed by the configured bucket and `gcloud storage`; do not replace - it with a recursive bucket or wildcard listing. + it with a recursive bucket or wildcard listing. Treat + `/tmp/openshell-nightly-failures.tsv` as authoritative and do not invent + job, build, or GCS path values. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index f0e1341..d09912e 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -54,10 +54,48 @@ list_nightly_jobs() { done } +collect_failures() { + local cutoff + local jobs + cutoff=$(date -u -d '24 hours ago' +%s 2>/dev/null || date -u -v-1d +%s) + jobs=$(list_nightly_jobs) + printf '# job\tbuild\tgcs_path\ttimestamp\n' + + while IFS= read -r job; do + [[ -z "$job" ]] && continue + ( + job_prefix="${job%/}/" + build=$(timeout 90s "$gcloud_bin" storage ls "$job_prefix" \ + | awk -v prefix="$job_prefix" 'index($0, prefix) == 1 { + rest = substr($0, length(prefix) + 1) + if (rest ~ /^[0-9]+\/$/) { + sub(/\/$/, "", rest) + print rest + } + }' \ + | sort -n | tail -n 1) + [[ -n "$build" ]] || exit 0 + gcs_path="${job_prefix}${build}" + finished=$(timeout 90s "$gcloud_bin" storage cat \ + "${gcs_path}/finished.json") + timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') + result=$(printf '%s' "$finished" | jq -r '.result // ""') + if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then + printf '%s\t%s\t%s\t%s\n' \ + "${job#gs://${configured_bucket}/}" "$build" "$gcs_path" "$timestamp" + fi + ) & + done <<<"$jobs" + wait +} + command=${1:-} shift || true case "$command" in + collect) + collect_failures + ;; ls) if [[ $# -ne 1 ]]; then echo "gsutil compatibility wrapper supports: gsutil ls URI" >&2 diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index 432f07b..443777e 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -43,6 +43,15 @@ payloads: if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi + if [ -z "${OPENSHELL_GCS_MANIFEST_STARTED:-}" ]; then + export OPENSHELL_GCS_MANIFEST_STARTED=1 + /bin/bash /sandbox/tools/gsutil collect \ + >/tmp/openshell-nightly-failures.tsv \ + 2>/tmp/openshell-nightly-failures.err || { + echo "collection failed; see /tmp/openshell-nightly-failures.err" \ + >>/tmp/openshell-nightly-failures.tsv + } + fi destination: /sandbox/.bash_env agent: From a814c389ebf20e4ae66d3e4aebda5bf31d1e7a7b Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:14:15 -0700 Subject: [PATCH 21/41] fix(acs-ci): avoid payload variable interpolation --- tasks/acs-ci-nightly/workflow/harness.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index 443777e..e727695 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -43,7 +43,7 @@ payloads: if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi - if [ -z "${OPENSHELL_GCS_MANIFEST_STARTED:-}" ]; then + if ! printenv OPENSHELL_GCS_MANIFEST_STARTED >/dev/null 2>&1; then export OPENSHELL_GCS_MANIFEST_STARTED=1 /bin/bash /sandbox/tools/gsutil collect \ >/tmp/openshell-nightly-failures.tsv \ From 14af438f5aec85047de646e71a45dd5dffced5a6 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:22:36 -0700 Subject: [PATCH 22/41] fix(acs-ci): handle first GCS listing page --- tasks/acs-ci-nightly/workflow/gsutil | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index d09912e..0cef9c9 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -31,19 +31,20 @@ list_nightly_jobs() { while :; do local response - local -a page_args=() + local -a query_args=( + --data-urlencode "prefix=logs/branch-ci-stackrox-stackrox-nightlies-" + --data-urlencode "delimiter=/" + --data-urlencode "maxResults=1000" + ) if [[ -n "$page_token" ]]; then - page_args=(--data-urlencode "pageToken=$page_token") + query_args+=(--data-urlencode "pageToken=$page_token") fi response=$(timeout 90s curl --silent --show-error --fail \ --connect-timeout 10 --max-time 60 \ --header "Authorization: Bearer $token" \ --get \ - --data-urlencode "prefix=logs/branch-ci-stackrox-stackrox-nightlies-" \ - --data-urlencode "delimiter=/" \ - --data-urlencode "maxResults=1000" \ - "${page_args[@]}" "$api") + "${query_args[@]}" "$api") printf '%s' "$response" \ | jq -r --arg bucket "$configured_bucket" \ From 4d62c9b07e487ca0b575fe0747d727773fb2062e Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:31:46 -0700 Subject: [PATCH 23/41] fix(acs-ci): build manifest from public GCS API --- tasks/acs-ci-nightly/workflow/gsutil | 75 ++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 0cef9c9..25f1088 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -24,11 +24,6 @@ list_nightly_jobs() { local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" local page_token="" - if [[ -z "$token" ]]; then - echo "gsutil compatibility wrapper: GCP_SA_ACCESS_TOKEN is missing" >&2 - return 1 - fi - while :; do local response local -a query_args=( @@ -40,11 +35,17 @@ list_nightly_jobs() { query_args+=(--data-urlencode "pageToken=$page_token") fi - response=$(timeout 90s curl --silent --show-error --fail \ - --connect-timeout 10 --max-time 60 \ - --header "Authorization: Bearer $token" \ - --get \ - "${query_args[@]}" "$api") + local -a curl_args=( + --silent --show-error --fail + --connect-timeout 10 --max-time 60 + --get + "${query_args[@]}" + "$api" + ) + if [[ -n "$token" ]]; then + curl_args+=(--header "Authorization: Bearer $token") + fi + response=$(timeout 90s curl "${curl_args[@]}") printf '%s' "$response" \ | jq -r --arg bucket "$configured_bucket" \ @@ -55,6 +56,47 @@ list_nightly_jobs() { done } +list_builds() { + local job=$1 + local token=${GCP_SA_ACCESS_TOKEN:-} + local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" + local object_prefix="${job#gs://${configured_bucket}/}" + local -a curl_args=( + --silent --show-error --fail + --connect-timeout 10 --max-time 60 + --get + --data-urlencode "prefix=$object_prefix" + --data-urlencode "delimiter=/" + --data-urlencode "maxResults=1000" + "$api" + ) + if [[ -n "$token" ]]; then + curl_args+=(--header "Authorization: Bearer $token") + fi + timeout 90s curl "${curl_args[@]}" \ + | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' +} + +read_finished() { + local gcs_path=$1 + local token=${GCP_SA_ACCESS_TOKEN:-} + local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" + local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" + local encoded_object + encoded_object=$(jq -rn --arg object "$object_path" '$object | @uri') + local -a curl_args=( + --silent --show-error --fail + --connect-timeout 10 --max-time 60 + --get + --data-urlencode "alt=media" + "$api/$encoded_object" + ) + if [[ -n "$token" ]]; then + curl_args+=(--header "Authorization: Bearer $token") + fi + timeout 90s curl "${curl_args[@]}" +} + collect_failures() { local cutoff local jobs @@ -66,19 +108,10 @@ collect_failures() { [[ -z "$job" ]] && continue ( job_prefix="${job%/}/" - build=$(timeout 90s "$gcloud_bin" storage ls "$job_prefix" \ - | awk -v prefix="$job_prefix" 'index($0, prefix) == 1 { - rest = substr($0, length(prefix) + 1) - if (rest ~ /^[0-9]+\/$/) { - sub(/\/$/, "", rest) - print rest - } - }' \ - | sort -n | tail -n 1) + build=$(list_builds "$job" | sort -n | tail -n 1) [[ -n "$build" ]] || exit 0 gcs_path="${job_prefix}${build}" - finished=$(timeout 90s "$gcloud_bin" storage cat \ - "${gcs_path}/finished.json") + finished=$(read_finished "$gcs_path") timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') result=$(printf '%s' "$finished" | jq -r '.result // ""') if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then From 18c19a5497137d9ff1bfdb1a8a2ec8a0cb6f6112 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:36:38 -0700 Subject: [PATCH 24/41] chore(acs-ci): capture manifest diagnostics --- tasks/acs-ci-nightly/workflow/harness.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index e727695..e0d7e92 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -65,3 +65,9 @@ outputs: - source: /sandbox/acs-triage-agent/artifacts/ci-triage.json destination: ci-triage.json required: false + - source: /tmp/openshell-nightly-failures.tsv + destination: nightly-failures.tsv + required: false + - source: /tmp/openshell-nightly-failures.err + destination: nightly-failures.err + required: false From 9ce1bd686b6700e8487befbc41ec1835273cf81a Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:39:53 -0700 Subject: [PATCH 25/41] fix(acs-ci): stage manifest diagnostics under sandbox --- tasks/acs-ci-nightly/workflow/harness.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index e0d7e92..0e44bc4 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -43,15 +43,17 @@ payloads: if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi - if ! printenv OPENSHELL_GCS_MANIFEST_STARTED >/dev/null 2>&1; then - export OPENSHELL_GCS_MANIFEST_STARTED=1 - /bin/bash /sandbox/tools/gsutil collect \ - >/tmp/openshell-nightly-failures.tsv \ - 2>/tmp/openshell-nightly-failures.err || { + if ! printenv OPENSHELL_GCS_MANIFEST_STARTED >/dev/null 2>&1; then + export OPENSHELL_GCS_MANIFEST_STARTED=1 + /bin/bash /sandbox/tools/gsutil collect \ + >/tmp/openshell-nightly-failures.tsv \ + 2>/tmp/openshell-nightly-failures.err || { echo "collection failed; see /tmp/openshell-nightly-failures.err" \ >>/tmp/openshell-nightly-failures.tsv } - fi + cp /tmp/openshell-nightly-failures.tsv /sandbox/nightly-failures.tsv 2>/dev/null || true + cp /tmp/openshell-nightly-failures.err /sandbox/nightly-failures.err 2>/dev/null || true + fi destination: /sandbox/.bash_env agent: @@ -65,9 +67,9 @@ outputs: - source: /sandbox/acs-triage-agent/artifacts/ci-triage.json destination: ci-triage.json required: false - - source: /tmp/openshell-nightly-failures.tsv + - source: /sandbox/nightly-failures.tsv destination: nightly-failures.tsv required: false - - source: /tmp/openshell-nightly-failures.err + - source: /sandbox/nightly-failures.err destination: nightly-failures.err required: false From 09d8405a603506abfb70d6b40486baad8da43a87 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:46:20 -0700 Subject: [PATCH 26/41] fix(acs-ci): select latest completed build --- tasks/acs-ci-nightly/workflow/gsutil | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 25f1088..1d3b9eb 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -106,21 +106,27 @@ collect_failures() { while IFS= read -r job; do [[ -z "$job" ]] && continue - ( - job_prefix="${job%/}/" - build=$(list_builds "$job" | sort -n | tail -n 1) - [[ -n "$build" ]] || exit 0 - gcs_path="${job_prefix}${build}" - finished=$(read_finished "$gcs_path") - timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') - result=$(printf '%s' "$finished" | jq -r '.result // ""') - if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then - printf '%s\t%s\t%s\t%s\n' \ - "${job#gs://${configured_bucket}/}" "$build" "$gcs_path" "$timestamp" + job_prefix="${job%/}/" + finished="" + build="" + while IFS= read -r candidate; do + [[ -z "$candidate" ]] && continue + candidate_path="${job_prefix}${candidate}" + if candidate_finished=$(read_finished "$candidate_path" 2>/dev/null); then + build="$candidate" + gcs_path="$candidate_path" + finished="$candidate_finished" + break fi - ) & + done < <(list_builds "$job" | sort -nr || true) + [[ -n "$build" && -n "$finished" ]] || continue + timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') + result=$(printf '%s' "$finished" | jq -r '.result // ""') + if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then + printf '%s\t%s\t%s\t%s\n' \ + "${job#gs://${configured_bucket}/}" "$build" "$gcs_path" "$timestamp" + fi done <<<"$jobs" - wait } command=${1:-} From 56a997d29dfd7dd10f22b4295872814c651150dc Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:52:31 -0700 Subject: [PATCH 27/41] fix(acs-ci): use anonymous access for public GCS manifest --- tasks/acs-ci-nightly/workflow/gsutil | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 1d3b9eb..dc62e51 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -20,7 +20,6 @@ rewrite_uri() { } list_nightly_jobs() { - local token=${GCP_SA_ACCESS_TOKEN:-} local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" local page_token="" @@ -42,9 +41,6 @@ list_nightly_jobs() { "${query_args[@]}" "$api" ) - if [[ -n "$token" ]]; then - curl_args+=(--header "Authorization: Bearer $token") - fi response=$(timeout 90s curl "${curl_args[@]}") printf '%s' "$response" \ @@ -58,7 +54,6 @@ list_nightly_jobs() { list_builds() { local job=$1 - local token=${GCP_SA_ACCESS_TOKEN:-} local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" local object_prefix="${job#gs://${configured_bucket}/}" local -a curl_args=( @@ -70,16 +65,12 @@ list_builds() { --data-urlencode "maxResults=1000" "$api" ) - if [[ -n "$token" ]]; then - curl_args+=(--header "Authorization: Bearer $token") - fi timeout 90s curl "${curl_args[@]}" \ | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' } read_finished() { local gcs_path=$1 - local token=${GCP_SA_ACCESS_TOKEN:-} local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" local encoded_object @@ -91,9 +82,6 @@ read_finished() { --data-urlencode "alt=media" "$api/$encoded_object" ) - if [[ -n "$token" ]]; then - curl_args+=(--header "Authorization: Bearer $token") - fi timeout 90s curl "${curl_args[@]}" } From c76497d8451eec657560e076a41d53461b9f618b Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 17:57:37 -0700 Subject: [PATCH 28/41] debug(acs-ci): record GCS list response shape --- tasks/acs-ci-nightly/workflow/gsutil | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index dc62e51..653ccd8 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -42,6 +42,8 @@ list_nightly_jobs() { "$api" ) response=$(timeout 90s curl "${curl_args[@]}") + printf 'GCS job-list response: %s\n' \ + "$(printf '%s' "$response" | jq -c '{kind, prefixes: (.prefixes | length), items: (.items | length), error}')" >&2 printf '%s' "$response" \ | jq -r --arg bucket "$configured_bucket" \ @@ -90,6 +92,8 @@ collect_failures() { local jobs cutoff=$(date -u -d '24 hours ago' +%s 2>/dev/null || date -u -v-1d +%s) jobs=$(list_nightly_jobs) + job_count=$(printf '%s\n' "$jobs" | awk 'NF { count++ } END { print count + 0 }') + printf 'GCS jobs discovered: %s\n' "$job_count" >&2 printf '# job\tbuild\tgcs_path\ttimestamp\n' while IFS= read -r job; do From b7abece531a304fe75396cf30208e127f09b8fc7 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:02:40 -0700 Subject: [PATCH 29/41] debug(acs-ci): inspect nested GCS listings --- tasks/acs-ci-nightly/workflow/gsutil | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 653ccd8..7d47c74 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -67,7 +67,11 @@ list_builds() { --data-urlencode "maxResults=1000" "$api" ) - timeout 90s curl "${curl_args[@]}" \ + local response + response=$(timeout 90s curl "${curl_args[@]}") + printf 'GCS build-list response: %s\n' \ + "$(printf '%s' "$response" | jq -c '{prefixes: (.prefixes | length), items: (.items | length), error}')" >&2 + printf '%s' "$response" \ | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' } @@ -111,7 +115,10 @@ collect_failures() { break fi done < <(list_builds "$job" | sort -nr || true) - [[ -n "$build" && -n "$finished" ]] || continue + if [[ -z "$build" || -z "$finished" ]]; then + printf 'GCS no completed build: %s\n' "$job" >&2 + continue + fi timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') result=$(printf '%s' "$finished" | jq -r '.result // ""') if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then From 778f1f8bd40dfc7a9953b2116764a1496a300eaf Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:07:14 -0700 Subject: [PATCH 30/41] fix(acs-ci): use GCS media download endpoint --- tasks/acs-ci-nightly/workflow/gsutil | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 7d47c74..a352a0c 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -77,7 +77,7 @@ list_builds() { read_finished() { local gcs_path=$1 - local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" + local api="https://storage.googleapis.com/download/storage/v1/b/${configured_bucket}/o" local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" local encoded_object encoded_object=$(jq -rn --arg object "$object_path" '$object | @uri') From 305bc288463efdc700edabae7244e0637c4b363a Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:12:21 -0700 Subject: [PATCH 31/41] fix(acs-ci): read GCS results from public object URLs --- tasks/acs-ci-nightly/workflow/gsutil | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index a352a0c..7bd52eb 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -77,16 +77,11 @@ list_builds() { read_finished() { local gcs_path=$1 - local api="https://storage.googleapis.com/download/storage/v1/b/${configured_bucket}/o" local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" - local encoded_object - encoded_object=$(jq -rn --arg object "$object_path" '$object | @uri') local -a curl_args=( --silent --show-error --fail --connect-timeout 10 --max-time 60 - --get - --data-urlencode "alt=media" - "$api/$encoded_object" + "https://storage.googleapis.com/${configured_bucket}/${object_path}" ) timeout 90s curl "${curl_args[@]}" } From 340b1287751fff68b1488f17a4fe4f4f9304b4b5 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:20:17 -0700 Subject: [PATCH 32/41] fix(acs-ci): read results with gcloud storage --- tasks/acs-ci-nightly/workflow/gsutil | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 7bd52eb..caa1c92 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -77,13 +77,7 @@ list_builds() { read_finished() { local gcs_path=$1 - local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" - local -a curl_args=( - --silent --show-error --fail - --connect-timeout 10 --max-time 60 - "https://storage.googleapis.com/${configured_bucket}/${object_path}" - ) - timeout 90s curl "${curl_args[@]}" + timeout 90s "$gcloud_bin" storage cat "${gcs_path}/finished.json" } collect_failures() { From 0caa1c9567f5402e72ad7768b19ee2bd590ade48 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:27:47 -0700 Subject: [PATCH 33/41] debug(acs-ci): bound GCS result reads --- tasks/acs-ci-nightly/workflow/gsutil | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index caa1c92..b1fc1da 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -77,7 +77,20 @@ list_builds() { read_finished() { local gcs_path=$1 - timeout 90s "$gcloud_bin" storage cat "${gcs_path}/finished.json" + local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" + local response + response=$(timeout 30s curl \ + --silent --show-error \ + --connect-timeout 10 --max-time 15 \ + --write-out $'\n__HTTP_STATUS__%{http_code}\n' \ + "https://storage.googleapis.com/${configured_bucket}/${object_path}") || return 1 + local status="${response##*$'\n__HTTP_STATUS__'}" + local body="${response%$'\n__HTTP_STATUS__'*}" + if [[ "$status" != 200 ]]; then + printf 'GCS finished read status=%s: %s\n' "$status" "$gcs_path" >&2 + return 1 + fi + printf '%s' "$body" } collect_failures() { From bd166d713e039eb59cd2b587914358b42bafe78e Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 18:49:23 -0700 Subject: [PATCH 34/41] fix(acs-ci): mark manifest rows as failures --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 5 +++-- tasks/acs-ci-nightly/workflow/gsutil | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index 801fa4c..12a05f3 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -14,8 +14,9 @@ Run only the CI failure analysis portion of the ACS triage agent. Prow failure lookup. The trusted task provides a `gsutil` compatibility command backed by the configured bucket and `gcloud storage`; do not replace it with a recursive bucket or wildcard listing. Treat - `/tmp/openshell-nightly-failures.tsv` as authoritative and do not invent - job, build, or GCS path values. + `/tmp/openshell-nightly-failures.tsv` as authoritative; every data row is a + verified `FAILURE` and must be analyzed as such. Do not invent job, build, + or GCS path values. 5. Spawn the repository's CI analysis agents as instructed and wait for their results. 6. Write exactly `/sandbox/acs-triage-agent/artifacts/ci-triage.json` using diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index b1fc1da..2b561cc 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -100,7 +100,8 @@ collect_failures() { jobs=$(list_nightly_jobs) job_count=$(printf '%s\n' "$jobs" | awk 'NF { count++ } END { print count + 0 }') printf 'GCS jobs discovered: %s\n' "$job_count" >&2 - printf '# job\tbuild\tgcs_path\ttimestamp\n' + printf '# Every data row below is a verified CI failure.\n' + printf '# job\tbuild\tgcs_path\ttimestamp\tresult\n' while IFS= read -r job; do [[ -z "$job" ]] && continue @@ -124,7 +125,7 @@ collect_failures() { timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') result=$(printf '%s' "$finished" | jq -r '.result // ""') if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then - printf '%s\t%s\t%s\t%s\n' \ + printf '%s\t%s\t%s\t%s\tFAILURE\n' \ "${job#gs://${configured_bucket}/}" "$build" "$gcs_path" "$timestamp" fi done <<<"$jobs" From a900cf14e8e556ac81a6e13f178a7e6275f8b1cb Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 19:00:06 -0700 Subject: [PATCH 35/41] chore(acs-ci): remove temporary manifest diagnostics --- tasks/acs-ci-nightly/workflow/gsutil | 25 +++------------------ tasks/acs-ci-nightly/workflow/harness.yaml | 26 ++++++++-------------- 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 2b561cc..d35de5d 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -42,8 +42,6 @@ list_nightly_jobs() { "$api" ) response=$(timeout 90s curl "${curl_args[@]}") - printf 'GCS job-list response: %s\n' \ - "$(printf '%s' "$response" | jq -c '{kind, prefixes: (.prefixes | length), items: (.items | length), error}')" >&2 printf '%s' "$response" \ | jq -r --arg bucket "$configured_bucket" \ @@ -69,8 +67,6 @@ list_builds() { ) local response response=$(timeout 90s curl "${curl_args[@]}") - printf 'GCS build-list response: %s\n' \ - "$(printf '%s' "$response" | jq -c '{prefixes: (.prefixes | length), items: (.items | length), error}')" >&2 printf '%s' "$response" \ | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' } @@ -78,19 +74,9 @@ list_builds() { read_finished() { local gcs_path=$1 local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" - local response - response=$(timeout 30s curl \ - --silent --show-error \ + timeout 30s curl --silent --show-error --fail \ --connect-timeout 10 --max-time 15 \ - --write-out $'\n__HTTP_STATUS__%{http_code}\n' \ - "https://storage.googleapis.com/${configured_bucket}/${object_path}") || return 1 - local status="${response##*$'\n__HTTP_STATUS__'}" - local body="${response%$'\n__HTTP_STATUS__'*}" - if [[ "$status" != 200 ]]; then - printf 'GCS finished read status=%s: %s\n' "$status" "$gcs_path" >&2 - return 1 - fi - printf '%s' "$body" + "https://storage.googleapis.com/${configured_bucket}/${object_path}" } collect_failures() { @@ -98,8 +84,6 @@ collect_failures() { local jobs cutoff=$(date -u -d '24 hours ago' +%s 2>/dev/null || date -u -v-1d +%s) jobs=$(list_nightly_jobs) - job_count=$(printf '%s\n' "$jobs" | awk 'NF { count++ } END { print count + 0 }') - printf 'GCS jobs discovered: %s\n' "$job_count" >&2 printf '# Every data row below is a verified CI failure.\n' printf '# job\tbuild\tgcs_path\ttimestamp\tresult\n' @@ -118,10 +102,7 @@ collect_failures() { break fi done < <(list_builds "$job" | sort -nr || true) - if [[ -z "$build" || -z "$finished" ]]; then - printf 'GCS no completed build: %s\n' "$job" >&2 - continue - fi + [[ -n "$build" && -n "$finished" ]] || continue timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') result=$(printf '%s' "$finished" | jq -r '.result // ""') if [[ "$timestamp" -ge "$cutoff" && "$result" == FAILURE ]]; then diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index 0e44bc4..dd653f3 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -43,17 +43,15 @@ payloads: if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" fi - if ! printenv OPENSHELL_GCS_MANIFEST_STARTED >/dev/null 2>&1; then - export OPENSHELL_GCS_MANIFEST_STARTED=1 - /bin/bash /sandbox/tools/gsutil collect \ - >/tmp/openshell-nightly-failures.tsv \ - 2>/tmp/openshell-nightly-failures.err || { - echo "collection failed; see /tmp/openshell-nightly-failures.err" \ - >>/tmp/openshell-nightly-failures.tsv - } - cp /tmp/openshell-nightly-failures.tsv /sandbox/nightly-failures.tsv 2>/dev/null || true - cp /tmp/openshell-nightly-failures.err /sandbox/nightly-failures.err 2>/dev/null || true - fi + if ! printenv OPENSHELL_GCS_MANIFEST_STARTED >/dev/null 2>&1; then + export OPENSHELL_GCS_MANIFEST_STARTED=1 + /bin/bash /sandbox/tools/gsutil collect \ + >/tmp/openshell-nightly-failures.tsv \ + 2>/tmp/openshell-nightly-failures.err || { + echo "collection failed; see /tmp/openshell-nightly-failures.err" \ + >>/tmp/openshell-nightly-failures.tsv + } + fi destination: /sandbox/.bash_env agent: @@ -67,9 +65,3 @@ outputs: - source: /sandbox/acs-triage-agent/artifacts/ci-triage.json destination: ci-triage.json required: false - - source: /sandbox/nightly-failures.tsv - destination: nightly-failures.tsv - required: false - - source: /sandbox/nightly-failures.err - destination: nightly-failures.err - required: false From 738941d2db7ecef1b78e9ade90f5d929f48e5bd3 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 20:16:50 -0700 Subject: [PATCH 36/41] fix(acs-ci): read GCS artifacts through public API --- tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md | 6 +-- tasks/acs-ci-nightly/workflow/gsutil | 60 ++++++++++++--------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md index 12a05f3..85d535f 100644 --- a/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md +++ b/tasks/acs-ci-nightly/workflow/CI-NIGHTLY.md @@ -11,9 +11,9 @@ Run only the CI failure analysis portion of the ACS triage agent. `git clone` or `git fetch` for these public repositories; do not run `gh auth login` or push to them. 4. Follow the repository's existing `ci-coordinator.md` instructions for the - Prow failure lookup. The trusted task provides a `gsutil` compatibility - command backed by the configured bucket and `gcloud storage`; do not replace - it with a recursive bucket or wildcard listing. Treat + Prow failure lookup. The trusted task provides a bounded `gsutil` compatibility + command backed by the configured bucket; do not replace it with a recursive + bucket or wildcard listing. Treat `/tmp/openshell-nightly-failures.tsv` as authoritative; every data row is a verified `FAILURE` and must be analyzed as such. Do not invent job, build, or GCS path values. diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index d35de5d..336aabf 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -1,10 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -# Compatibility shim for the existing ACS triage instructions. Exact reads -# use the image's gcloud binary. The top-level logs listing is bounded because -# recursively listing the bucket is too large for this task. -gcloud_bin=/opt/google-cloud-sdk/bin/gcloud +# Compatibility shim for the existing ACS triage instructions. The public +# bucket is read through bounded GCS JSON/object requests because the image's +# gcloud object reads do not work reliably through the OpenShell proxy. configured_bucket="${GCS_BUCKET:-test-platform-results-public}" rewrite_uri() { @@ -68,17 +67,42 @@ list_builds() { local response response=$(timeout 90s curl "${curl_args[@]}") printf '%s' "$response" \ - | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' + | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' } -read_finished() { - local gcs_path=$1 - local object_path="${gcs_path#gs://${configured_bucket}/}/finished.json" +list_objects() { + local uri=$1 + local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" + local object_prefix="${uri#gs://${configured_bucket}/}" + local -a curl_args=( + --silent --show-error --fail + --connect-timeout 10 --max-time 60 + --get + --data-urlencode "prefix=$object_prefix" + --data-urlencode "delimiter=/" + --data-urlencode "maxResults=1000" + "$api" + ) + local response + response=$(timeout 90s curl "${curl_args[@]}") + printf '%s' "$response" \ + | jq -r --arg bucket "$configured_bucket" \ + '(.items[]?.name, .prefixes[]?) | "gs://" + $bucket + "/" + .' +} + +read_object() { + local uri=$1 + local object_path="${uri#gs://${configured_bucket}/}" timeout 30s curl --silent --show-error --fail \ --connect-timeout 10 --max-time 15 \ "https://storage.googleapis.com/${configured_bucket}/${object_path}" } +read_finished() { + local gcs_path=$1 + read_object "${gcs_path%/}/finished.json" +} + collect_failures() { local cutoff local jobs @@ -128,10 +152,7 @@ case "$command" in if [[ "$uri" == "gs://${configured_bucket}/logs/" ]]; then list_nightly_jobs else - if [[ -n "${GCP_SA_ACCESS_TOKEN:-}" && -z "${CLOUDSDK_AUTH_ACCESS_TOKEN:-}" ]]; then - export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" - fi - exec timeout 90s "$gcloud_bin" storage ls "$uri" + list_objects "$uri" fi ;; cat) @@ -142,15 +163,7 @@ case "$command" in uri=$(rewrite_uri "$1") if [[ "$uri" == gs://*/logs/*/latest-build.txt ]]; then job_prefix="${uri%/latest-build.txt}/" - build=$(timeout 90s "$gcloud_bin" storage ls "$job_prefix" \ - | awk -v prefix="$job_prefix" 'index($0, prefix) == 1 { - rest = substr($0, length(prefix) + 1) - if (rest ~ /^[0-9]+\/$/) { - sub(/\/$/, "", rest) - print rest - } - }' \ - | sort -n | tail -n 1) + build=$(list_builds "$job_prefix" | sort -n | tail -n 1) if [[ -z "$build" ]]; then echo "gsutil compatibility wrapper: no numeric build for $job_prefix" >&2 exit 1 @@ -158,10 +171,7 @@ case "$command" in printf '%s\n' "$build" exit 0 fi - if [[ -n "${GCP_SA_ACCESS_TOKEN:-}" && -z "${CLOUDSDK_AUTH_ACCESS_TOKEN:-}" ]]; then - export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" - fi - exec timeout 90s "$gcloud_bin" storage cat "$uri" + read_object "$uri" ;; *) echo "gsutil compatibility wrapper supports only ls and cat" >&2 From 6e06f9c86b45c2e89baeb90985c0b0b44d4fee40 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 20:47:10 -0700 Subject: [PATCH 37/41] fix(acs-ci): harden GCS compatibility reads --- tasks/acs-ci-nightly/workflow/gsutil | 65 +++++++++++++++----- tasks/acs-ci-nightly/workflow/harness.yaml | 5 +- tasks/acs-ci-nightly/workflow/preflight.yaml | 2 +- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 336aabf..487c600 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -12,9 +12,13 @@ rewrite_uri() { gs://test-platform-results|gs://test-platform-results/*) printf 'gs://%s%s\n' "$configured_bucket" "${uri#gs://test-platform-results}" ;; - *) + gs://$configured_bucket|gs://$configured_bucket/*) printf '%s\n' "$uri" ;; + *) + echo "gsutil compatibility wrapper: unsupported bucket in $uri" >&2 + return 2 + ;; esac } @@ -55,19 +59,33 @@ list_builds() { local job=$1 local api="https://storage.googleapis.com/storage/v1/b/${configured_bucket}/o" local object_prefix="${job#gs://${configured_bucket}/}" - local -a curl_args=( - --silent --show-error --fail - --connect-timeout 10 --max-time 60 - --get - --data-urlencode "prefix=$object_prefix" - --data-urlencode "delimiter=/" - --data-urlencode "maxResults=1000" - "$api" - ) - local response - response=$(timeout 90s curl "${curl_args[@]}") - printf '%s' "$response" \ - | jq -r '.prefixes[]? | capture("/(?[0-9]+)/$").build' + local page_token="" + + while :; do + local response + local -a query_args=( + --data-urlencode "prefix=$object_prefix" + --data-urlencode "delimiter=/" + --data-urlencode "maxResults=1000" + ) + if [[ -n "$page_token" ]]; then + query_args+=(--data-urlencode "pageToken=$page_token") + fi + + local -a curl_args=( + --silent --show-error --fail + --connect-timeout 10 --max-time 60 + --get + "${query_args[@]}" + "$api" + ) + response=$(timeout 90s curl "${curl_args[@]}") + printf '%s' "$response" \ + | jq -r '.prefixes[]? | rtrimstr("/") | split("/") | .[-1] | select(test("^[0-9]+$"))' + + page_token=$(printf '%s' "$response" | jq -r '.nextPageToken // empty') + [[ -z "$page_token" ]] && break + done } list_objects() { @@ -106,7 +124,18 @@ read_finished() { collect_failures() { local cutoff local jobs - cutoff=$(date -u -d '24 hours ago' +%s 2>/dev/null || date -u -v-1d +%s) + local job + local job_prefix + local finished + local build + local gcs_path + local timestamp + local result + local candidates + case "$(uname -s)" in + Darwin) cutoff=$(date -u -v-1d +%s) ;; + *) cutoff=$(date -u -d '24 hours ago' +%s) ;; + esac jobs=$(list_nightly_jobs) printf '# Every data row below is a verified CI failure.\n' printf '# job\tbuild\tgcs_path\ttimestamp\tresult\n' @@ -116,6 +145,10 @@ collect_failures() { job_prefix="${job%/}/" finished="" build="" + if ! candidates=$(list_builds "$job"); then + echo "gsutil compatibility wrapper: failed to list builds for $job" >&2 + return 1 + fi while IFS= read -r candidate; do [[ -z "$candidate" ]] && continue candidate_path="${job_prefix}${candidate}" @@ -125,7 +158,7 @@ collect_failures() { finished="$candidate_finished" break fi - done < <(list_builds "$job" | sort -nr || true) + done < <(printf '%s\n' "$candidates" | sort -nr) [[ -n "$build" && -n "$finished" ]] || continue timestamp=$(printf '%s' "$finished" | jq -r '.timestamp // 0') result=$(printf '%s' "$finished" | jq -r '.result // ""') diff --git a/tasks/acs-ci-nightly/workflow/harness.yaml b/tasks/acs-ci-nightly/workflow/harness.yaml index dd653f3..d95cdd8 100644 --- a/tasks/acs-ci-nightly/workflow/harness.yaml +++ b/tasks/acs-ci-nightly/workflow/harness.yaml @@ -37,6 +37,8 @@ payloads: - source: gsutil destination: /sandbox/tools/gsutil - content: | + # OpenShell injects the short-lived provider token; gcloud needs this + # runtime-only compatibility mapping. export PATH="/sandbox/tools:$PATH" gsutil() { /bin/bash /sandbox/tools/gsutil "$@"; } export -f gsutil @@ -48,8 +50,7 @@ payloads: /bin/bash /sandbox/tools/gsutil collect \ >/tmp/openshell-nightly-failures.tsv \ 2>/tmp/openshell-nightly-failures.err || { - echo "collection failed; see /tmp/openshell-nightly-failures.err" \ - >>/tmp/openshell-nightly-failures.tsv + echo "collection failed; see /tmp/openshell-nightly-failures.err" >&2 } fi destination: /sandbox/.bash_env diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 949f645..1933494 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -52,7 +52,7 @@ agent: echo echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" - if ! printenv GCP_SA_ACCESS_TOKEN >/dev/null; then + if [ -z "${GCP_SA_ACCESS_TOKEN:-}" ]; then echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" status=1 elif timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ From c9c7ee1e2f77a05c663a3573aad45c0587514849 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 20:50:16 -0700 Subject: [PATCH 38/41] fix(acs-ci): avoid workflow interpolation in preflight check --- tasks/acs-ci-nightly/workflow/preflight.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml index 1933494..2292e76 100644 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ b/tasks/acs-ci-nightly/workflow/preflight.yaml @@ -52,7 +52,8 @@ agent: echo echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" - if [ -z "${GCP_SA_ACCESS_TOKEN:-}" ]; then + gcp_token="$(printenv GCP_SA_ACCESS_TOKEN 2>/dev/null || true)" + if [ -z "$gcp_token" ]; then echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" status=1 elif timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ From 0e06c999dc0007f8f61bac9ccb96f8314767bcbd Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 20:55:28 -0700 Subject: [PATCH 39/41] fix(acs-ci): validate GCS wrapper paths --- tasks/acs-ci-nightly/workflow/gsutil | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index 487c600..e31c887 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -10,16 +10,22 @@ rewrite_uri() { local uri=$1 case "$uri" in gs://test-platform-results|gs://test-platform-results/*) - printf 'gs://%s%s\n' "$configured_bucket" "${uri#gs://test-platform-results}" - ;; - gs://$configured_bucket|gs://$configured_bucket/*) - printf '%s\n' "$uri" + uri="gs://${configured_bucket}${uri#gs://test-platform-results}" ;; + "gs://${configured_bucket}"|"gs://${configured_bucket}"/*) ;; *) echo "gsutil compatibility wrapper: unsupported bucket in $uri" >&2 return 2 ;; esac + + case "$uri" in + */../*|*/./*|*/..|*/.) + echo "gsutil compatibility wrapper: unsupported path in $uri" >&2 + return 2 + ;; + esac + printf '%s\n' "$uri" } list_nightly_jobs() { From e9f5abf0e21959418923cf5df33e6951013f0ff1 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 21:08:33 -0700 Subject: [PATCH 40/41] chore(acs-ci): remove preflight workflow --- tasks/acs-ci-nightly/README.md | 4 - tasks/acs-ci-nightly/workflow/preflight.yaml | 83 -------------------- 2 files changed, 87 deletions(-) delete mode 100644 tasks/acs-ci-nightly/workflow/preflight.yaml diff --git a/tasks/acs-ci-nightly/README.md b/tasks/acs-ci-nightly/README.md index bd60c8b..e6ec2e3 100644 --- a/tasks/acs-ci-nightly/README.md +++ b/tasks/acs-ci-nightly/README.md @@ -25,10 +25,6 @@ validated. - Output: `/sandbox/acs-triage-agent/artifacts/ci-triage.json`, downloaded to the caller's output directory. The output is optional so partial diagnostics can still be retained when analysis fails. -- Preflight: the trusted caller may run `workflow/preflight.yaml` in the same - workspace before the main task. It verifies read-only GCS bucket-listing and - Jira access from inside the sandbox without exposing response bodies or - credentials. - Cleanup: the sandbox and host-side source staging are removed after outputs are downloaded. Downloaded artifacts and any external reads remain with the caller. diff --git a/tasks/acs-ci-nightly/workflow/preflight.yaml b/tasks/acs-ci-nightly/workflow/preflight.yaml deleted file mode 100644 index 2292e76..0000000 --- a/tasks/acs-ci-nightly/workflow/preflight.yaml +++ /dev/null @@ -1,83 +0,0 @@ -version: 1 -name: acs-preflight - -sandbox: - image: ${ACS_TRIAGE_IMAGE} - policy: - file: ../openshell/policy.yaml - providers: - - atlassian-triage-read - - prow-gcs-read - env: - JIRA_URL: https://redhat.atlassian.net - GCS_BUCKET: test-platform-results-public - BASH_ENV: /sandbox/.bash_env - CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE: /etc/openshell-tls/ca-bundle.pem - CURL_CA_BUNDLE: /etc/openshell-tls/ca-bundle.pem - -agent: - type: /bin/bash - args: - - -c - - | - set -u - mkdir -p /sandbox/artifacts - log=/sandbox/artifacts/preflight.log - gcs_error=/tmp/preflight-gcs-error - jira_error=/tmp/preflight-jira-error - status=0 - - { - echo "OpenShell ACS preflight" - echo "GCS bucket: $GCS_BUCKET" - echo "Jira URL: $JIRA_URL" - echo - - echo "Jira: GET $JIRA_URL/rest/api/2/myself" - jira_status=$(curl --silent --show-error --output /dev/null \ - --write-out '%{http_code}' --connect-timeout 10 --max-time 30 \ - --user "$JIRA_EMAIL:$JIRA_API_TOKEN" \ - "$JIRA_URL/rest/api/2/myself" 2>"$jira_error") - jira_exit=$? - if [ "$jira_exit" -ne 0 ]; then - echo "Jira: FAIL (curl exit $jira_exit)" - sed -n '1,20p' "$jira_error" - status=1 - elif [ "$jira_status" = 200 ]; then - echo "Jira: PASS (HTTP 200)" - else - echo "Jira: FAIL (HTTP $jira_status)" - status=1 - fi - echo - - echo "GCS: /opt/google-cloud-sdk/bin/gcloud storage ls gs://$GCS_BUCKET" - gcp_token="$(printenv GCP_SA_ACCESS_TOKEN 2>/dev/null || true)" - if [ -z "$gcp_token" ]; then - echo "GCS: FAIL (OpenShell provider token GCP_SA_ACCESS_TOKEN is missing)" - status=1 - elif timeout 30s /opt/google-cloud-sdk/bin/gcloud storage ls "gs://$GCS_BUCKET" \ - >/dev/null 2>"$gcs_error"; then - echo "GCS: PASS" - else - gcs_status=$? - echo "GCS: FAIL (exit $gcs_status)" - sed -n '1,20p' "$gcs_error" - status=1 - fi - - } >"$log" 2>&1 - - cat "$log" - exit "$status" - -payloads: - - content: | - if [ -n "$GCP_SA_ACCESS_TOKEN" ]; then - export CLOUDSDK_AUTH_ACCESS_TOKEN="$GCP_SA_ACCESS_TOKEN" - fi - destination: /sandbox/.bash_env - -outputs: - - source: /sandbox/artifacts/preflight.log - destination: preflight.log From 996837b39d9d0fa47120071d19abf8cb24968eca Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 17 Sep 2026 21:12:52 -0700 Subject: [PATCH 41/41] fix(acs-ci): harden wrapper path validation --- tasks/acs-ci-nightly/workflow/gsutil | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tasks/acs-ci-nightly/workflow/gsutil b/tasks/acs-ci-nightly/workflow/gsutil index e31c887..932b05e 100755 --- a/tasks/acs-ci-nightly/workflow/gsutil +++ b/tasks/acs-ci-nightly/workflow/gsutil @@ -8,6 +8,8 @@ configured_bucket="${GCS_BUCKET:-test-platform-results-public}" rewrite_uri() { local uri=$1 + local lower_uri + lower_uri=$(printf '%s' "$uri" | tr '[:upper:]' '[:lower:]') case "$uri" in gs://test-platform-results|gs://test-platform-results/*) uri="gs://${configured_bucket}${uri#gs://test-platform-results}" @@ -19,8 +21,8 @@ rewrite_uri() { ;; esac - case "$uri" in - */../*|*/./*|*/..|*/.) + case "$lower_uri" in + */../*|*/./*|*/..|*/.|*%2e*) echo "gsutil compatibility wrapper: unsupported path in $uri" >&2 return 2 ;; @@ -138,10 +140,7 @@ collect_failures() { local timestamp local result local candidates - case "$(uname -s)" in - Darwin) cutoff=$(date -u -v-1d +%s) ;; - *) cutoff=$(date -u -d '24 hours ago' +%s) ;; - esac + cutoff=$(date -u -d '24 hours ago' +%s) jobs=$(list_nightly_jobs) printf '# Every data row below is a verified CI failure.\n' printf '# job\tbuild\tgcs_path\ttimestamp\tresult\n'