From 5dbf90310acc0c6562a6ce346be22bae2abe8145 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 09:04:49 +0900 Subject: [PATCH 1/7] fix: separate Strix baseline provider exhaustion --- .github/workflows/strix.yml | 9 +++ CHANGELOG.md | 5 ++ .../strix-pr-baseline-provider-exhaustion.md | 29 +++++++++ scripts/ci/strix_quick_gate.sh | 12 ++++ scripts/ci/test_strix_quick_gate.sh | 65 +++++++++++++++++++ ...est_strix_nvidia_nim_not_found_fallback.py | 10 +++ 6 files changed, 130 insertions(+) create mode 100644 docs/doctoring/strix-pr-baseline-provider-exhaustion.md diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4155c7346..8ec9c51c2 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -858,6 +858,15 @@ jobs: exit 0 fi + # Exit 3 is emitted only by the trusted gate after its exact PR-scope + # mapper excludes every reported finding as unchanged and all model + # providers are unavailable. Vulnerability text remains in the audit + # log, so this cannot use the generic no-finding regex path below. + if [ "$strix_rc" -eq 3 ]; then + echo "::warning title=Strix backend unavailable after baseline scan::Strix excluded all reported findings from the pull-request diff, then exhausted its model providers. Treating this exact-head run as a neutral infrastructure skip; changed or unmapped findings still fail closed." + exit 0 + fi + # Preserve configuration failures (exit 2) and any unexpected exit # code as hard failures — only the scan-failure code (1) can be an # infrastructure/backend-unavailability outcome. diff --git a/CHANGELOG.md b/CHANGELOG.md index 89199da27..bba2745b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,11 @@ Semantic Versioning where the repository publishes a release. ### Changed +- Preserve the trusted PR-scope decision when Strix finds only unchanged-file + baseline issues and then exhausts every LLM provider, so provider retirement + or outage is a neutral infrastructure result while changed and unmapped + findings still fail closed; the reproduced incident and gate boundary are + recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and exact-name dispatch ledger, so one slow repository cannot hide ready sibling diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md new file mode 100644 index 000000000..1b08cf00e --- /dev/null +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -0,0 +1,29 @@ +# Strix PR baseline/provider-exhaustion incident + +## Observed failure + +LineageWeave PR 392 run `32530198775` reported a critical secret in the +nonexistent `frontend/src/config.ts`. The trusted changed-file mapper correctly +classified that report as unchanged, but later fallback attempts ended in +provider HTTP 410 retirement brownouts. The gate retained the earlier severity +rank and returned the same exit code used for changed-file findings, so the +outer workflow could not distinguish the cleared baseline report from a real +pull-request vulnerability. + +## Root cause and repair + +The quick gate already owns the exact PR-head changed-file mapping decision. +After it has classified every report as `allow_baseline`, a later provider +exhaustion now returns the dedicated status 3. The trusted reusable workflow +maps only that status to a neutral infrastructure warning. Changed, unmapped, +or manifest findings still return the blocking status, and configuration or +unexpected statuses still fail closed. + +## Verification + +- A three-attempt regression reproduces an unchanged critical report followed + by two provider failures and requires status 3. +- Existing source tests require changed findings to remain blocking and clean + unchanged findings to remain admissible. +- The central Python suite, native workflow validation, Bash syntax checks, and + the complete Strix shell regression suite run on the final tree. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..208f6756e 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -4040,6 +4040,18 @@ run_current_target_scan() { return 1 fi + # The trusted PR-scope mapper has already proved these reports belong only + # to unchanged files. Keep that decision distinct from a real changed-file + # finding when every remaining provider is unavailable. + if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + fi + return 3 + fi + local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 1c05feb6f..831cb6094 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4650,6 +4650,28 @@ EOS echo "Penetration test failed: baseline critical finding" exit 1 ;; + pr-baseline-provider-exhausted) + case "${STRIX_LLM:-}" in + vertex_ai/baseline-primary) + mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities/vuln-0001.md" <<'EOS' +Severity: CRITICAL +Location 1: +sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/service/impl/SysUserServiceImpl.java:5 +EOS + echo "litellm.APIConnectionError: GeminiException LLM CONNECTION FAILED after baseline report" + exit 1 + ;; + vertex_ai/fallback-one | vertex_ai/fallback-two) + echo "litellm.APIConnectionError: GeminiException LLM CONNECTION FAILED: provider unavailable" + exit 1 + ;; + *) + echo "Error: pr-baseline-provider-exhausted unexpected model (${STRIX_LLM:-})" >&2 + exit 35 + ;; + esac + ;; pr-critical-changed) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-changed/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-changed/vulnerabilities/vuln-0001.md" <<'EOS' @@ -6276,6 +6298,28 @@ run_filtered_gate_case_if_requested() { "Materialized PR-head changed-file scope" \ "repository_dispatch" ;; + pr-baseline-provider-exhausted) + run_gate_case "pr-baseline-provider-exhausted" \ + "vertex_ai/baseline-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "3" \ + "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." \ + "3" \ + "vertex_ai/baseline-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ + "||" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -10909,6 +10953,27 @@ run_gate_case "pr-baseline-critical-unchanged" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "pr-baseline-provider-exhausted" \ + "vertex_ai/baseline-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "3" \ + "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." \ + "3" \ + "vertex_ai/baseline-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \ + "||" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + run_gate_case "pr-baseline-critical-absolute-target" \ "openai/gpt-4o-mini" \ "" \ diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..4c067c822 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -256,6 +256,16 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: workflow, ) + def test_workflow_accepts_trusted_unchanged_finding_provider_exhaustion(self) -> None: + """Keep gate-mapped baseline findings separate from provider exhaustion.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn('if [ "$strix_rc" -eq 3 ]; then', workflow) + self.assertIn( + "Strix excluded all reported findings from the pull-request diff", + workflow, + ) + if __name__ == "__main__": unittest.main() From d065e69fa48fbf96251433f8be77934ccf03b159 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 09:37:59 +0900 Subject: [PATCH 2/7] fix(strix): neutralize empty Caido bootstrap outages --- .github/workflows/strix.yml | 6 +++--- CHANGELOG.md | 8 +++++--- .../strix-pr-baseline-provider-exhaustion.md | 12 ++++++++++++ tests/test_strix_nvidia_nim_not_found_fallback.py | 10 ++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8ec9c51c2..be9ced67c 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -874,8 +874,8 @@ jobs: exit "$strix_rc" fi - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + # Recognized signals that the scanner or LLM backend was unavailable. + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|RuntimeError:[[:space:]]*loginAsGuest failed after [0-9]+ attempts:[^[:cntrl:]]*curl exit 7:[^[:cntrl:]]*Failed to connect to 127\.0\.0\.1 port 48080' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines @@ -888,7 +888,7 @@ jobs: # from blocking current-head merge progress. if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." + echo "::warning title=Strix infrastructure unavailable::Strix could not complete because its scanner bootstrap or LLM backend was unavailable before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." exit 0 fi diff --git a/CHANGELOG.md b/CHANGELOG.md index bba2745b0..b291cb988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,11 @@ Semantic Versioning where the repository publishes a release. - Preserve the trusted PR-scope decision when Strix finds only unchanged-file baseline issues and then exhausts every LLM provider, so provider retirement - or outage is a neutral infrastructure result while changed and unmapped - findings still fail closed; the reproduced incident and gate boundary are - recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. + or outage is a neutral infrastructure result; recognize the exact local Caido + bootstrap connection failure the same way only when no vulnerability exists, + while changed and unmapped findings and all other runtime failures still fail + closed. The reproduced incidents and gate boundary are recorded in + `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and exact-name dispatch ledger, so one slow repository cannot hide ready sibling diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 1b08cf00e..3add57692 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -10,6 +10,11 @@ rank and returned the same exit code used for changed-file findings, so the outer workflow could not distinguish the cleared baseline report from a real pull-request vulnerability. +A later central PR run reported zero vulnerabilities and then failed before +scanning when Strix's local Caido process did not accept connections on +`127.0.0.1:48080`; the outer workflow did not yet recognize that exact scanner +bootstrap outage as infrastructure. + ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. @@ -19,11 +24,18 @@ maps only that status to a neutral infrastructure warning. Changed, unmapped, or manifest findings still return the blocking status, and configuration or unexpected statuses still fail closed. +The outer workflow also recognizes only the observed `loginAsGuest` retry +exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutral +only when no positive vulnerability or severity signal exists; every other +runtime failure remains blocking. + ## Verification - A three-attempt regression reproduces an unchanged critical report followed by two provider failures and requires status 3. - Existing source tests require changed findings to remain blocking and clean unchanged findings to remain admissible. +- A workflow regression requires the exact Caido bootstrap outage to be neutral + with zero findings and blocking when any vulnerability is reported. - The central Python suite, native workflow validation, Bash syntax checks, and the complete Strix shell regression suite run on the final tree. diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 4c067c822..b1c8275f9 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -243,6 +243,16 @@ def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None ) ) + def test_outer_workflow_neutralizes_only_empty_caido_bootstrap_failure(self) -> None: + """Treat the exact scanner-local Caido outage as infrastructure only.""" + + caido_error = ( + "RuntimeError: loginAsGuest failed after 10 attempts: curl exit 7: " + "curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + ) + self.assertTrue(_workflow_neutralizes(caido_error + "Vulnerabilities 0\n")) + self.assertFalse(_workflow_neutralizes(caido_error + "Vulnerabilities 1\n")) + def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: """Retain the static fail-closed vulnerability evidence contract.""" From 7179abb91f0069c49e7efbf5f158f1e0cc4231af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 15:55:00 +0900 Subject: [PATCH 3/7] fix(strix): preserve primary-only baseline result --- CHANGELOG.md | 9 ++-- .../strix-pr-baseline-provider-exhaustion.md | 13 +++--- scripts/ci/strix_quick_gate.sh | 32 +++++++------ scripts/ci/test_strix_quick_gate.sh | 45 ++++++++++++++++++- 4 files changed, 76 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f89ef8cb..b9219dc4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,11 @@ Semantic Versioning where the repository publishes a release. - Preserve the trusted PR-scope decision when Strix finds only unchanged-file baseline issues and then exhausts every LLM provider, so provider retirement - or outage is a neutral infrastructure result; recognize the exact local Caido - bootstrap connection failure the same way only when no vulnerability exists, - while changed and unmapped findings and all other runtime failures still fail - closed. The reproduced incidents and gate boundary are recorded in + or outage is a neutral infrastructure result even when no distinct fallback + is configured; recognize the exact local Caido bootstrap connection failure + the same way only when no vulnerability exists, while changed and unmapped + findings and all other runtime failures still fail closed. The reproduced + incidents and gate boundary are recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 3add57692..010af2cad 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -18,11 +18,12 @@ bootstrap outage as infrastructure. ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. -After it has classified every report as `allow_baseline`, a later provider -exhaustion now returns the dedicated status 3. The trusted reusable workflow -maps only that status to a neutral infrastructure warning. Changed, unmapped, -or manifest findings still return the blocking status, and configuration or -unexpected statuses still fail closed. +After it has classified every report as `allow_baseline`, provider exhaustion +now returns the dedicated status 3, including a deployment with no distinct +fallback configured. The trusted reusable workflow maps only that status to a +neutral infrastructure warning. Changed, unmapped, or manifest findings still +return the blocking status, and configuration or unexpected statuses still +fail closed. The outer workflow also recognizes only the observed `loginAsGuest` retry exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutral @@ -33,6 +34,8 @@ runtime failure remains blocking. - A three-attempt regression reproduces an unchanged critical report followed by two provider failures and requires status 3. +- A primary-only regression requires the same trusted baseline outcome without + treating the absent fallback as a pull-request finding. - Existing source tests require changed findings to remain blocking and clean unchanged findings to remain admissible. - A workflow regression requires the exact Caido bootstrap outage to be neutral diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 208f6756e..38f04de07 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3894,7 +3894,7 @@ run_current_target_scan() { local strict_primary_provider_fallback=0 if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - if is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then + if is_model_retryable_error "$PRIMARY_MODEL"; then strict_primary_provider_fallback=1 else echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 @@ -4024,6 +4024,24 @@ run_current_target_scan() { return 1 fi + # The trusted PR-scope mapper has already proved these reports belong only + # to unchanged files. Keep that decision distinct from a real changed-file + # finding when every configured model path is unavailable. + if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + if [ "$fallback_tried" -eq 0 ]; then + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model was unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model was unavailable after unchanged-file findings were excluded." >&2 + fi + elif is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + else + echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 + fi + return 3 + fi + if [ "$fallback_tried" -eq 0 ]; then local fallback_config_name fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" @@ -4040,18 +4058,6 @@ run_current_target_scan() { return 1 fi - # The trusted PR-scope mapper has already proved these reports belong only - # to unchanged files. Keep that decision distinct from a real changed-file - # finding when every remaining provider is unavailable. - if [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Configured Vertex model and fallback models were unavailable after unchanged-file findings were excluded." >&2 - else - echo "Configured model and fallback models were unavailable after unchanged-file findings were excluded." >&2 - fi - return 3 - fi - local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 831cb6094..432dcee71 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4650,7 +4650,7 @@ EOS echo "Penetration test failed: baseline critical finding" exit 1 ;; - pr-baseline-provider-exhausted) + pr-baseline-provider-exhausted | pr-baseline-provider-exhausted-no-fallback) case "${STRIX_LLM:-}" in vertex_ai/baseline-primary) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider/vulnerabilities" @@ -6320,6 +6320,28 @@ run_filtered_gate_case_if_requested() { "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" ;; + pr-baseline-provider-exhausted-no-fallback) + run_gate_case "pr-baseline-provider-exhausted-no-fallback" \ + "vertex_ai/baseline-primary" \ + " " \ + "3" \ + "Configured Vertex model was unavailable after unchanged-file findings were excluded." \ + "1" \ + "vertex_ai/baseline-primary" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -10974,6 +10996,27 @@ run_gate_case "pr-baseline-provider-exhausted" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "pr-baseline-provider-exhausted-no-fallback" \ + "vertex_ai/baseline-primary" \ + " " \ + "3" \ + "Configured Vertex model was unavailable after unchanged-file findings were excluded." \ + "1" \ + "vertex_ai/baseline-primary" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + run_gate_case "pr-baseline-critical-absolute-target" \ "openai/gpt-4o-mini" \ "" \ From 94eafa4ca107f7e6b57e013e312223d4944faefc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 16:03:48 +0900 Subject: [PATCH 4/7] refactor(strix): remove obsolete fallback probe --- scripts/ci/strix_quick_gate.sh | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 38f04de07..996328b9f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2212,28 +2212,6 @@ fallback_models_config_name_for_model() { printf '%s\n' "STRIX_FALLBACK_MODELS" } -has_distinct_fallback_model_for_model() { - local model="$1" - local fallback_models_raw - fallback_models_raw="$(fallback_models_raw_for_model "$model")" - fallback_models_raw="${fallback_models_raw//$'\r'/ }" - fallback_models_raw="${fallback_models_raw//$'\n'/ }" - - local fallback_models=() - read -r -a fallback_models <<<"$fallback_models_raw" - - local candidate_raw - local candidate - for candidate_raw in "${fallback_models[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - if [ -n "$candidate" ] && [ "$candidate" != "$model" ]; then - return 0 - fi - done - - return 1 -} - resolved_llm_api_base_for_model() { local model="$1" From 51a4aa7d2da0e8cf4e940e0ef1ea3f5d8281bba6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 16:40:18 +0900 Subject: [PATCH 5/7] fix(strix): fallback on unsupported sampling defaults --- CHANGELOG.md | 6 +- .../strix-pr-baseline-provider-exhaustion.md | 36 +++++ scripts/ci/strix_quick_gate.sh | 21 +++ scripts/ci/test_strix_quick_gate.sh | 148 +++++++++++++++++- 4 files changed, 208 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9219dc4b..eec4c525e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,11 @@ Semantic Versioning where the repository publishes a release. baseline issues and then exhausts every LLM provider, so provider retirement or outage is a neutral infrastructure result even when no distinct fallback is configured; recognize the exact local Caido bootstrap connection failure - the same way only when no vulnerability exists, while changed and unmapped + the same way only when no vulnerability exists; route the exact single-line + LiteLLM/Azure unsupported-temperature failure to the existing distinct outer + fallback without accepting split-line imitations, while changed and unmapped findings and all other runtime failures still fail closed. The reproduced - incidents and gate boundary are recorded in + incidents, APA 7 sources, and gate boundary are recorded in `docs/doctoring/strix-pr-baseline-provider-exhaustion.md`. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md index 010af2cad..aa3d0cc2c 100644 --- a/docs/doctoring/strix-pr-baseline-provider-exhaustion.md +++ b/docs/doctoring/strix-pr-baseline-provider-exhaustion.md @@ -15,6 +15,14 @@ scanning when Strix's local Caido process did not accept connections on `127.0.0.1:48080`; the outer workflow did not yet recognize that exact scanner bootstrap outage as infrastructure. +Another observed run selected Azure `gpt-5.6-sol`, where LiteLLM forwarded +Strix's `temperature=0.2`. Azure rejected the unsupported sampling parameter +and LiteLLM had no fallback group for that model. Microsoft documents +`temperature` as unsupported for reasoning models (Microsoft, 2026), while the +documented Strix configuration surface has no generation-parameter control and +the missing capability remains an upstream request (AkikoOrenji, 2026; +usestrix, n.d.). + ## Root cause and repair The quick gate already owns the exact PR-head changed-file mapping decision. @@ -30,6 +38,16 @@ exhaustion with curl exit 7 against Strix's fixed local Caido port. It is neutra only when no positive vulnerability or severity signal exists; every other runtime failure remains blocking. +The preferred request-boundary repair is to omit a sampling parameter a caller +did not explicitly provide. The pinned Strix integration cannot currently do +that through its documented configuration, so the quick gate recognizes only a +single log line containing the complete LiteLLM/Azure unsupported-temperature +failure and the missing internal model group. It skips deterministic same-model +retry and moves to the existing distinct outer fallback. Split-line signal +assembly stays non-retryable, and provider exhaustion becomes neutral only +after the existing trusted PR-scope mapper has classified every report as +`allow_baseline`. + ## Verification - A three-attempt regression reproduces an unchanged critical report followed @@ -40,5 +58,23 @@ runtime failure remains blocking. unchanged findings to remain admissible. - A workflow regression requires the exact Caido bootstrap outage to be neutral with zero findings and blocking when any vulnerability is reported. +- An Azure capability regression requires the exact unsupported-temperature + line to reach the configured GitHub Models fallback without a same-model + retry; a split-line imitation must remain non-recoverable. - The central Python suite, native workflow validation, Bash syntax checks, and the complete Strix shell regression suite run on the final tree. + +## References + +AkikoOrenji. (2026, June 4). *[Feature] Expose LLM generation parameters to +control local/OpenAI-compatible model behaviour and prevent runaway tool-call +loops* (Issue No. 514) [GitHub issue]. GitHub. +https://github.com/usestrix/strix/issues/514 + +Microsoft. (2026, August 20). *Azure OpenAI reasoning models—GPT-5 series, +o3-mini, o1, o1-mini*. Microsoft Learn. +https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/reasoning + +usestrix. (n.d.). *Configuration* [Computer software documentation]. GitHub. +Retrieved August 22, 2026, from +https://github.com/usestrix/strix/blob/main/docs/advanced/configuration.mdx diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 996328b9f..666fe9ed0 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2638,6 +2638,19 @@ is_nvidia_nim_not_found_error() { return 1 } +is_unsupported_model_parameter_error() { + # Strix currently has no generation-parameter override. Match the exact + # single-line LiteLLM/Azure capability failure so a reasoning model that + # rejects Strix's temperature can move to the already-configured fallback. + if grep -Ei 'litellm(\.exceptions)?\.BadRequestError' "$STRIX_LOG" | + grep -Ei '(AzureException|OpenAIException)' | + grep -Eiq "Unsupported value:[[:space:]]*['\"]temperature['\"].*Only the default[[:space:]]*\\(1\\)[[:space:]]*value is supported.*No fallback model group found"; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Four error families qualify: @@ -2954,6 +2967,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3835,6 +3852,10 @@ is_model_retryable_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 432dcee71..56c613aad 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3386,6 +3386,27 @@ REPORT ;; esac ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + case "${STRIX_LLM:-}" in + openai/gpt-5.6-sol) + echo "litellm.BadRequestError: AzureException BadRequestError - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found for original model_group=gpt-5.6-sol." + exit 1 + ;; + openai/o3) + echo "scan ok with GitHub Models fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + echo "litellm.BadRequestError: request rejected" + echo "AzureException - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found." + exit 1 + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5519,7 +5540,8 @@ PY FAKE_STRIX_OUTSIDE_REPORT_DIR="$repo_root_dir/outside-strix-report" ) fi - if [ "$scenario" = "openai-direct-quota-github-models-fallback-success" ]; then + if [ "$scenario" = "openai-direct-quota-github-models-fallback-success" ] || + [ "$scenario" = "openai-direct-unsupported-temperature-github-models-fallback-success" ]; then printf '%s' 'https://models.github.ai/inference' >"$tmp_dir/github_models_api_base.txt" printf '%s' 'github-models-fallback-token' >"$tmp_dir/github_models_key.txt" env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") @@ -5921,6 +5943,66 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/openai/o3" ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -12107,6 +12189,70 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "" \ "github_models/openai/o3" +# Strix currently reaches LiteLLM with a sampling default and exposes no +# documented generation-parameter override. Azure reasoning deployments reject +# that temperature before Strix's internal model-group fallback can run, so the +# trusted outer gate must try its already-configured distinct provider. +run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + +# Cross-line assembly is deliberately rejected: target/source text cannot +# manufacture a provider capability signal from independent log lines. +run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ From a3b149bd5d924922d892d768b7832d9148e17bbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 18:38:40 +0900 Subject: [PATCH 6/7] fix(strix): align manifest finding decisions --- scripts/ci/strix_quick_gate.sh | 4 ++-- scripts/ci/test_strix_quick_gate.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 666fe9ed0..b5581688b 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3912,7 +3912,7 @@ run_current_target_scan() { fi case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) + block_changed | block_unmapped | block_manifest_finding) if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then return 1 fi @@ -3989,7 +3989,7 @@ run_current_target_scan() { fi case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) + block_changed | block_unmapped | block_manifest_finding) if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then return 1 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 56c613aad..dcdc54d88 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -316,6 +316,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$GATE_SCRIPT" "os.walk(root, topdown=True, followlinks=False)" "strix gate does not recurse into symlinked report directories" assert_file_not_contains "$GATE_SCRIPT" 'root.rglob("*.log")' "strix gate avoids recursive pathlib glob traversal for report logs" assert_file_contains "$GATE_SCRIPT" "has_strix_report_failure_signal" "strix gate fails closed on warning-class Strix report artifacts" + assert_file_not_contains "$GATE_SCRIPT" "block_manifest_unverified" "strix gate has no dead manifest-decision alias" + assert_file_contains "$GATE_SCRIPT" 'block_changed | block_unmapped | block_manifest_finding)' "strix gate handles the manifest finding decision in both fallback paths" assert_file_not_contains "$workflow_file" "ignore::UserWarning" "strix workflow must not blanket-suppress all UserWarning output" assert_file_contains "$GATE_SCRIPT" "vulnerability_file_reports_generic_github_actions_workflow_insecurity" "strix gate fact-checks generic GitHub Actions workflow security reports before accepting whole-file claims" assert_file_not_contains "$workflow_file" "vertex_ai/* | vertex_ai_beta/*" "strix workflow must not accept arbitrary Vertex models" From 60f5150f1dcc89c622f5b379561e622d491dd06e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 20:53:41 +0900 Subject: [PATCH 7/7] docs(strix): clarify provider exhaustion exits --- .github/workflows/strix.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index ddd01b508..130d99fe1 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -868,8 +868,9 @@ jobs: fi # Preserve configuration failures (exit 2) and any unexpected exit - # code as hard failures — only the scan-failure code (1) can be an - # infrastructure/backend-unavailability outcome. + # code as hard failures. Exit 1 uses the log-based infrastructure + # fallback; exit 3 is the trusted post-baseline provider-exhaustion + # outcome handled above. if [ "$strix_rc" -ne 1 ]; then exit "$strix_rc" fi