From f3aeb37d418076d0348d7b830e37d81a3ff8522c Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 01:26:50 +0000 Subject: [PATCH 1/8] Report streaming failures with the real status, and fail over locally A streaming request that fails before the first byte reaches the client used to answer HTTP 200 with a single in-band `error` event. `StreamingResponse` commits the response headers as soon as it is iterated, while the upstream call happens inside the generator, so an upstream 429, a connect/write timeout or a content-filter rejection had nowhere to go but the SSE body. Clients parse per chunk: an OpenAI SDK sees a chunk without `choices`, Codex never receives `response.completed`, and the turn silently ends as "the model answered with nothing" - no retry, no error, and the audit log records a success. Non-streaming requests of the same failure already returned the real status, so the two paths disagreed about what failed means. - Prefetch the first segment before committing to 200, so a failure that happened before any byte went out is surfaced with the real HTTP status on all three endpoints; a stream that breaks after bytes were already sent still reports in band, because its status can no longer be recalled. - Treat a write timeout like a connect failure in `open_backend_stream`: the request body was never accepted, so replaying it on a fresh connection cannot double-bill. Large cross-border sessions hit the 60 s write timeout far more often than they hit connection setup. - Add `--failover-max` (default 0, off): retry such a failure on another credential, bounded by N, so the client still sees one normal response. Only upstream HTTP rejections (401/403/429/502/503/504) and never-accepted request bodies are replayed; content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors never are. When no other credential can be picked, the first real status is surfaced instead of masking it with a 503. - `observe_recovery()` keeps the audit outcome honest after a saved replay: the failed attempt stays in `attempts` (plus a `failover_recovered` marker) without leaving an `error` + 200 row behind. `tests/test_refusal.py` and `tests/test_workbuddy_filter.py` pinned the old 200-plus-in-band-error contract on the streaming paths; they now assert that both stream modes report the same status. `test_runtime_endpoints.py` counted exactly three pooled `_route_chat` call sites; the failover paths add two more, and the invariant it protects (never select credentials on the event loop) is asserted as a minimum now. --- .env.example | 6 + app/observability.py | 16 ++ app/upstream_io.py | 19 +- converter.py | 252 +++++++++++++++++---- docs/advanced.md | 11 +- docs/advanced.zh-CN.md | 11 +- tests/test_refusal.py | 18 +- tests/test_runtime_endpoints.py | 4 +- tests/test_stream_failover.py | 318 +++++++++++++++++++++++++++ tests/test_stream_status_contract.py | 173 +++++++++++++++ tests/test_upstream_io.py | 42 ++++ tests/test_workbuddy_filter.py | 11 +- 12 files changed, 815 insertions(+), 66 deletions(-) create mode 100644 tests/test_stream_failover.py create mode 100644 tests/test_stream_status_contract.py diff --git a/.env.example b/.env.example index 7f27b20..519bba5 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,9 @@ CODEBUDDY2API_LOG_BODY_LIMIT=65536 # 国际 WorkBuddy 一次性体验积分领取；默认关闭，仅对符合上游资格的账号生效。 CODEBUDDY2API_AUTO_TRIAL=false + +# 换凭证重放次数：失败发生在向下游落第一个字节之前时，最多再换几个凭证就地重放。 +# 默认 0（关闭）：如实把上游 429/502 回给下游。只重放上游没收下请求体（建连失败、写请求体 +# 超时）或用 401/403/429/502/503/504 拒绝的失败；内容审核拒绝与上游已回 200 之后合成的 +# 502 一律不重放。详见 docs/advanced.zh-CN.md 的「换凭证重放」。 +CODEBUDDY2API_FAILOVER_MAX=0 diff --git a/app/observability.py b/app/observability.py index bdc6d0b..70eaac9 100644 --- a/app/observability.py +++ b/app/observability.py @@ -164,6 +164,22 @@ def observe_failure(code): observation.fail(code) +def observe_recovery(): + """标记「先前记录的失败已经被就地重放救回」：请求对下游是完整正常响应。 + + 失败尝试仍留在 `attempts` 里（另加一条 `failover_recovered` 标记），只是不再决定 outcome + —— 否则一次成功的换凭证重放会留下 `outcome=error` + `status_code=200` 这种自相矛盾的 + 审计记录，看板和排障都会把它读成失败。 + """ + observation = _current.get() + if observation is not None and observation.failed: + code = observation.record.get("error_code") or "upstream_error" + observation.failed = False + observation.record["error_code"] = None + if len(observation.attempts) < 32: + observation.attempts.append(safe_attempt({"stage": "failover_recovered", "code": code})) + + class _Parser: def __init__(self, observation): self.observation = observation diff --git a/app/upstream_io.py b/app/upstream_io.py index a4267de..c9912a5 100644 --- a/app/upstream_io.py +++ b/app/upstream_io.py @@ -18,6 +18,14 @@ def __init__(self, status, raw): super().__init__(f"upstream HTTP {status}") +class UpstreamHTTPError(UpstreamResponseError): + """上游**用 HTTP 状态码**给出的答复（429 / 401 / 503 …）。 + + 与聚合器从 200 响应体里合成的 502（空流、坏 SSE、已开流后断连）区分开：只有前者的请求 + 确定没被上游收下处理，换一个账号重放不会重复计费；后者上游已经回了 200，可能已经计费。 + """ + + class ChatSSEAccumulator: """聚合 Chat SSE，拒绝错误事件、空输出和无结束标记的残流。""" @@ -171,7 +179,14 @@ async def read_bounded_error(response, limit: int = ERROR_BODY_LIMIT) -> bytes: @asynccontextmanager async def open_backend_stream(url, headers, body, *, read_timeout=300, on_retry=None): - """只重试一次建连失败，其他错误交给调用方按协议返回。""" + """只重试一次「上游还没收下请求体」的失败（建连失败 / 写请求体超时），其余交给调用方按协议返回。 + + 写超时意味着请求体没有被完整接收，上游不会处理也不会计费，因此换一条全新连接重放是安全的： + 这里每次新建 `AsyncClient`，重试即重建 TCP+TLS，通常能换到另一个边缘节点。跨境链路上大 + 请求体（长会话）最容易撞到的正是 60s 写超时，而不是建连失败。 + + 响应已经开始之后（`opened` 置位）绝不重放 POST。 + """ timeout = httpx.Timeout(read_timeout, connect=15, write=60, pool=15) for attempt in range(2): opened = False @@ -181,7 +196,7 @@ async def open_backend_stream(url, headers, body, *, read_timeout=300, on_retry= opened = True yield response return - except (httpx.ConnectError, httpx.ConnectTimeout) as error: + except (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) as error: if opened or attempt == 1: raise if on_retry is not None: diff --git a/converter.py b/converter.py index a61becc..ce69889 100644 --- a/converter.py +++ b/converter.py @@ -68,11 +68,12 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app import trial_rewards from app import model_policy from app.model_blocks import ModelBlocks -from app.observability import (AuditMiddleware, observe_route, observe_usage, - observe_attempt, observe_failure) +from app.observability import (AuditMiddleware, observe_recovery, observe_route, + observe_usage, observe_attempt, observe_failure) from app.credential_io import (CredentialFileError, read_import_file, atomic_write_credential, credential_file_lock) -from app.upstream_io import ChatSSEAccumulator, UpstreamResponseError, open_backend_stream, read_bounded_error +from app.upstream_io import (ChatSSEAccumulator, UpstreamHTTPError, UpstreamResponseError, + open_backend_stream, read_bounded_error) from app.inference_auth import require_api_key from app.content_filter import ContentFilterDetector, is_filter_error from app.request_limits import ImageLimitError, apply_image_policy @@ -798,9 +799,15 @@ def _evict_sticky(self): else: break - def _candidates(self, model: str | None, *, region=None) -> list[dict]: - """可用凭证按（零计费优先, 快过期积分优先）排序；同级由调用方轮询。""" - healthy = [entry for entry in self._entries if self._healthy(entry) + def _candidates(self, model: str | None, *, region=None, tried=()) -> list[dict]: + """可用凭证按（零计费优先, 快过期积分优先）排序；同级由调用方轮询。 + + `tried` 是本轮已经打过的凭证管理器：换凭证重放时把它们排除在候选外，避免又选回 + 同一个刚失败的站点。 + """ + tried = set(tried) + healthy = [entry for entry in self._entries if entry["cm"] not in tried + and self._healthy(entry) and self._eligible(entry, model, region=region) and self._model_healthy(entry, model) and self._model_servable(entry, model)] if not healthy: @@ -809,7 +816,8 @@ def _candidates(self, model: str | None, *, region=None) -> list[dict]: healthy.sort(key=lambda entry: (not self._model_free(entry, model), *self._expiry_rank(entry))) return healthy - def pick(self, skey: str | None, model: str | None = None, *, region=None) -> CredentialManager | None: + def pick(self, skey: str | None, model: str | None = None, *, region=None, + tried=()) -> CredentialManager | None: """按黏绑选凭证；未绑定/已失效则轮询取健康凭证并绑定。 model 非空时跳过该模型 429 冷却中的凭证（黏性会话自动换绑）； @@ -819,7 +827,7 @@ def pick(self, skey: str | None, model: str | None = None, *, region=None) -> Cr self._rescan() # 锁外扫描，reload/prune 各自取锁，避免死锁 with self._lock: self._evict_sticky() - candidates = self._candidates(model, region=region) + candidates = self._candidates(model, region=region, tried=tried) if not candidates: if skey: self._sticky.pop(skey, None) @@ -841,10 +849,11 @@ def pick(self, skey: str | None, model: str | None = None, *, region=None) -> Cr self._sticky[skey] = (e["id"], time.time()) return e["cm"] - def headers_for(self, skey: str | None, model: str | None = None, *, region=None, with_generation=False): + def headers_for(self, skey: str | None, model: str | None = None, *, region=None, + with_generation=False, tried=()): """在发送前复核凭据代次和站点，避免重载竞态导致跨站调用。""" for _ in range(max(1, len(self._entries))): - cm = self.pick(skey, model, region=region) + cm = self.pick(skey, model, region=region, tried=tried) if cm is None: return None reason = None @@ -1417,6 +1426,7 @@ async def _protocol_http_exception(request: Request, exc: HTTPException): "max_request_bytes": 32 * 1024 * 1024, "log_body_limit": 65536, "max_inbound_bytes": 64 * 1024 * 1024, "max_collect_bytes": 8 * 1024 * 1024, "max_concurrent": 64, + "failover_max": 0, # 流式失败在第一个字节之前发生时可换凭证重放的最大次数 "usage_daily": None, # 官方用量聚合视图（日期×模型 credit），供 billing/usage 出 daily_costs "usage_daily_accounts": None, # 按账号的用量快照；单账号失败不丢历史 "credit_price_cny": None, "credit_price_usd": None, "usd_rate": None, @@ -1499,14 +1509,17 @@ def _check_admin_auth(authorization: Optional[str], x_api_key: Optional[str]): _check_auth(authorization, x_api_key) -def _cred_for(payload: dict, model: str | None = None, *, region=None): - """返回 ((凭据管理器, 代次), headers)；无可用凭据返回 503，模型冷却返回 429。""" +def _cred_for(payload: dict, model: str | None = None, *, region=None, tried=()): + """返回 ((凭据管理器, 代次), headers)；无可用凭据返回 503，模型冷却返回 429。 + + `tried` 里的凭证不再入选，供换凭证重放使用（见 `_routed_stream`）。 + """ raw_key = session_key(payload) skey = f"{region}:{raw_key}" if raw_key and region is not None else raw_key skey = model_policy.sticky_scope(CONFIG, skey, model) pool = CONFIG.get("cred_pool") if pool is not None: - picked = pool.headers_for(skey, model, region=region, with_generation=True) + picked = pool.headers_for(skey, model, region=region, with_generation=True, tried=tried) if picked is None: until = pool.model_cooldown_until(model, region=region) if until: @@ -1529,7 +1542,7 @@ def _cred_for(payload: dict, model: str | None = None, *, region=None): cm, headers = picked else: cm = CONFIG["cred"] - if cm is None: + if cm is None or cm in {_cred_manager(item) for item in tried}: raise HTTPException(status_code=503, detail={"error": {"message": "未找到登录凭据，请先在桌面端登录 CodeBuddy/WorkBuddy", "type": "auth_error"}}) with cm._lock: headers = cm.get_headers() @@ -1541,9 +1554,9 @@ def _cred_for(payload: dict, model: str | None = None, *, region=None): return cm, headers -def _route_chat(payload, body, rid): +def _route_chat(payload, body, rid, *, tried=()): """根据所选账号自动确定后端地域、产品及模型，不改变客户端地址。""" - cred, headers = _cred_for(payload, body.get("model")) + cred, headers = _cred_for(payload, body.get("model"), tried=tried) profile = profile_for_headers(headers) routed_model = _upstream_model(body.get("model"), profile) if routed_model != body.get("model"): @@ -2336,18 +2349,17 @@ async def chat_completions(request: Request, t0 = time.time() if client_wants_stream: - return StreamingResponse( - _stream_upstream(url, headers, body, model_name, t0, rid, cred=cred), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + def attempt(routed, cred, headers, url): + return _stream_upstream(url, headers, routed, model_name, t0, rid, cred=cred) + return await _routed_stream(payload, body, model_name, rid, t0, attempt, + body, cred, headers, url) # 非流式：后端只支持流式，这里把后端 SSE 聚合成单个 chat.completion 响应 - try: - collected = await _fetch_checked_chat(url, headers, body, model_name, rid, cred, filter_retry=True) - except (httpx.HTTPError, UpstreamResponseError) as error: - status, raw = _upstream_failure(error, model_name, t0, rid) - raise HTTPException(status_code=status, detail=_safe_err_raw(raw, status)) from None + async def fetch(routed, cred, headers, url): + return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, + filter_retry=True) + collected = await _routed_fetch(payload, body, model_name, rid, t0, fetch, + body, cred, headers, url) _log_finish(model_name, t0, collected, rid) if CONFIG.get("control_store") is not None: collected = {**collected, "model": model_name} @@ -2552,7 +2564,7 @@ def _check_upstream_status(status, raw, cred, model): if status != 200: if not is_filter_error(raw): _note_cred_status(cred, status, model=model, raw=raw) - raise UpstreamResponseError(status, raw) + raise UpstreamHTTPError(status, raw) def _upstream_failure(error, model_name, t0, rid): @@ -2675,10 +2687,14 @@ async def _chat_sse_lines(url, headers, body, model_name, t0, rid, cred=None, *, async def _stream_upstream(url: str, headers: dict, body: dict, model_name: str = "?", t0: float = 0.0, rid: str = "", cred=None): + sent = False try: async for line in _chat_sse_lines(url, headers, body, model_name, t0, rid, cred, aggregate=bool(body.get("tools"))): + sent = True yield (_public_sse_line(line, model_name) + "\n").encode("utf-8") except (httpx.HTTPError, UpstreamResponseError) as error: + if not sent: + raise # 一个字节都没发出去：交给端点还原成真实状态码，别把失败写成 200 status, raw = _upstream_failure(error, model_name, t0, rid) yield _err_event(raw, status) @@ -2691,6 +2707,142 @@ def _err_event(msg: bytes, status: int) -> bytes: return f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n".encode("utf-8") +def _cred_manager(cred): + """凭证统一是 (管理器, 代次)；兼容裸管理器（`_cred_for` 的单凭证回退分支）。""" + return cred[0] if isinstance(cred, tuple) else cred + + +# 可换凭证重放的上游 HTTP 状态：限流、认证、网关抖动。400/404/413 是确定性拒绝，换账号 +# 也一样，不在其中。 +FAILOVER_CODES = frozenset({401, 403, 429, 502, 503, 504}) +# 请求体确定没被上游收下的传输失败（建连失败 / 写请求体超时），重放不会重复计费。 +REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) + + +def _failover_safe(error, raw=b"") -> bool: + """这次失败能不能换账号重放：只认「上游没收下请求体」和「上游用 HTTP 状态码拒绝」。 + + 三条硬边界：内容审核拒绝不切号重放（那是模型的真实答复，换账号只会再撞一次同一堵墙， + 还白烧一次额度）；聚合器从 200 响应体里合成的 502（空流、坏 SSE、已开流后断连）不重放， + 因为上游可能已经处理并计费；真正的重放窗口由 `open_backend_stream` 的 `opened` 标记与 + `_preflight_stream` 守住。 + """ + if is_filter_error(raw): + return False + if isinstance(error, UpstreamHTTPError): + return error.status in FAILOVER_CODES + if isinstance(error, UpstreamResponseError): + return False + return isinstance(error, REPLAYABLE_TRANSPORT) + + +class _StreamFailure(Exception): + """流式预取阶段的失败：状态码、错误体，以及原始异常（重放判定要看它是什么类型）。""" + + def __init__(self, status, raw, error=None): + self.status = status + self.raw = raw + self.error = error + super().__init__(f"stream failed before first byte (HTTP {status})") + + +async def _preflight_stream(agen, model_name, t0, rid): + """取到第一段输出之后再决定怎么回 200。 + + `StreamingResponse` 一旦被迭代就把响应头发出去，而打上游发生在生成器里面 —— 于是上游的 + 429、建连/写超时乃至审核拒绝，在流式下全都只能塞进 SSE 正文：客户端看到的是一个没有 + `choices`、也等不到 `response.completed` 的 200 流，被读成「模型答了个空」，会话静默 + 结束，既不重试也不报错，审计里还记成一次成功。预取第一段之后，「一个字节都还没发出去」 + 的失败可以还原成真实状态码，流式与非流式同一口径；真的中途断流才继续用带内 error 事件 + （那时状态码已经收不回来了）。 + """ + try: + return await agen.__anext__() + except StopAsyncIteration: + empty = UpstreamResponseError(502, b'{"error":{"message":"upstream returned an empty stream",' + b'"type":"upstream_error","code":"empty_response"}}') + status, raw = _upstream_failure(empty, model_name, t0, rid) + raise _StreamFailure(status, raw, empty) from None + except (httpx.HTTPError, UpstreamResponseError) as error: + status, raw = _upstream_failure(error, model_name, t0, rid) + raise _StreamFailure(status, raw, error) from None + + +def _stream_response(agen, first): + """把预取到的第一段接回流，之后保持原有生成器语义。""" + async def body(): + yield first + async for chunk in agen: + yield chunk + return StreamingResponse(body(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + +def _failover_limit() -> int: + return int(CONFIG.get("failover_max") or 0) + + +async def _routed_stream(payload, body, model_name, rid, t0, make, routed, cred, headers, url): + """流式端点：预取失败时按策略换凭证重打，全部失败才把真实状态码回给下游。 + + 重放只发生在「一个字节都没发给下游」的时候（`_preflight_stream` 保证了这点），所以下游 + 看到的仍然是一次正常请求。`make(routed, cred, headers, url)` 每轮只建一个生成器。 + """ + tried = [] + while True: + stream = make(routed, cred, headers, url) + try: + first = await _preflight_stream(stream, model_name, t0, rid) + if tried: + observe_recovery() # 重放救回来的请求对下游是正常响应，不该记成失败 + return _stream_response(stream, first) + except _StreamFailure as failure: + tried.append(cred) + limit = _failover_limit() + surface = HTTPException(status_code=failure.status, + detail=_safe_err_raw(failure.raw, failure.status)) + if limit <= 0 or len(tried) > limit or not _failover_safe(failure.error, failure.raw): + raise surface from None + try: + attempt = await run_in_threadpool(_route_chat, payload, body, rid, + tried={_cred_manager(item) for item in tried}) + except HTTPException: + raise surface from None # 换不出别的凭证，就如实回第一次的错 + if _cred_manager(attempt[1]) in {_cred_manager(item) for item in tried}: + raise surface from None + routed, cred, headers, url = attempt + _log(f"[{rid}] ↻ 换凭证重放 {len(tried)}/{limit} | {model_name} | 上游 HTTP " + f"{failure.status} → {profile_for_headers(headers)}") + + +async def _routed_fetch(payload, body, model_name, rid, t0, fetch, routed, cred, headers, url): + """非流式请求：失败时按同一策略换凭证重打（此时一个字节都还没回给下游）。""" + tried = [] + while True: + try: + collected = await fetch(routed, cred, headers, url) + if tried: + observe_recovery() # 同上：换凭证后成功的请求不该记成失败 + return collected + except (httpx.HTTPError, UpstreamResponseError) as error: + status, raw = _upstream_failure(error, model_name, t0, rid) + tried.append(cred) + limit = _failover_limit() + surface = HTTPException(status_code=status, detail=_safe_err_raw(raw, status)) + if limit <= 0 or len(tried) > limit or not _failover_safe(error, raw): + raise surface from None + try: + attempt = await run_in_threadpool(_route_chat, payload, body, rid, + tried={_cred_manager(item) for item in tried}) + except HTTPException: + raise surface from None + if _cred_manager(attempt[1]) in {_cred_manager(item) for item in tried}: + raise surface from None + routed, cred, headers, url = attempt + _log(f"[{rid}] ↻ 换凭证重放 {len(tried)}/{limit} | {model_name} | 上游 HTTP " + f"{status} → {profile_for_headers(headers)}") + + def _note_content_filter(rid, model_name, *, final): stage = "content_filter" if final else "content_filter_retry" observe_attempt(stage, error_code="content_filter") @@ -2772,19 +2924,25 @@ async def create_response(request: Request, t0 = time.time() if client_wants_stream: - return StreamingResponse( - _stream_responses(url, headers, chat_body, model_name, t0, rid, cred=cred), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + def attempt(routed, cred, headers, url): + return _stream_responses(url, headers, routed, model_name, t0, rid, cred=cred) + return await _routed_stream(payload, chat_body, model_name, rid, t0, attempt, + chat_body, cred, headers, url) - return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred) + return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, + payload=payload) -async def _nonstream_adapted(url, headers, body, model_name, t0, rid, cred, *, anthropic=False): +async def _nonstream_adapted(url, headers, body, model_name, t0, rid, cred, *, anthropic=False, + payload=None): converter = (AnthropicStreamConverter(model=model_name) if anthropic else ResponsesStreamConverter(model=model_name, parallel_tool_calls=body.get("parallel_tool_calls", True))) + + async def fetch(routed, cred, headers, url): + return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, + filter_retry=True) try: - collected = await _fetch_checked_chat(url, headers, body, model_name, rid, cred, filter_retry=True) + collected = await _routed_fetch(payload, body, model_name, rid, t0, fetch, + body, cred, headers, url) for line in _chat_result_to_sse_lines(_completion_to_merged(collected)): converter.feed_line(_public_sse_line(line, model_name)) converter.finish() @@ -2799,17 +2957,22 @@ async def _nonstream_adapted(url, headers, body, model_name, t0, rid, cred, *, a async def _stream_adapted(url, headers, body, model_name, t0, rid, cred=None, *, anthropic=False): """协议适配只处理事件映射，连接、聚合与错误边界共用。""" converter = (AnthropicStreamConverter(model=model_name) if anthropic else ResponsesStreamConverter(model=model_name, parallel_tool_calls=body.get("parallel_tool_calls", True))) + sent = False try: async for line in _chat_sse_lines( url, headers, body, model_name, t0, rid, cred, aggregate=not anthropic or bool(body.get("tools"))): events = converter.feed_line(_public_sse_line(line, model_name)) if events: + sent = True yield events.encode("utf-8") events = converter.finish() if events: + sent = True yield events.encode("utf-8") except (httpx.HTTPError, UpstreamResponseError) as error: + if not sent: + raise # 一个字节都没发出去：交给端点还原成真实状态码，别把失败写成 200 status, raw = _upstream_failure(error, model_name, t0, rid) event = {"type": "error", "error": { "message": sanitize_log_text(raw.decode("utf-8", "replace"), 512), @@ -2867,13 +3030,13 @@ async def create_message(request: Request, t0 = time.time() if not _client_wants_stream(payload): - return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, anthropic=True) + return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, + anthropic=True, payload=payload) - return StreamingResponse( - _stream_anthropic(url, headers, chat_body, model_name, t0, rid, cred=cred), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) + def attempt(routed, cred, headers, url): + return _stream_anthropic(url, headers, routed, model_name, t0, rid, cred=cred) + return await _routed_stream(payload, chat_body, model_name, rid, t0, attempt, + chat_body, cred, headers, url) async def _stream_anthropic(url: str, headers: dict, body: dict, @@ -3083,6 +3246,10 @@ def main(): ap.add_argument("--tool-call-max-retry", type=_nonnegative_int, metavar="N", default=os.environ.get("CODEBUDDY2API_TOOL_CALL_MAX_RETRY", "3"), help="工具参数损坏时的额外生成上限，默认 3；0 表示不重试（每次额外生成都消耗额度）") + ap.add_argument("--failover-max", type=_nonnegative_int, metavar="N", + default=os.environ.get("CODEBUDDY2API_FAILOVER_MAX", "0"), + help="失败发生在向下游落第一个字节之前时，最多换几个凭证就地重放，默认 0（关闭）；" + "只重放上游没收下请求体或用 401/403/429/502/503/504 拒绝的失败") ap.add_argument("--auto-trial", type=_boolean_arg, nargs="?", const=True, default=os.environ.get("CODEBUDDY2API_AUTO_TRIAL", "false"), help="自动领取国际 WorkBuddy 一次性体验积分，默认关闭") @@ -3093,7 +3260,8 @@ def main(): return login(site=args.site, open_browser=not args.no_browser) for key in ("max_images", "image_policy", "max_request_bytes", "log_body_limit", "auto_trial", - "tool_call_max_retry", "max_inbound_bytes", "max_collect_bytes", "max_concurrent"): + "tool_call_max_retry", "max_inbound_bytes", "max_collect_bytes", "max_concurrent", + "failover_max"): CONFIG[key] = getattr(args, key) CONFIG["api_key"] = args.api_key CONFIG["desensitize"] = args.desensitize diff --git a/docs/advanced.md b/docs/advanced.md index 25d7480..47697b4 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -33,10 +33,11 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--max-inbound-bytes` | `67108864` | Raw body limit for generation and token-count POSTs, before parsing (chunked included); other routes are not buffered; 413 beyond it | | `--max-collect-bytes` | `8388608` | Total collection budget for aggregated output (content + reasoning + tool arguments); `response_too_large` beyond it; `0` disables | | `--max-concurrent` | `64` | Concurrency limit for the three generation endpoints only; excess requests get 503 with Retry-After; token counting is unaffected; `0` disables | +| `--failover-max` | `0` | Extra credentials tried when a request fails before the first response byte reaches the client; `0` keeps the upstream behaviour of surfacing the failure directly | | `--max-request-bytes` | `33554432` | Positive byte limit for the processed upstream JSON | | `--log-body-limit` | `65536` | Text-log body preview bytes; `0` logs summaries only, not the SQLite diagnostic budget | -Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT` and `CODEBUDDY2API_AUTO_TRIAL`. See [deployment](deployment.md) for startup examples. +Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_AUTO_TRIAL` and `CODEBUDDY2API_FAILOVER_MAX`. See [deployment](deployment.md) for startup examples. Trial-credit claims are off by default and only apply to upstream-eligible `intl-work` accounts. Successful/already-claimed results persist per account in `auth/trial-ledger.json`. Failures wait at least 24 hours without immediate POST replay; eligibility and amounts are determined upstream. Keep this file when upgrading. @@ -167,10 +168,12 @@ Credential domain / token issuer determine the product identity. Chat and refres | Cannot sign in to WebUI | Configure an API key; sign in and restart unfinished OAuth after changing it | | Local 401 | Client key differs from the gateway key | | Upstream 401 / 403 | Credential-level authentication circuit opens; inspect and log in again in the WebUI | -| 429 | Cool down that upstream model on the credential; later requests may rebind, but the current request is not replayed. All candidates cooling down still returns 429 | +| 429 | Cool down that upstream model on the credential; later requests rebind automatically. All candidates cooling down still returns 429; with `--failover-max` the in-flight request is replayed on another credential instead | | Upstream `service info not found` (code 11102) | That backend does not serve the model at all: avoid it for the `(backend, model)` pair, route the model to another backend, and return 404 when none has it. Half-open after 6 h, exponential backoff up to 24 h, cleared at once by one successful call; inspect via `GET /admin/model-blocks` | -| Connection setup failure | Retry only `ConnectError` / `ConnectTimeout` once after backoff | -| Post-send disconnect, read/write timeout or HTTP error | No network replay, avoiding duplicate billing; logs include exception type and elapsed time | +| Connection setup failure, write timeout | Retry once on a fresh connection: `ConnectError`, `ConnectTimeout` and `WriteTimeout` all mean the request body was never accepted, so replaying cannot double-bill | +| Post-send disconnect, read timeout or protocol error | No network replay, avoiding duplicate billing; logs include exception type and elapsed time | +| Streaming request fails before the first byte | Reported with the real HTTP status, exactly like `stream=false`. A 200 carrying only an in-band `error` event is read by clients as an empty answer, so the session ends silently while the audit log records a success | +| Credential failover (`--failover-max`) | Off by default. When enabled, a failure that happened before any byte reached the client is retried on another credential, up to N times, and is audited as `success` with a `failover_recovered` attempt marker. Only upstream HTTP rejections (401/403/429/502/503/504) and never-accepted request bodies qualify: content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors are never replayed, and if no other credential is available the original status is surfaced | | Malformed tool calls | Aggregate validation permits up to `--tool-call-max-retry` (default 3) additional generations, each consuming credits and recorded with its usage in the attempt details; exhaustion returns an error | | Empty or truncated upstream stream | No valid output, a missing end marker or an error is not reported as success | | Content-filter rejection | With desensitization and `--no-compact`, a complete non-streaming filter-only rejection may receive one shorter-template retry on the same account. No streaming filter retry, circuit opening or account rotation | diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index b67e91d..7a59b6c 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -33,10 +33,11 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--max-inbound-bytes` | `67108864` | 生成及 token 估算 POST 的解析前原始字节上限（含 chunked），超限 413；其他路由不缓冲请求体 | | `--max-collect-bytes` | `8388608` | 聚合路径输出收集总字节上限（正文+思考+工具参数），超限返回 `response_too_large`；`0` 不限制 | | `--max-concurrent` | `64` | 仅限制三个生成端点；占满立即 503（含 Retry-After），不限制 token 估算；`0` 不限制 | +| `--failover-max` | `0` | 请求在「一个字节都还没发给下游」之前失败时，最多再换几个凭证就地重放；`0` 表示如实把失败回给下游 | | `--max-request-bytes` | `33554432` | 处理后的上游 JSON 字节上限，须为正整数 | | `--log-body-limit` | `65536` | 兼容文本日志正文预览字节；`0` 只记摘要，不控制 SQLite 诊断预算 | -环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_AUTO_TRIAL`。启动示例见 [部署指南](deployment.zh-CN.md)。 +环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_AUTO_TRIAL`、`CODEBUDDY2API_FAILOVER_MAX`。启动示例见 [部署指南](deployment.zh-CN.md)。 体验积分领取默认关闭，仅适用于符合上游资格的 `intl-work` 账号。成功或已领取的结果按账号保存到 `auth/trial-ledger.json`，失败至少退避 24 小时，不立即重放 POST；资格与额度以上游为准，升级时保留该文件。 @@ -164,10 +165,12 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` | WebUI 无法登录 | 确认设置了 API key；更换 key 后重新登录并重新发起未完成的 OAuth | | 本地 401 | 客户端密钥与网关不一致 | | 上游 401 / 403 | 凭证级认证熔断；在 WebUI 检查并重新登录 | -| 429 | 对该凭证的上游模型冷却，后续请求可重新绑定；当前请求不换账号重放。全部候选都在冷却时仍返回 429 | +| 429 | 对该凭证的上游模型冷却，后续请求自动换绑；全部候选都在冷却时仍返回 429。设了 `--failover-max` 时，当前请求就地换凭证重放 | | 上游 `service info not found`（11102） | 该后端根本不服务这个模型：按 (后端, 模型) 避让，把该模型派给其他后端，全部后端都没有时返回 404。6 小时后半开放行重试，反复命中最长退避 24 小时，一次成功调用即刻解除；可用 `GET /admin/model-blocks` 查看 | -| 建连失败 | 仅 `ConnectError` / `ConnectTimeout` 退避重试一次 | -| 发送后断连、读写超时、HTTP 错误 | 不做网络重放，避免重复计费；日志记录异常类型与耗时 | +| 建连失败、写请求体超时 | `ConnectError` / `ConnectTimeout` / `WriteTimeout` 换新连接重放一次：三者都意味着上游没收下请求体，重放不会重复计费 | +| 发送后断连、读超时、协议错误 | 不做网络重放，避免重复计费；日志记录异常类型与耗时 | +| 流式在第一个字节之前失败 | 按**真实状态码**返回，与 `stream=false` 同口径。只带一个流内 `error` 事件的 200 会被客户端读成「模型答了个空」，会话静默结束，审计里还记成一次成功 | +| 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败落在「一个字节都没发给下游」之前时换一个凭证重打，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。只重放上游用 HTTP 状态码给出的拒绝（401/403/429/502/503/504）与没收下请求体的传输失败：内容审核拒绝、上游已回 200 之后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码 | | 工具参数损坏 | 聚合校验失败按 `--tool-call-max-retry`（默认 3）额外生成，可能消耗更多额度；被丢弃的生成带用量记入尝试明细；耗尽后返回错误 | | 上游空流或残流 | 没有有效输出、缺少结束标记或包含错误的流不伪装为成功 | | 内容审核拒绝 | 脱敏 + `--no-compact` 下，仅完整非流式纯拒绝且模板确实缩短时，最多同账号兜底一次；流式不做审核重试，也不因此熔断或切号 | diff --git a/tests/test_refusal.py b/tests/test_refusal.py index 9753d1b..983cce5 100644 --- a/tests/test_refusal.py +++ b/tests/test_refusal.py @@ -233,18 +233,14 @@ def test_empty_stop_done_errors_on_all_six_paths_without_normal_completion_or_re for tools in (False, True): with self.subTest(route=route, stream=stream, tools=tools): response = self.request(route, stream, tools) - if not stream: - self.assertEqual(response.status_code, 502, response.text) - self.assertIn("error", response.json().get("detail", response.json())) - continue - parsed = events(response.text) - self.assertTrue(any("error" in event for event in parsed), response.text) + # 空终止在落第一个字节之前就能判定，所以流式与非流式同一口径：真实 502。 + # 过去流式回 200 + 带内 error 帧，下游 SDK 解析不到 choices / + # response.completed，会把失败读成「模型答了个空」并静默结束会话。 + self.assertEqual(response.status_code, 502, response.text) + self.assertIn("error", response.json().get("detail", response.json())) self.assertNotIn("data: [DONE]", response.text) - self.assertFalse(any(choice.get("finish_reason") for event in parsed - for choice in event.get("choices", [])), response.text) - self.assertFalse(any(event.get("type") in ( - "response.completed", "response.output_item.done", "response.output_text.done", - "message_stop", "message_delta") for event in parsed), response.text) + self.assertNotIn("response.completed", response.text) + self.assertNotIn("message_stop", response.text) if __name__ == "__main__": diff --git a/tests/test_runtime_endpoints.py b/tests/test_runtime_endpoints.py index 4267ae8..c7d1ec6 100644 --- a/tests/test_runtime_endpoints.py +++ b/tests/test_runtime_endpoints.py @@ -247,7 +247,9 @@ def test_credential_selection_runs_off_the_event_loop(self): direct = re.findall(r"^\s+(?:body|chat_body), cred, headers, url = _route_chat\(", src, re.M) pooled = re.findall(r"await run_in_threadpool\(_route_chat", src) self.assertEqual(direct, []) - self.assertEqual(len(pooled), 3) + # 三个端点各一次，另外换凭证重放（_routed_stream / _routed_fetch）还要再路由一次： + # 只要没有任何直调（direct 为空），线程池约束就仍然成立。 + self.assertGreaterEqual(len(pooled), 3) def test_tool_metadata_policy_reaches_all_protocols(self): description = "Read sandbox data without destructive changes." diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py new file mode 100644 index 0000000..8926303 --- /dev/null +++ b/tests/test_stream_failover.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""流式换凭证重放回归（`--failover-max`）：本地换账号重试，而不是把 429/502 甩给下游。 + +背景：流式请求在「一个字节都还没发给下游」时失败，已经被 `_preflight_stream` 还原成真实 +状态码（见 tests/test_stream_status_contract.py）。但还原成 429 只是诚实，不是解决问题 —— +限流/认证/网关抖动这类失败换一个账号大概率就能成，下游（尤其 Codex CLI）不该看到 502。 + +这里钉住重放的边界： + - 默认关闭（`failover_max=0`）时行为与上游完全一致：一次都不多重放，如实回真实状态码； + - 开启后只在确定「上游没收下请求体 / 上游用 HTTP 状态码拒绝」时重放，且必须换凭证； + - 审核拒绝、聚合器合成的 502（上游已回 200，可能已计费）永不重放； + - 重放次数有上界，换不出别的凭证时如实回第一次的状态码，绝不死循环。 +运行：.venv/bin/python -B -m unittest -v tests/test_stream_failover.py +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # 仓库根：允许直接运行本文件 + +import json +import unittest +from contextlib import contextmanager +from unittest.mock import patch + +import httpx +from fastapi import FastAPI +from fastapi.testclient import TestClient + +import converter +from app.audit_store import AuditStore +from app.observability import AuditMiddleware +from tests import test_region_routing as fixtures + +REPLAYABLE_STATUS = (401, 403, 429, 502, 503, 504) +DETERMINISTIC_STATUS = (400, 404, 405, 413, 422) +# 上游没收下请求体：换连接/换账号重放安全 +REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) +# 请求体已经发出去了（甚至响应已经开始）：上游可能已处理并计费，禁止重放 +AMBIGUOUS_TRANSPORT = (httpx.ReadError, httpx.ReadTimeout, httpx.WriteError, + httpx.RemoteProtocolError) +FAILOVER_LOG = "换凭证重放" + + +def error_body(message="synthetic rejection", code="rate_limit"): + return {"error": {"message": message, "type": "upstream_error", "code": code}} + + +@contextmanager +def allow_failover(times: int): + """打开换凭证重放开关（等价于 --failover-max N）。""" + with patch.dict(converter.CONFIG, {"failover_max": times}): + yield + + +class StreamFailoverTests(fixtures.RegionRoutingTests): + """复用四档合成凭证 + MockTransport，只把「被点名的那一个凭证」改成会失败。""" + + def setUp(self): + super().setUp() + self.arm_next = False + self.stream = True + self.poison_uid = None + self.poison = lambda request: httpx.Response(429, json=error_body()) + self.logs = [] + self.enterContext(patch.object(converter, "_log", side_effect=self.capture_log)) + self.allowed_profiles = set(fixtures.PROFILES) + + def fresh_pool(self): + """重建凭证池：401/403 熔断与 429 冷却都是池内状态，子例之间必须清干净。""" + self.configure() + self.allowed_profiles = set(fixtures.PROFILES) + + def capture_log(self, message, *args, **kwargs): + self.logs.append(str(message)) + + def handle_upstream(self, request): + uid = request.headers.get("x-user-id") + if self.arm_next: + self.arm_next = False + self.poison_uid = uid # 不依赖轮询顺序：下一个被选中的凭证开始失败 + if self.poison_uid is not None and uid == self.poison_uid: + self.requests.append(request) + return self.poison(request) # 可以返回错误状态，也可以直接抛传输层异常 + return super().handle_upstream(request) + + def poison_with_status(self, status, body=None): + self.poison_uid = None + self.arm_next = True + payload = json.dumps(body or error_body()).encode() + self.poison = lambda request: httpx.Response(status, content=payload, + headers={"content-type": "application/json"}) + + def poison_with_transport(self, error_type): + self.poison_uid = None + self.arm_next = True + def raise_transport(request): + raise error_type("synthetic transport failure") + self.poison = raise_transport + + def stream_post(self, endpoint="chat/completions"): + self.allowed_profiles = set(fixtures.PROFILES) + before = len(self.requests) + response = self.client.post("/v1/" + endpoint, json=self.payload(endpoint, stream=self.stream)) + return response, self.requests[before:] + + def uids(self, requests): + return [request.headers.get("x-user-id") for request in requests] + + def failover_lines(self): + return [line for line in self.logs if FAILOVER_LOG in line] + + # --- 开启后：下游只看到一次正常成功 --- + def test_429_is_replayed_on_another_credential(self): + with allow_failover(1): + self.poison_with_status(429) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("ok", response.text) + self.assertIn("data: [DONE]", response.text) + self.assertEqual(len(sent), 2, "应当恰好重放一次") + self.assertEqual(len(set(self.uids(sent))), 2, "重放必须换凭证") + self.assertEqual(len(self.failover_lines()), 1, self.failover_lines()) + + def test_every_stream_endpoint_can_fail_over(self): + for endpoint in fixtures.GENERATIONS: + with self.subTest(endpoint=endpoint): + self.fresh_pool() + with allow_failover(1): + self.poison_with_status(429) + response, sent = self.stream_post(endpoint) + terminal = {"chat/completions": "data: [DONE]", + "responses": '"response.completed"', + "messages": "event: message_stop"}[endpoint] + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("ok", response.text) + self.assertIn(terminal, response.text) + self.assertEqual(len(sent), 2) + self.assertEqual(len(set(self.uids(sent))), 2) + + def test_failover_limit_bounds_upstream_attempts(self): + self.response_status = 429 # 所有凭证都失败 + for maximum, expected in ((1, 2), (2, 3)): + with self.subTest(failover_max=maximum): + self.fresh_pool() # 上一轮的 429 冷却会让这一轮少打一次上游 + self.requests.clear() + self.logs.clear() + with allow_failover(maximum): + response = self.client.post("/v1/chat/completions", + json=self.payload(stream=True)) + self.assertEqual(response.status_code, 429, response.text) + self.assertEqual(len(self.requests), expected, + f"重放次数必须恰好等于 failover_max={maximum}") + self.assertEqual(len(self.failover_lines()), maximum, self.failover_lines()) + self.assertEqual(len(set(self.uids(self.requests))), expected, "每轮都必须是新凭证") + + def test_last_resort_surfaces_the_real_status(self): + """换不出别的凭证（只剩一个账号）时，如实回第一次的状态码，且不再打上游。""" + self.configure(profiles=("cn-cli",)) + self.allowed_profiles = {"cn-cli"} + with allow_failover(3): + self.poison_with_status(429) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 429, response.text) + self.assertEqual(len(sent), 1, "无凭证可换时不得重复打上游") + + def test_reroute_cannot_loop_back_to_the_same_credential(self): + """重放选回同一个凭证时（单凭证池/黏绑）必须立刻收敛，不能死循环。""" + self.configure(profiles=("cn-cli", "cn-work")) + self.allowed_profiles = set(fixtures.PROFILES) + with allow_failover(5): + self.response_status = 401 + response = self.client.post("/v1/chat/completions", json=self.payload(stream=True)) + self.assertEqual(response.status_code, 401, response.text) + self.assertLessEqual(len(self.requests), 3, "每轮都要换新凭证，池子耗尽即停") + self.assertEqual(len(set(self.uids(self.requests))), len(self.requests), + "同一凭证不得被打两次") + + # --- 默认关闭：与上游一致，一次都不多重放 --- + def test_disabled_by_default_replays_nothing(self): + self.assertEqual(converter.CONFIG["failover_max"], 0, "默认必须关闭，行为与上游一致") + for status in REPLAYABLE_STATUS: + for stream in (True, False): + with self.subTest(status=status, stream=stream): + self.fresh_pool() + self.stream = stream + self.requests.clear() + self.logs.clear() + self.poison_with_status(status, error_body(code="quota")) + response, sent = self.stream_post() + self.assertEqual(response.status_code, status, response.text) + self.assertEqual(len(sent), 1, "关闭时禁止任何重放") + self.assertEqual(self.failover_lines(), []) + self.stream = True + + # --- 审计口径：重放救回来的请求不得留在 error --- + def audited_client(self): + store = AuditStore(self.root / "failover-audit.sqlite3") + self.addCleanup(store.close) + application = FastAPI() + application.router.routes = list(converter.app.router.routes) + application.add_middleware(AuditMiddleware, store) + return store, self.enterContext(TestClient(application)) + + def only_record(self, store): + records = store.list_records()["items"] + self.assertEqual(len(records), 1, records) + return records[0] + + def test_replayed_request_is_audited_as_success(self): + """重放成功的请求审计必须是 success，不能又退回「error + 200」这个骗人的签名。""" + store, client = self.audited_client() + with allow_failover(1): + self.poison_with_status(429) + response = client.post("/v1/chat/completions", json=self.payload(stream=True)) + self.assertEqual(response.status_code, 200, response.text) + record = self.only_record(store) + self.assertEqual(record["outcome"], "success", record) + self.assertFalse(record.get("error_code"), record) + statuses = [attempt.get("status_code") for attempt in record["attempts"]] + self.assertEqual(statuses[:2], [429, 200], record["attempts"]) + self.assertIn("failover_recovered", json.dumps(record["attempts"], ensure_ascii=False)) + + def test_unreplayed_failure_is_still_audited_as_error(self): + store, client = self.audited_client() + self.poison_with_status(429) + response = client.post("/v1/chat/completions", json=self.payload(stream=True)) + self.assertEqual(response.status_code, 429, response.text) + record = self.only_record(store) + self.assertEqual(record["outcome"], "error", record) + self.assertEqual(record["error_code"], "upstream_429", record) + + # --- 重放判定矩阵：只重放「上游确定没收下/没处理」的失败 --- + def test_replayable_http_statuses(self): + for status in REPLAYABLE_STATUS: + with self.subTest(status=status): + self.assertTrue(converter._failover_safe( + converter.UpstreamHTTPError(status, b'{"error":{"code":"quota"}}'))) + + def test_deterministic_http_statuses_are_not_replayed(self): + for status in DETERMINISTIC_STATUS: + with self.subTest(status=status): + self.assertFalse(converter._failover_safe( + converter.UpstreamHTTPError(status, b'{"error":{"code":"bad_request"}}'))) + + def test_aggregator_synthesized_502_is_not_replayed(self): + """上游已经回了 200，聚合器合成的 502 可能对应已计费的请求：不换账号重放。""" + self.assertFalse(converter._failover_safe( + converter.UpstreamResponseError(502, json.dumps(error_body(code="empty_response")).encode()))) + with allow_failover(2): + self.poison_uid = None + self.arm_next = True + self.poison = lambda request: httpx.Response( + 200, content=b"", headers={"content-type": "text/event-stream"}) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 502, response.text) + self.assertEqual(len(sent), 1, "空流不得重放") + self.assertEqual(self.failover_lines(), []) + + def test_filter_rejection_is_never_replayed(self): + """内容审核是模型的真实答复，换账号只会再撞同一堵墙，还会白烧一次额度。""" + refusal = error_body("请求包含违规内容，已被拦截", code="content_filter") + for status in (403, 429): + with self.subTest(status=status): + self.fresh_pool() + with allow_failover(2): + self.poison_with_status(status, refusal) + response, sent = self.stream_post() + self.assertEqual(response.status_code, status, response.text) + self.assertEqual(len(sent), 1) + self.assertEqual(self.failover_lines(), []) + self.assertFalse(converter._failover_safe( + converter.UpstreamHTTPError(403, json.dumps(refusal).encode()), + json.dumps(refusal).encode())) + + def test_transport_before_request_body_fails_over(self): + for error_type in REPLAYABLE_TRANSPORT: + with self.subTest(error=error_type.__name__): + self.requests.clear() + self.logs.clear() + with allow_failover(1): + self.poison_with_transport(error_type) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("data: [DONE]", response.text) + # 建连失败/写超时在 open_backend_stream 内部已换连接重试一次，再换凭证重放一次 + self.assertGreaterEqual(len(sent), 2) + self.assertNotEqual(len(set(self.uids(sent))), 1, "必须换过凭证") + self.assertEqual(len(self.failover_lines()), 1, self.failover_lines()) + + def test_ambiguous_transport_never_fails_over(self): + for error_type in AMBIGUOUS_TRANSPORT: + with self.subTest(error=error_type.__name__): + self.fresh_pool() + self.requests.clear() + self.logs.clear() + with allow_failover(2): + self.poison_with_transport(error_type) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 502, response.text) + self.assertEqual(len(sent), 1, "请求体可能已被上游处理，禁止换账号重放") + self.assertEqual(self.failover_lines(), []) + + def test_non_streaming_request_is_replayed_too(self): + """非流式一个字节都没回下游，判定同一流式口径：换凭证重放，下游只看到一次成功。""" + with allow_failover(1): + self.stream = False + self.poison_with_status(429) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.json()["choices"][0]["message"]["content"], "ok") + self.assertEqual(len(sent), 2, "应当恰好重放一次") + self.assertEqual(len(set(self.uids(sent))), 2, "重放必须换凭证") + self.assertEqual(len(self.failover_lines()), 1, self.failover_lines()) + self.stream = True + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_stream_status_contract.py b/tests/test_stream_status_contract.py new file mode 100644 index 0000000..53fd957 --- /dev/null +++ b/tests/test_stream_status_contract.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""流式状态码契约回归：上游在落第一个字节之前就把请求判死时，不许回 HTTP 200。 + +下游（Codex CLI、Claude Code、任意 OpenAI 兼容 SDK）是按 chunk / 事件解析的：一个既没有 +choices、也等不到 response.completed 的 200 流会被读成「模型答了个空」，会话静默结束 —— +既不重试也不报错，审计里还记成一次成功。StreamingResponse 一旦被迭代就把响应头发出去， +而打上游发生在生成器里面，所以过去只有 stream=false 才吃得到真实状态码。 + +这里钉住修好的口径：**同一个失败，流式与非流式必须给同一个状态码**；而真的中途断流 +（字节已经发出去了）仍然只能在流内报错，收不回状态码。 +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # 仓库根：允许直接运行本文件 + +import json +import unittest +from unittest.mock import patch + +import httpx +from fastapi.testclient import TestClient + +import converter +from app import upstream_io + +ROUTES = ("/v1/chat/completions", "/v1/responses", "/v1/messages") +TOOLS = [{"type": "function", "function": {"name": "synthetic_tool", "parameters": {"type": "object"}}}] +# 写超时/建连失败时上游还没收下请求体，open_backend_stream 会换新连接重放一次； +# 中途 reset 与协议错误属于「歧义请求」，绝不重放 POST。 +REPLAYABLE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) +AMBIGUOUS = (httpx.ReadError, httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.WriteError) +HTTP_STATUSES = (400, 403, 429, 503) +TERMINALS = ("data: [DONE]", '"response.completed"', '"message_stop"') + + +def sse_body(text="ok"): + chunks = [{"choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}], "model": "auto"}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "model": "auto", + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}}] + return "".join("data: " + json.dumps(c) + "\n\n" for c in chunks).encode() + b"data: [DONE]\n\n" + + +class StreamStatusTests(unittest.TestCase): + def setUp(self): + self.enterContext(patch.dict(converter.CONFIG, { + "api_key": "", "cred": None, "cred_pool": None, "model_guard": False, + "max_images": 16, "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024, + "log_body_limit": 65536, "log_path": None, "desensitize": False, "no_compact": False})) + self.enterContext(patch.object(converter, "_cred_for", return_value=(None, {}))) + self.enterContext(patch.object(converter, "_log")) + self.enterContext(patch.object(converter, "_note_cred_status")) + self.requests = [] + self.respond = lambda request: httpx.Response(200, content=sse_body()) + real_client = httpx.AsyncClient + transport = httpx.MockTransport(self.handle) + self.enterContext(patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw))) + self.client = self.enterContext(TestClient(converter.app)) + + def handle(self, request): + self.requests.append(request) + return self.respond(request) + + def post(self, route, *, stream, tools=False, model="auto"): + if route == "/v1/responses": + payload = {"model": model, "stream": stream, "input": [{"role": "user", "content": "hi"}]} + if tools: + payload["tools"] = TOOLS + elif route == "/v1/messages": + payload = {"model": model, "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]} + if stream is False: + payload["stream"] = False + else: + payload["stream"] = True + if tools: + payload["tools"] = [{"name": "synthetic_tool", "input_schema": {"type": "object"}}] + else: + payload = {"model": model, "stream": stream, "messages": [{"role": "user", "content": "hi"}]} + if tools: + payload["tools"] = TOOLS + return self.client.post(route, json=payload) + + # --- 上游用 HTTP 状态码说的不：流式必须原样转出去 --- + def test_upstream_http_status_is_not_swallowed_by_streaming(self): + for route in ROUTES: + for status in HTTP_STATUSES: + for tools in (False, True): + with self.subTest(route=route, status=status, tools=tools): + self.respond = lambda request: httpx.Response( + status, json={"error": {"message": "synthetic rejection", "code": "rate_limit"}}) + self.requests.clear() + streamed = self.post(route, stream=True, tools=tools) + self.requests.clear() + plain = self.post(route, stream=False, tools=tools) + self.assertEqual(streamed.status_code, status, streamed.text) + self.assertEqual(plain.status_code, status, plain.text) + self.assertEqual(len(self.requests), 1, "失败不得重放上游") + + # --- 传输层失败（写超时/中途 reset）：非流式一直是 502，流式过去是 200 --- + def test_transport_failure_is_502_on_both_stream_modes(self): + for route in ROUTES: + for error_type in REPLAYABLE + AMBIGUOUS: + expected = 2 if error_type in REPLAYABLE else 1 + with self.subTest(route=route, error=error_type.__name__): + def raise_transport(request): + raise error_type("synthetic transport failure") + self.respond = raise_transport + for stream in (True, False): + with self.subTest(stream=stream): + self.requests.clear() + response = self.post(route, stream=stream) + self.assertEqual(response.status_code, 502, response.text) + self.assertIn(error_type.__name__, response.text, response.text) + self.assertEqual(len(self.requests), expected, response.text) + + # --- 一个字节都没发出去时，不得留下「看起来成功」的流 --- + def test_failed_stream_never_carries_a_success_terminal(self): + self.respond = lambda request: httpx.Response(429, json={"error": {"message": "slow down"}}) + for route in ROUTES: + for tools in (False, True): + with self.subTest(route=route, tools=tools): + response = self.post(route, stream=True, tools=tools) + self.assertEqual(response.status_code, 429, response.text) + self.assertNotEqual(response.headers.get("content-type", "").split(";")[0], + "text/event-stream") + for marker in TERMINALS: + self.assertNotIn(marker, response.text) + + # --- 已经吐过字节的中途断流：状态码收不回来，只能在流内报错（不许过度修正）--- + def test_break_after_first_byte_stays_in_band(self): + class Partial(httpx.AsyncByteStream): + async def __aiter__(self): + yield b'data: {"choices":[{"index":0,"delta":{"content":"partial"},"finish_reason":null}]}\n\n' + raise httpx.ReadError("synthetic mid-stream reset") + + for route in ROUTES: + if route == "/v1/responses": + continue # Responses 端点总是先聚合再落字节，失败时一个字节都没发出去 → 502 + with self.subTest(route=route): + self.respond = lambda request: httpx.Response(200, stream=Partial(), + headers={"content-type": "text/event-stream"}) + response = self.post(route, stream=True) + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("ReadError", response.text) + self.assertIn("partial", response.text) + self.assertNotIn("data: [DONE]", response.text) + self.assertNotIn('"message_stop"', response.text) + + # --- 成功路径不能被预取吞掉第一段 --- + def test_happy_stream_still_delivers_every_event_once(self): + for route in ROUTES: + for tools in (False, True): + with self.subTest(route=route, tools=tools): + self.requests.clear() + response = self.post(route, stream=True, tools=tools) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(response.headers["content-type"].split(";")[0], "text/event-stream") + self.assertEqual(len(self.requests), 1) + if route == "/v1/responses": + self.assertEqual(response.text.count('"response.created"'), 1) + self.assertEqual(response.text.count('"response.completed"'), 1) + elif route == "/v1/messages": + self.assertEqual(response.text.count("event: message_start"), 1) + self.assertEqual(response.text.count("event: message_stop"), 1) + else: + self.assertIn("ok", response.text) + self.assertIn("data: [DONE]", response.text) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_upstream_io.py b/tests/test_upstream_io.py index 09365a4..14ddf99 100644 --- a/tests/test_upstream_io.py +++ b/tests/test_upstream_io.py @@ -316,6 +316,48 @@ def handler(request): self.assertEqual(len(requests), 1) self.assertEqual(requests[0].method, "POST") + async def test_body_not_accepted_is_replayed_on_a_fresh_connection(self): + """建连失败 / 写请求体超时：上游没收下请求体，换新连接重放一次不会重复计费。""" + real_client = httpx.AsyncClient + for error_type in (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout): + with self.subTest(error=error_type.__name__): + attempts = [] + + def handler(request): + attempts.append(request) + if len(attempts) == 1: + raise error_type("synthetic failure before the body was accepted") + return httpx.Response(200, content=b"data: [DONE]\n\n") + + transport = httpx.MockTransport(handler) + with patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw)): + async with upstream_io.open_backend_stream("https://synthetic.invalid", {}, {}) as response: + self.assertEqual(response.status_code, 200) + self.assertEqual(await response.aread(), b"data: [DONE]\n\n") + self.assertEqual(len(attempts), 2, "只允许重放一次") + self.assertEqual([request.method for request in attempts], ["POST", "POST"]) + + async def test_ambiguous_transport_failures_never_replay_post(self): + """请求体已经发出（甚至响应已经开始）的失败有计费歧义，一律交给调用方按协议返回。""" + real_client = httpx.AsyncClient + for error_type in (httpx.ReadError, httpx.ReadTimeout, httpx.WriteError, + httpx.RemoteProtocolError): + with self.subTest(error=error_type.__name__): + attempts = [] + + def handler(request): + attempts.append(request) + raise error_type("synthetic failure after the body was sent") + + transport = httpx.MockTransport(handler) + with patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw)): + with self.assertRaises(error_type): + async with upstream_io.open_backend_stream("https://synthetic.invalid", {}, {}) as response: + await response.aread() + self.assertEqual(len(attempts), 1, "歧义请求不得重放 POST") + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_workbuddy_filter.py b/tests/test_workbuddy_filter.py index b413f1d..600d17f 100644 --- a/tests/test_workbuddy_filter.py +++ b/tests/test_workbuddy_filter.py @@ -305,7 +305,10 @@ def test_http_and_sse_filter_errors_are_preserved_without_auth_cooldown_or_retry self.assert_one_request() self.credential_status.assert_not_called() self.failures.assert_called_once_with("content_filter") - self.assertEqual(response.status_code, 200 if stream else 502 if status == 200 else status) + # 审核拒绝同样落在第一个字节之前，流式必须与 stream=False 吃同一个真实 + # 状态码；一律 200 + 带内 error 会让下游把失败当成空回答静默结束会话。 + self.assertEqual(response.status_code, 502 if status == 200 else status, + response.text) self.assertIn("content_filter", response.text) def test_explicit_empty_filter_can_retry_nonstream_but_never_stream(self): @@ -370,7 +373,11 @@ def respond(req): self.respond = respond response = self.post(route, stream=stream) self.assert_one_request() - self.assertEqual(response.status_code, 200 if stream else 502) + # chat / messages 的真流式此时已经吐过拒绝文本，收不回状态码，只能带内 + # 报错；responses 总是先聚合再落字节，失败时一个字节都没发出去 → 真实 502。 + already_streamed = stream and route != "/v1/responses" + self.assertEqual(response.status_code, 200 if already_streamed else 502, + response.text) self.assertNotIn("response.completed", response.text) self.assertNotIn("message_stop", response.text) From ce373e7b9d54b1ca0daacf09084810a885934f34 Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:03:03 +0000 Subject: [PATCH 2/8] Tag the replays that may already have been billed upstream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on this PR asked whether replaying `httpx.WriteTimeout` can double-bill: a write timeout proves the client did not finish writing the body, not that the upstream accepted no bytes of it. Same question for the 502/504 in `FAILOVER_CODES` — the upstream gateway can give up on a backend that had already run the request. Both are true, and neither changes what the operator should do, because a failure at this point has no response headers and no result: the client must retry regardless. Refusing to replay does not return the credit that attempt may already have spent; it only turns a paid-for attempt into a session that ends with nothing. The choice is one billed attempt and a broken session, or two attempts and an answer. So keep the behaviour and make the cost visible instead: - `POSSIBLY_CHARGED_CODES` (502/504) marks the class where the upstream may already have processed the request; `_replay_cost_note()` appends "上游可能已处理该请求" to those replay log lines and only those. 401/403/ 429/503 and the transport failures happen at admission time, so they stay untagged. Reconciliation against the official usage detail is now a grep away rather than a guess. - Say why `WriteTimeout` still counts as "body never accepted": the request carries a `Content-Length`, so a write timeout means the declared body was not fully written — the upstream holds no complete request to run. - `--failover-max` stays default 0, so all of this is opt-in and bounded. Docs (en + zh) and `.env.example` now state the billing boundary per status class instead of claiming none of it can ever cost anything. --- .env.example | 5 ++++- converter.py | 28 ++++++++++++++++++++++++---- docs/advanced.md | 2 +- docs/advanced.zh-CN.md | 2 +- tests/test_stream_failover.py | 21 +++++++++++++++++++++ 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 519bba5..da852e3 100644 --- a/.env.example +++ b/.env.example @@ -36,5 +36,8 @@ CODEBUDDY2API_AUTO_TRIAL=false # 换凭证重放次数：失败发生在向下游落第一个字节之前时，最多再换几个凭证就地重放。 # 默认 0（关闭）：如实把上游 429/502 回给下游。只重放上游没收下请求体（建连失败、写请求体 # 超时）或用 401/403/429/502/503/504 拒绝的失败；内容审核拒绝与上游已回 200 之后合成的 -# 502 一律不重放。详见 docs/advanced.zh-CN.md 的「换凭证重放」。 +# 502 一律不重放。 +# 计费口径：401/403/429/503 与三类传输失败都发生在受理阶段，不产生扣费；502/504 有可能已经 +# 被后端处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——日志会给这类重打单独标 +# 「上游可能已处理该请求」，方便按官方用量明细核对。详见 docs/advanced.zh-CN.md「换凭证重放」。 CODEBUDDY2API_FAILOVER_MAX=0 diff --git a/converter.py b/converter.py index ce69889..b4e9a2e 100644 --- a/converter.py +++ b/converter.py @@ -2716,7 +2716,18 @@ def _cred_manager(cred): # 也一样，不在其中。 FAILOVER_CODES = frozenset({401, 403, 429, 502, 503, 504}) # 请求体确定没被上游收下的传输失败（建连失败 / 写请求体超时），重放不会重复计费。 +# 写超时按定义就是「Content-Length 声明的正文没写完」：上游手里没有完整请求，跑不出结果。 REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) +# 上游网关在拿到后端答复之前就把错误抛回来的状态：后端那侧可能已经处理完并计费。仍然重放 +# （理由见 _failover_safe），但要如实标出来，便于事后拿官方账本核对。 +POSSIBLY_CHARGED_CODES = frozenset({502, 504}) + + +def _replay_cost_note(error) -> str: + """重放日志里的代价标记：只给「可能已经付费」的那一类加，别把 429 也说成有风险。""" + if isinstance(error, UpstreamHTTPError) and error.status in POSSIBLY_CHARGED_CODES: + return " | 上游可能已处理该请求" + return "" def _failover_safe(error, raw=b"") -> bool: @@ -2724,8 +2735,15 @@ def _failover_safe(error, raw=b"") -> bool: 三条硬边界：内容审核拒绝不切号重放（那是模型的真实答复，换账号只会再撞一次同一堵墙， 还白烧一次额度）；聚合器从 200 响应体里合成的 502（空流、坏 SSE、已开流后断连）不重放， - 因为上游可能已经处理并计费；真正的重放窗口由 `open_backend_stream` 的 `opened` 标记与 - `_preflight_stream` 守住。 + 因为上游已经回了 200、可能已经计费，而且那时状态码还收得回来；真正的重放窗口由 + `open_backend_stream` 的 `opened` 标记与 `_preflight_stream` 守住。 + + 为什么 502/504 这类「上游可能已经处理并计费」的失败仍然重放：这类失败对下游是**彻底 + 失败**——连响应头都没有，更没有可用的结果。不重放并不能把已经花掉的额度退回来，只是把 + 一次已经付出的请求换成一段静默断掉的会话。所以取舍不是「省钱 vs 花钱」，而是「花一次已 + 付的学费 vs 花两次并给出结果」。代价因此被严格夹住：默认 `--failover-max=0` 完全关闭， + 开启后每请求最多多打 N 次，且这类重放在日志里由 `_replay_cost_note()` 单独标注，可事后 + 按官方用量明细核对。 """ if is_filter_error(raw): return False @@ -2812,7 +2830,8 @@ async def _routed_stream(payload, body, model_name, rid, t0, make, routed, cred, raise surface from None routed, cred, headers, url = attempt _log(f"[{rid}] ↻ 换凭证重放 {len(tried)}/{limit} | {model_name} | 上游 HTTP " - f"{failure.status} → {profile_for_headers(headers)}") + f"{failure.status} → {profile_for_headers(headers)}" + f"{_replay_cost_note(failure.error)}") async def _routed_fetch(payload, body, model_name, rid, t0, fetch, routed, cred, headers, url): @@ -2840,7 +2859,8 @@ async def _routed_fetch(payload, body, model_name, rid, t0, fetch, routed, cred, raise surface from None routed, cred, headers, url = attempt _log(f"[{rid}] ↻ 换凭证重放 {len(tried)}/{limit} | {model_name} | 上游 HTTP " - f"{status} → {profile_for_headers(headers)}") + f"{status} → {profile_for_headers(headers)}" + f"{_replay_cost_note(error)}") def _note_content_filter(rid, model_name, *, final): diff --git a/docs/advanced.md b/docs/advanced.md index 47697b4..e8a866e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -173,7 +173,7 @@ Credential domain / token issuer determine the product identity. Chat and refres | Connection setup failure, write timeout | Retry once on a fresh connection: `ConnectError`, `ConnectTimeout` and `WriteTimeout` all mean the request body was never accepted, so replaying cannot double-bill | | Post-send disconnect, read timeout or protocol error | No network replay, avoiding duplicate billing; logs include exception type and elapsed time | | Streaming request fails before the first byte | Reported with the real HTTP status, exactly like `stream=false`. A 200 carrying only an in-band `error` event is read by clients as an empty answer, so the session ends silently while the audit log records a success | -| Credential failover (`--failover-max`) | Off by default. When enabled, a failure that happened before any byte reached the client is retried on another credential, up to N times, and is audited as `success` with a `failover_recovered` attempt marker. Only upstream HTTP rejections (401/403/429/502/503/504) and never-accepted request bodies qualify: content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors are never replayed, and if no other credential is available the original status is surfaced | +| Credential failover (`--failover-max`) | Off by default. When enabled, a failure that happened before any byte reached the client is retried on another credential, up to N times, and is audited as `success` with a `failover_recovered` attempt marker. Only upstream HTTP rejections (401/403/429/502/503/504) and never-accepted request bodies qualify: content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors are never replayed, and if no other credential is available the original status is surfaced — **Billing**: 401/403/429/503 and the transport failures above happen at admission time and cannot be billed; a 502/504 may already have been processed and billed upstream, but its result never reached the client, so refusing to replay it recovers no credit — it only turns a paid-for attempt into a broken session. Such replays are tagged `上游可能已处理该请求` in the log for reconciliation | | Malformed tool calls | Aggregate validation permits up to `--tool-call-max-retry` (default 3) additional generations, each consuming credits and recorded with its usage in the attempt details; exhaustion returns an error | | Empty or truncated upstream stream | No valid output, a missing end marker or an error is not reported as success | | Content-filter rejection | With desensitization and `--no-compact`, a complete non-streaming filter-only rejection may receive one shorter-template retry on the same account. No streaming filter retry, circuit opening or account rotation | diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 7a59b6c..74af135 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -170,7 +170,7 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` | 建连失败、写请求体超时 | `ConnectError` / `ConnectTimeout` / `WriteTimeout` 换新连接重放一次：三者都意味着上游没收下请求体，重放不会重复计费 | | 发送后断连、读超时、协议错误 | 不做网络重放，避免重复计费；日志记录异常类型与耗时 | | 流式在第一个字节之前失败 | 按**真实状态码**返回，与 `stream=false` 同口径。只带一个流内 `error` 事件的 200 会被客户端读成「模型答了个空」，会话静默结束，审计里还记成一次成功 | -| 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败落在「一个字节都没发给下游」之前时换一个凭证重打，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。只重放上游用 HTTP 状态码给出的拒绝（401/403/429/502/503/504）与没收下请求体的传输失败：内容审核拒绝、上游已回 200 之后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码 | +| 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败落在「一个字节都没发给下游」之前时换一个凭证重打，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。只重放上游用 HTTP 状态码给出的拒绝（401/403/429/502/503/504）与没收下请求体的传输失败：内容审核拒绝、上游已回 200 之后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码。**计费口径**：401/403/429/503 与上述三类传输失败都发生在受理阶段，不会扣费；502/504 有可能已被后端处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——只是把一次已经付费的请求换成一段断掉的会话。这类重放在日志里单独标注「上游可能已处理该请求」，便于按官方用量明细核对 | | 工具参数损坏 | 聚合校验失败按 `--tool-call-max-retry`（默认 3）额外生成，可能消耗更多额度；被丢弃的生成带用量记入尝试明细；耗尽后返回错误 | | 上游空流或残流 | 没有有效输出、缺少结束标记或包含错误的流不伪装为成功 | | 内容审核拒绝 | 脱敏 + `--no-compact` 下，仅完整非流式纯拒绝且模板确实缩短时，最多同账号兜底一次；流式不做审核重试，也不因此熔断或切号 | diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py index 8926303..d072389 100644 --- a/tests/test_stream_failover.py +++ b/tests/test_stream_failover.py @@ -121,6 +121,27 @@ def test_429_is_replayed_on_another_credential(self): self.assertEqual(len(set(self.uids(sent))), 2, "重放必须换凭证") self.assertEqual(len(self.failover_lines()), 1, self.failover_lines()) + def test_replay_log_separates_the_maybe_billed_class(self): + """受理期拒绝不标风险；502/504 可能已被后端处理并计费，必须在日志里单独标出来。 + + 重放的取舍不是「省钱 vs 花钱」：这类失败连响应头都没有，那次结果对下游永远拿不到， + 不重放也退不回额度，只是把一次已付费的请求换成一段断掉的会话。所以保留重放，但要 + 如实标注，便于事后按官方用量明细核对。 + """ + for status, marked in ((429, False), (401, False), (403, False), (503, False), + (502, True), (504, True)): + with self.subTest(status=status): + self.fresh_pool() + self.logs.clear() + with allow_failover(1): + self.poison_with_status(status) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(sent), 2) + lines = self.failover_lines() + self.assertEqual(len(lines), 1, lines) + self.assertEqual(("上游可能已处理该请求" in lines[0]), marked, lines[0]) + def test_every_stream_endpoint_can_fail_over(self): for endpoint in fixtures.GENERATIONS: with self.subTest(endpoint=endpoint): From 977ee32ad82197a4b80670e0f79fe8f626482e4d Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:40:29 +0000 Subject: [PATCH 3/8] Move write timeouts out of the default replay set A write timeout proves that the declared body was not fully sent. It does not prove that the upstream ignored the bytes it did receive: what a peer does with a truncated body is upstream behaviour and cannot be observed from this side. Asserting otherwise was the weak point of the previous revision, so stop treating WriteTimeout as proof of an unaccepted request. - default replay set is back to ConnectError/ConnectTimeout only, in both the connection retry (`open_backend_stream`) and credential failover (`_failover_safe`); - `--retry-write-timeout` / `CODEBUDDY2API_RETRY_WRITE_TIMEOUT` opts in, because long cross-border sessions hit the write timeout far more often than the handshake (in one deployment's logs 100% of transport failures were write timeouts) and operators who have confirmed their upstream does not bill partial bodies should be able to recover those sessions; opt-in replays carry the same "may already have been billed" log tag; - both knobs are now hot entries in the management console's settings page, so failover can be tuned without a restart; - docs and CLI help reworded to match what the exception actually proves. --- .env.example | 16 +++--- app/settings.py | 4 ++ app/upstream_io.py | 23 ++++++--- converter.py | 31 +++++++++--- docs/advanced.md | 8 +-- docs/advanced.zh-CN.md | 8 +-- tests/test_stream_failover.py | 73 +++++++++++++++++++++++++++- tests/test_stream_status_contract.py | 9 ++-- tests/test_upstream_io.py | 46 +++++++++++++++++- 9 files changed, 184 insertions(+), 34 deletions(-) diff --git a/.env.example b/.env.example index da852e3..f9c6098 100644 --- a/.env.example +++ b/.env.example @@ -34,10 +34,14 @@ CODEBUDDY2API_LOG_BODY_LIMIT=65536 CODEBUDDY2API_AUTO_TRIAL=false # 换凭证重放次数：失败发生在向下游落第一个字节之前时，最多再换几个凭证就地重放。 -# 默认 0（关闭）：如实把上游 429/502 回给下游。只重放上游没收下请求体（建连失败、写请求体 -# 超时）或用 401/403/429/502/503/504 拒绝的失败；内容审核拒绝与上游已回 200 之后合成的 -# 502 一律不重放。 -# 计费口径：401/403/429/503 与三类传输失败都发生在受理阶段，不产生扣费；502/504 有可能已经 -# 被后端处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——日志会给这类重打单独标 -# 「上游可能已处理该请求」，方便按官方用量明细核对。详见 docs/advanced.zh-CN.md「换凭证重放」。 +# 默认 0（关闭）：如实把上游 429/502 回给下游。只重放上游确定没收下请求体（建连失败/建连超时） +# 或用 401/403/429/502/503/504 拒绝的失败；内容审核拒绝与上游已回 200 之后合成的 502 不重放。 +# 计费口径：401/403/429/503 与建连类失败都发生在受理阶段，不产生扣费；502/504 有可能已被后端 +# 处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——日志给这类重打单独标 +# 「上游可能已处理该请求」，便于按官方用量明细核对。详见 docs/advanced.zh-CN.md「换凭证重放」。 CODEBUDDY2API_FAILOVER_MAX=0 + +# 是否把「写请求体超时」也算作上游没收下请求体（同时作用于连接重试与上面的换凭证重放）。 +# 默认 false：写超时只能证明正文没写完，上游有没有按已收到的半截正文计费，从这一侧看不到。 +# 跨境长会话最容易撞的恰是 60s 写超时，确认自己的上游不会按半截正文计费再打开。 +CODEBUDDY2API_RETRY_WRITE_TIMEOUT=false diff --git a/app/settings.py b/app/settings.py index 2893029..f51f846 100644 --- a/app/settings.py +++ b/app/settings.py @@ -40,6 +40,10 @@ def _item(default, type_, label, *, mode="hot", env=None, minimum=None, maximum= "max_request_bytes": _item(32 * 1024 * 1024, "integer", "请求字节上限", env="CODEBUDDY2API_MAX_REQUEST_BYTES", minimum=1, maximum=1024**3), "log_body_limit": _item(65536, "integer", "文本正文预览字节", env="CODEBUDDY2API_LOG_BODY_LIMIT", minimum=0, maximum=1024**2), "auto_trial": _item(False, "boolean", "自动领取体验积分", env="CODEBUDDY2API_AUTO_TRIAL"), + "failover_max": _item(0, "integer", "换凭证重放次数", env="CODEBUDDY2API_FAILOVER_MAX", + minimum=0, maximum=10), + "retry_write_timeout": _item(False, "boolean", "写超时参与重放", + env="CODEBUDDY2API_RETRY_WRITE_TIMEOUT"), "audit_max_bytes": _item(256 * 1024 * 1024, "integer", "审计明细预算", minimum=1024**2, maximum=1024**4), "audit_retention_days": _item(30, "integer", "审计明细保留天数", minimum=1, maximum=36500), "audit_diagnostic_bytes": _item(8192, "integer", "失败诊断最大字节", minimum=0, maximum=8192), diff --git a/app/upstream_io.py b/app/upstream_io.py index c9912a5..170f37c 100644 --- a/app/upstream_io.py +++ b/app/upstream_io.py @@ -177,16 +177,27 @@ async def read_bounded_error(response, limit: int = ERROR_BODY_LIMIT) -> bytes: return bytes(buf) +# 写下第一个请求体字节之前就失败：上游手里没有任何正文，重放零风险。 +BODY_NOT_ACCEPTED = (httpx.ConnectError, httpx.ConnectTimeout) +# 写请求体超时：只证明「声明的正文没写完」，证不了上游没收到或没处理已经收到的那部分。 +# 半截正文会怎样是上游的行为，从这一侧观察不到，因此默认不重放（见 `retry_write_timeout`）。 +WRITE_TIMEOUT = (httpx.WriteTimeout,) + + @asynccontextmanager -async def open_backend_stream(url, headers, body, *, read_timeout=300, on_retry=None): - """只重试一次「上游还没收下请求体」的失败（建连失败 / 写请求体超时），其余交给调用方按协议返回。 +async def open_backend_stream(url, headers, body, *, read_timeout=300, on_retry=None, + retry_write_timeout=False): + """只重试一次「上游确定没收下请求体」的失败（建连失败 / 建连超时），其余交给调用方按协议返回。 + + 重试每次新建 `AsyncClient`，即重建 TCP+TLS，通常能换到另一个边缘节点。 - 写超时意味着请求体没有被完整接收，上游不会处理也不会计费，因此换一条全新连接重放是安全的： - 这里每次新建 `AsyncClient`，重试即重建 TCP+TLS，通常能换到另一个边缘节点。跨境链路上大 - 请求体（长会话）最容易撞到的正是 60s 写超时，而不是建连失败。 + `retry_write_timeout=True` 把 60s 写超时也算进重放集。跨境长会话最容易撞的正是写超时 + 而不是建连失败，实测某部署的传输失败 100% 是它；但写超时能证明的只有「正文没写完」， + 上游是否已按半截正文动过账，这一侧看不到，所以留给运维显式决定。 响应已经开始之后（`opened` 置位）绝不重放 POST。 """ + retryable = BODY_NOT_ACCEPTED + (WRITE_TIMEOUT if retry_write_timeout else ()) timeout = httpx.Timeout(read_timeout, connect=15, write=60, pool=15) for attempt in range(2): opened = False @@ -196,7 +207,7 @@ async def open_backend_stream(url, headers, body, *, read_timeout=300, on_retry= opened = True yield response return - except (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) as error: + except retryable as error: if opened or attempt == 1: raise if on_retry is not None: diff --git a/converter.py b/converter.py index b4e9a2e..89b8ab1 100644 --- a/converter.py +++ b/converter.py @@ -1427,6 +1427,7 @@ async def _protocol_http_exception(request: Request, exc: HTTPException): "max_inbound_bytes": 64 * 1024 * 1024, "max_collect_bytes": 8 * 1024 * 1024, "max_concurrent": 64, "failover_max": 0, # 流式失败在第一个字节之前发生时可换凭证重放的最大次数 + "retry_write_timeout": False, # 写请求体超时是否也算「上游没收下请求体」（默认否，见 --retry-write-timeout） "usage_daily": None, # 官方用量聚合视图（日期×模型 credit），供 billing/usage 出 daily_costs "usage_daily_accounts": None, # 按账号的用量快照；单账号失败不丢历史 "credit_price_cny": None, "credit_price_usd": None, "usd_rate": None, @@ -2541,7 +2542,8 @@ def retry(error): duration_ms=(time.monotonic() - started) * 1000) _log(f"[{rid}] 建连失败，重试 1/1 | {model_name} | {_network_error_text(error)}") try: - async with open_backend_stream(url, headers, body, read_timeout=timeout, on_retry=retry) as response: + async with open_backend_stream(url, headers, body, read_timeout=timeout, on_retry=retry, + retry_write_timeout=bool(CONFIG.get("retry_write_timeout"))) as response: opened = True observe_attempt("upstream_http", status_code=response.status_code, duration_ms=(time.monotonic() - started) * 1000) @@ -2715,9 +2717,11 @@ def _cred_manager(cred): # 可换凭证重放的上游 HTTP 状态：限流、认证、网关抖动。400/404/413 是确定性拒绝，换账号 # 也一样，不在其中。 FAILOVER_CODES = frozenset({401, 403, 429, 502, 503, 504}) -# 请求体确定没被上游收下的传输失败（建连失败 / 写请求体超时），重放不会重复计费。 -# 写超时按定义就是「Content-Length 声明的正文没写完」：上游手里没有完整请求，跑不出结果。 -REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) +# 上游手里没有任何正文的传输失败（建连阶段就失败），重放零风险。 +REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout) +# 写超时：正文没写完是确定的，上游有没有按已收到的半截正文动过账则观察不到， +# 因此只有 `--retry-write-timeout` 打开后才参与重放（两层重放都受这个开关约束）。 +WRITE_TIMEOUT_TRANSPORT = (httpx.WriteTimeout,) # 上游网关在拿到后端答复之前就把错误抛回来的状态：后端那侧可能已经处理完并计费。仍然重放 # （理由见 _failover_safe），但要如实标出来，便于事后拿官方账本核对。 POSSIBLY_CHARGED_CODES = frozenset({502, 504}) @@ -2727,6 +2731,8 @@ def _replay_cost_note(error) -> str: """重放日志里的代价标记：只给「可能已经付费」的那一类加，别把 429 也说成有风险。""" if isinstance(error, UpstreamHTTPError) and error.status in POSSIBLY_CHARGED_CODES: return " | 上游可能已处理该请求" + if isinstance(error, WRITE_TIMEOUT_TRANSPORT): + return " | 上游可能已处理该请求（正文未写完）" return "" @@ -2735,8 +2741,9 @@ def _failover_safe(error, raw=b"") -> bool: 三条硬边界：内容审核拒绝不切号重放（那是模型的真实答复，换账号只会再撞一次同一堵墙， 还白烧一次额度）；聚合器从 200 响应体里合成的 502（空流、坏 SSE、已开流后断连）不重放， - 因为上游已经回了 200、可能已经计费，而且那时状态码还收得回来；真正的重放窗口由 - `open_backend_stream` 的 `opened` 标记与 `_preflight_stream` 守住。 + 因为上游已经回了 200、可能已经计费，而且那时状态码还收得回来；写超时默认也不重放， + 要显式 `--retry-write-timeout`。真正的重放窗口由 `open_backend_stream` 的 `opened` 标记 + 与 `_preflight_stream` 守住。 为什么 502/504 这类「上游可能已经处理并计费」的失败仍然重放：这类失败对下游是**彻底 失败**——连响应头都没有，更没有可用的结果。不重放并不能把已经花掉的额度退回来，只是把 @@ -2751,6 +2758,8 @@ def _failover_safe(error, raw=b"") -> bool: return error.status in FAILOVER_CODES if isinstance(error, UpstreamResponseError): return False + if isinstance(error, WRITE_TIMEOUT_TRANSPORT): + return bool(CONFIG.get("retry_write_timeout")) return isinstance(error, REPLAYABLE_TRANSPORT) @@ -3269,7 +3278,13 @@ def main(): ap.add_argument("--failover-max", type=_nonnegative_int, metavar="N", default=os.environ.get("CODEBUDDY2API_FAILOVER_MAX", "0"), help="失败发生在向下游落第一个字节之前时，最多换几个凭证就地重放，默认 0（关闭）；" - "只重放上游没收下请求体或用 401/403/429/502/503/504 拒绝的失败") + "只重放上游没收下请求体或用 401/403/429/502/503/504 拒绝的失败；" + "写请求体超时需另开 --retry-write-timeout 才参与") + ap.add_argument("--retry-write-timeout", type=_boolean_arg, nargs="?", const=True, + default=os.environ.get("CODEBUDDY2API_RETRY_WRITE_TIMEOUT", "false"), + help="把「写请求体超时」也算作上游没收下请求体从而参与重放，默认 false。写超时只能" + "证明正文没写完，上游是否已按半截正文计费看不到，因此要显式开启（同时作用于连接" + "重试与 --failover-max 换凭证重放）") ap.add_argument("--auto-trial", type=_boolean_arg, nargs="?", const=True, default=os.environ.get("CODEBUDDY2API_AUTO_TRIAL", "false"), help="自动领取国际 WorkBuddy 一次性体验积分，默认关闭") @@ -3281,7 +3296,7 @@ def main(): for key in ("max_images", "image_policy", "max_request_bytes", "log_body_limit", "auto_trial", "tool_call_max_retry", "max_inbound_bytes", "max_collect_bytes", "max_concurrent", - "failover_max"): + "failover_max", "retry_write_timeout"): CONFIG[key] = getattr(args, key) CONFIG["api_key"] = args.api_key CONFIG["desensitize"] = args.desensitize diff --git a/docs/advanced.md b/docs/advanced.md index e8a866e..fbb0dcc 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -34,10 +34,11 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--max-collect-bytes` | `8388608` | Total collection budget for aggregated output (content + reasoning + tool arguments); `response_too_large` beyond it; `0` disables | | `--max-concurrent` | `64` | Concurrency limit for the three generation endpoints only; excess requests get 503 with Retry-After; token counting is unaffected; `0` disables | | `--failover-max` | `0` | Extra credentials tried when a request fails before the first response byte reaches the client; `0` keeps the upstream behaviour of surfacing the failure directly | +| `--retry-write-timeout` | `false` | Opt a request-body write timeout into replay (fresh connection and `--failover-max`), accepting that bytes already sent may have been processed | | `--max-request-bytes` | `33554432` | Positive byte limit for the processed upstream JSON | | `--log-body-limit` | `65536` | Text-log body preview bytes; `0` logs summaries only, not the SQLite diagnostic budget | -Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_AUTO_TRIAL` and `CODEBUDDY2API_FAILOVER_MAX`. See [deployment](deployment.md) for startup examples. +Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_AUTO_TRIAL`, `CODEBUDDY2API_FAILOVER_MAX` and `CODEBUDDY2API_RETRY_WRITE_TIMEOUT`. See [deployment](deployment.md) for startup examples. Trial-credit claims are off by default and only apply to upstream-eligible `intl-work` accounts. Successful/already-claimed results persist per account in `auth/trial-ledger.json`. Failures wait at least 24 hours without immediate POST replay; eligibility and amounts are determined upstream. Keep this file when upgrading. @@ -170,10 +171,11 @@ Credential domain / token issuer determine the product identity. Chat and refres | Upstream 401 / 403 | Credential-level authentication circuit opens; inspect and log in again in the WebUI | | 429 | Cool down that upstream model on the credential; later requests rebind automatically. All candidates cooling down still returns 429; with `--failover-max` the in-flight request is replayed on another credential instead | | Upstream `service info not found` (code 11102) | That backend does not serve the model at all: avoid it for the `(backend, model)` pair, route the model to another backend, and return 404 when none has it. Half-open after 6 h, exponential backoff up to 24 h, cleared at once by one successful call; inspect via `GET /admin/model-blocks` | -| Connection setup failure, write timeout | Retry once on a fresh connection: `ConnectError`, `ConnectTimeout` and `WriteTimeout` all mean the request body was never accepted, so replaying cannot double-bill | +| Connection setup failure | Retry once on a fresh connection: `ConnectError` and `ConnectTimeout` fail before the first body byte, so the upstream holds nothing and replaying cannot double-bill | | Post-send disconnect, read timeout or protocol error | No network replay, avoiding duplicate billing; logs include exception type and elapsed time | | Streaming request fails before the first byte | Reported with the real HTTP status, exactly like `stream=false`. A 200 carrying only an in-band `error` event is read by clients as an empty answer, so the session ends silently while the audit log records a success | -| Credential failover (`--failover-max`) | Off by default. When enabled, a failure that happened before any byte reached the client is retried on another credential, up to N times, and is audited as `success` with a `failover_recovered` attempt marker. Only upstream HTTP rejections (401/403/429/502/503/504) and never-accepted request bodies qualify: content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors are never replayed, and if no other credential is available the original status is surfaced — **Billing**: 401/403/429/503 and the transport failures above happen at admission time and cannot be billed; a 502/504 may already have been processed and billed upstream, but its result never reached the client, so refusing to replay it recovers no credit — it only turns a paid-for attempt into a broken session. Such replays are tagged `上游可能已处理该请求` in the log for reconciliation | +| Credential failover (`--failover-max`) | Off by default. When enabled, a failure that happened before any byte reached the client is retried on another credential, up to N times, and is audited as `success` with a `failover_recovered` attempt marker. Only upstream HTTP rejections (401/403/429/502/503/504) and request bodies the upstream provably never started receiving (`ConnectError`/`ConnectTimeout`) qualify: content-filter rejections, 502s synthesized from an already-open stream, read timeouts and protocol errors are never replayed, and if no other credential is available the original status is surfaced — **Billing**: 401/403/429/503 and those transport failures happen at admission time and cannot be billed; a 502/504 may already have been processed and billed upstream, but its result never reached the client, so refusing to replay it recovers no credit — it only turns a paid-for attempt into a broken session. Such replays are tagged `上游可能已处理该请求` in the log for reconciliation | +| Write-timeout replay (`--retry-write-timeout`) | Off by default. A write timeout proves the declared body was not fully sent, not that the upstream ignored the bytes it did receive, so it is excluded from both the connect retry and credential failover until explicitly enabled. Long cross-border sessions fail here more often than in the handshake, so operators who have confirmed their upstream does not bill partial bodies can turn this on; those replays carry the same `上游可能已处理该请求` log tag | | Malformed tool calls | Aggregate validation permits up to `--tool-call-max-retry` (default 3) additional generations, each consuming credits and recorded with its usage in the attempt details; exhaustion returns an error | | Empty or truncated upstream stream | No valid output, a missing end marker or an error is not reported as success | | Content-filter rejection | With desensitization and `--no-compact`, a complete non-streaming filter-only rejection may receive one shorter-template retry on the same account. No streaming filter retry, circuit opening or account rotation | diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 74af135..c2d4de4 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -34,10 +34,11 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--max-collect-bytes` | `8388608` | 聚合路径输出收集总字节上限（正文+思考+工具参数），超限返回 `response_too_large`；`0` 不限制 | | `--max-concurrent` | `64` | 仅限制三个生成端点；占满立即 503（含 Retry-After），不限制 token 估算；`0` 不限制 | | `--failover-max` | `0` | 请求在「一个字节都还没发给下游」之前失败时，最多再换几个凭证就地重放；`0` 表示如实把失败回给下游 | +| `--retry-write-timeout` | `false` | 让「写请求体超时」也参与重放（换新连接与 `--failover-max` 换凭证），代价是已发出的那半截正文可能已被上游处理 | | `--max-request-bytes` | `33554432` | 处理后的上游 JSON 字节上限，须为正整数 | | `--log-body-limit` | `65536` | 兼容文本日志正文预览字节；`0` 只记摘要，不控制 SQLite 诊断预算 | -环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_AUTO_TRIAL`、`CODEBUDDY2API_FAILOVER_MAX`。启动示例见 [部署指南](deployment.zh-CN.md)。 +环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_AUTO_TRIAL`、`CODEBUDDY2API_FAILOVER_MAX`、`CODEBUDDY2API_RETRY_WRITE_TIMEOUT`。启动示例见 [部署指南](deployment.zh-CN.md)。 体验积分领取默认关闭，仅适用于符合上游资格的 `intl-work` 账号。成功或已领取的结果按账号保存到 `auth/trial-ledger.json`，失败至少退避 24 小时，不立即重放 POST；资格与额度以上游为准，升级时保留该文件。 @@ -167,10 +168,11 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` | 上游 401 / 403 | 凭证级认证熔断；在 WebUI 检查并重新登录 | | 429 | 对该凭证的上游模型冷却，后续请求自动换绑；全部候选都在冷却时仍返回 429。设了 `--failover-max` 时，当前请求就地换凭证重放 | | 上游 `service info not found`（11102） | 该后端根本不服务这个模型：按 (后端, 模型) 避让，把该模型派给其他后端，全部后端都没有时返回 404。6 小时后半开放行重试，反复命中最长退避 24 小时，一次成功调用即刻解除；可用 `GET /admin/model-blocks` 查看 | -| 建连失败、写请求体超时 | `ConnectError` / `ConnectTimeout` / `WriteTimeout` 换新连接重放一次：三者都意味着上游没收下请求体，重放不会重复计费 | +| 建连失败 | `ConnectError` / `ConnectTimeout` 换新连接重放一次：两者都发生在写下第一个正文字节之前，上游手里什么都没有，重放不会重复计费 | | 发送后断连、读超时、协议错误 | 不做网络重放，避免重复计费；日志记录异常类型与耗时 | | 流式在第一个字节之前失败 | 按**真实状态码**返回，与 `stream=false` 同口径。只带一个流内 `error` 事件的 200 会被客户端读成「模型答了个空」，会话静默结束，审计里还记成一次成功 | -| 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败落在「一个字节都没发给下游」之前时换一个凭证重打，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。只重放上游用 HTTP 状态码给出的拒绝（401/403/429/502/503/504）与没收下请求体的传输失败：内容审核拒绝、上游已回 200 之后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码。**计费口径**：401/403/429/503 与上述三类传输失败都发生在受理阶段，不会扣费；502/504 有可能已被后端处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——只是把一次已经付费的请求换成一段断掉的会话。这类重放在日志里单独标注「上游可能已处理该请求」，便于按官方用量明细核对 | +| 换凭证重放（`--failover-max`） | 默认关闭。开启后，失败落在「一个字节都没发给下游」之前时换一个凭证重打，最多 N 次，审计记为 `success` 并留下 `failover_recovered` 尝试标记。只重放上游用 HTTP 状态码给出的拒绝（401/403/429/502/503/504）与确定没开始收正文的传输失败（建连失败/建连超时）：内容审核拒绝、上游已回 200 之后合成的 502、读超时与协议错误一律不重放；换不出其他凭证时如实回第一次的状态码。**计费口径**：401/403/429/503 与建连类失败都发生在受理阶段，不会扣费；502/504 有可能已被后端处理并计费，但那次结果对下游根本拿不到，不重放也退不回额度——只是把一次已经付费的请求换成一段断掉的会话。这类重放在日志里单独标注「上游可能已处理该请求」，便于按官方用量明细核对 | +| 写超时重放（`--retry-write-timeout`） | 默认关闭。写超时只能证明声明的正文没发完，不能证明上游忽略了已经收到的那部分，因此它默认既不参与连接重试也不参与换凭证重放；显式打开后两层都生效。跨境长会话最容易撞的恰恰是 60s 写超时（实测某部署传输失败 100% 是它），确认自己的上游不会按半截正文计费再开；这类重放同样带「上游可能已处理该请求」日志标记 | | 工具参数损坏 | 聚合校验失败按 `--tool-call-max-retry`（默认 3）额外生成，可能消耗更多额度；被丢弃的生成带用量记入尝试明细；耗尽后返回错误 | | 上游空流或残流 | 没有有效输出、缺少结束标记或包含错误的流不伪装为成功 | | 内容审核拒绝 | 脱敏 + `--no-compact` 下，仅完整非流式纯拒绝且模板确实缩短时，最多同账号兜底一次；流式不做审核重试，也不因此熔断或切号 | diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py index d072389..d04bfb1 100644 --- a/tests/test_stream_failover.py +++ b/tests/test_stream_failover.py @@ -33,8 +33,10 @@ REPLAYABLE_STATUS = (401, 403, 429, 502, 503, 504) DETERMINISTIC_STATUS = (400, 404, 405, 413, 422) -# 上游没收下请求体：换连接/换账号重放安全 -REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) +# 上游手里没有任何正文：换连接/换账号重放安全 +REPLAYABLE_TRANSPORT = (httpx.ConnectError, httpx.ConnectTimeout) +# 写超时：正文没写完是确定的，是否已按半截正文计费看不到，只有显式 opt-in 才参与重放 +WRITE_TIMEOUT_TRANSPORT = (httpx.WriteTimeout,) # 请求体已经发出去了（甚至响应已经开始）：上游可能已处理并计费，禁止重放 AMBIGUOUS_TRANSPORT = (httpx.ReadError, httpx.ReadTimeout, httpx.WriteError, httpx.RemoteProtocolError) @@ -308,6 +310,73 @@ def test_transport_before_request_body_fails_over(self): self.assertNotEqual(len(set(self.uids(sent))), 1, "必须换过凭证") self.assertEqual(len(self.failover_lines()), 1, self.failover_lines()) + def test_write_timeout_needs_an_explicit_opt_in(self): + """默认：写超时按歧义处理——如实回 502，一次都不多重放。""" + self.assertEqual(converter.CONFIG["retry_write_timeout"], False, "默认必须关闭") + for error_type in WRITE_TIMEOUT_TRANSPORT: + with self.subTest(error=error_type.__name__): + self.fresh_pool() + self.requests.clear() + self.logs.clear() + with allow_failover(2): + self.poison_with_transport(error_type) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 502, response.text) + self.assertEqual(len(sent), 1, "未开启 opt-in 时禁止重放") + self.assertEqual(self.failover_lines(), []) + + def test_write_timeout_opt_in_replays_and_flags_the_billing_risk(self): + """开启 `--retry-write-timeout`：重放救回会话，但必须在日志里标出计费歧义。""" + for error_type in WRITE_TIMEOUT_TRANSPORT: + with self.subTest(error=error_type.__name__): + self.fresh_pool() + self.requests.clear() + self.logs.clear() + with allow_failover(1), patch.dict(converter.CONFIG, {"retry_write_timeout": True}): + self.poison_with_transport(error_type) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("data: [DONE]", response.text) + self.assertGreaterEqual(len(sent), 2, "应当重放过") + lines = self.failover_lines() + self.assertTrue(lines, "重放必须留日志") + self.assertIn("上游可能已处理该请求", lines[0], lines[0]) + + def test_write_timeout_opt_in_does_not_replay_ambiguous_transport(self): + """开关只管写超时：读超时/中途 reset 这些歧义失败照旧禁止重放。""" + with patch.dict(converter.CONFIG, {"retry_write_timeout": True}): + for error_type in AMBIGUOUS_TRANSPORT: + with self.subTest(error=error_type.__name__): + self.fresh_pool() + self.requests.clear() + with allow_failover(2): + self.poison_with_transport(error_type) + response, sent = self.stream_post() + self.assertEqual(response.status_code, 502, response.text) + self.assertEqual(len(sent), 1) + self.assertEqual(self.failover_lines(), []) + + def test_failover_switches_are_hot_public_settings(self): + """两个开关都必须能在大控制台「系统设置」里改，且不需要重启进程。""" + from app import settings + + self.assertEqual(converter.CONFIG["failover_max"], 0) + self.assertEqual(converter.CONFIG["retry_write_timeout"], False) + for key, value in (("failover_max", 2), ("retry_write_timeout", True)): + with self.subTest(key=key): + spec = settings.SCHEMA[key] + self.assertEqual(spec["mode"], "hot", f"{key} 改了要能立即生效，不能要求重启") + self.assertFalse(spec["sensitive"], "运维开关必须对管理台可见") + self.assertEqual(settings.validate_settings({key: value}), {key: value}) + listed = {item["key"]: item for item in + settings.resolve_settings({"failover_max": 2, "retry_write_timeout": True})} + self.assertIn(key, listed) + self.assertFalse(listed[key]["locked"], listed[key]) + with self.assertRaises(ValueError): + settings.validate_settings({"failover_max": 99}) # 上界 10 + with self.assertRaises(ValueError): + settings.validate_settings({"retry_write_timeout": "yes"}) + def test_ambiguous_transport_never_fails_over(self): for error_type in AMBIGUOUS_TRANSPORT: with self.subTest(error=error_type.__name__): diff --git a/tests/test_stream_status_contract.py b/tests/test_stream_status_contract.py index 53fd957..c542bfb 100644 --- a/tests/test_stream_status_contract.py +++ b/tests/test_stream_status_contract.py @@ -26,10 +26,11 @@ ROUTES = ("/v1/chat/completions", "/v1/responses", "/v1/messages") TOOLS = [{"type": "function", "function": {"name": "synthetic_tool", "parameters": {"type": "object"}}}] -# 写超时/建连失败时上游还没收下请求体，open_backend_stream 会换新连接重放一次； -# 中途 reset 与协议错误属于「歧义请求」，绝不重放 POST。 -REPLAYABLE = (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout) -AMBIGUOUS = (httpx.ReadError, httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.WriteError) +# 建连失败/建连超时：上游手里没有正文，open_backend_stream 会换新连接重放一次； +# 中途 reset、协议错误、以及写超时（正文没写完 ≠ 上游没处理）都属于歧义请求，默认不重放。 +REPLAYABLE = (httpx.ConnectError, httpx.ConnectTimeout) +AMBIGUOUS = (httpx.ReadError, httpx.ReadTimeout, httpx.RemoteProtocolError, httpx.WriteError, + httpx.WriteTimeout) HTTP_STATUSES = (400, 403, 429, 503) TERMINALS = ("data: [DONE]", '"response.completed"', '"message_stop"') diff --git a/tests/test_upstream_io.py b/tests/test_upstream_io.py index 14ddf99..e432962 100644 --- a/tests/test_upstream_io.py +++ b/tests/test_upstream_io.py @@ -317,9 +317,9 @@ def handler(request): self.assertEqual(requests[0].method, "POST") async def test_body_not_accepted_is_replayed_on_a_fresh_connection(self): - """建连失败 / 写请求体超时：上游没收下请求体，换新连接重放一次不会重复计费。""" + """建连失败 / 建连超时：上游手里没有正文，换新连接重放一次不会重复计费。""" real_client = httpx.AsyncClient - for error_type in (httpx.ConnectError, httpx.ConnectTimeout, httpx.WriteTimeout): + for error_type in (httpx.ConnectError, httpx.ConnectTimeout): with self.subTest(error=error_type.__name__): attempts = [] @@ -338,6 +338,48 @@ def handler(request): self.assertEqual(len(attempts), 2, "只允许重放一次") self.assertEqual([request.method for request in attempts], ["POST", "POST"]) + async def test_write_timeout_is_not_replayed_without_the_opt_in(self): + """写超时只证明正文没写完，证不了上游没处理已收到的部分：默认不重放。""" + real_client = httpx.AsyncClient + for error_type in upstream_io.WRITE_TIMEOUT: + with self.subTest(error=error_type.__name__): + attempts = [] + + def handler(request): + attempts.append(request) + raise error_type("synthetic write timeout") + + transport = httpx.MockTransport(handler) + with patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw)): + with self.assertRaises(error_type): + async with upstream_io.open_backend_stream("https://synthetic.invalid", {}, {}) as response: + await response.aread() + self.assertEqual(len(attempts), 1, "默认必须一次都不重放") + + async def test_write_timeout_is_replayed_only_when_opted_in(self): + """`retry_write_timeout=True` 是运维显式承担计费歧义，重放行为与建连失败一致。""" + real_client = httpx.AsyncClient + for error_type in upstream_io.WRITE_TIMEOUT: + with self.subTest(error=error_type.__name__): + attempts = [] + + def handler(request): + attempts.append(request) + if len(attempts) == 1: + raise error_type("synthetic write timeout") + return httpx.Response(200, content=b"data: [DONE]\n\n") + + transport = httpx.MockTransport(handler) + with patch.object(upstream_io.httpx, "AsyncClient", + side_effect=lambda **kw: real_client(transport=transport, **kw)): + async with upstream_io.open_backend_stream( + "https://synthetic.invalid", {}, {}, retry_write_timeout=True) as response: + self.assertEqual(response.status_code, 200) + self.assertEqual(await response.aread(), b"data: [DONE]\n\n") + self.assertEqual(len(attempts), 2, "只允许重放一次") + self.assertEqual([request.method for request in attempts], ["POST", "POST"]) + async def test_ambiguous_transport_failures_never_replay_post(self): """请求体已经发出（甚至响应已经开始）的失败有计费歧义，一律交给调用方按协议返回。""" real_client = httpx.AsyncClient From a7df12d3bf5dde375f4b90dc7e7f22524969ac73 Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:46:02 +0000 Subject: [PATCH 4/8] Cover the failover switches through the console settings API The two knobs are only useful if an operator can flip them while traffic is running, so pin the whole path: listed as hot and unlocked, defaults still match the upstream behaviour, PATCH writes through to CONFIG and persists, and the gateway is told to apply them without a restart. --- tests/test_admin_api.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_admin_api.py b/tests/test_admin_api.py index 2317f15..bfb4dda 100644 --- a/tests/test_admin_api.py +++ b/tests/test_admin_api.py @@ -336,6 +336,28 @@ def test_tool_metadata_setting_is_hot_persisted_and_respects_locks(self): self.assertIs(self.config[key], True) self.assertIs(self.store.snapshot()["settings"][key], True) + def test_failover_settings_are_hot_and_persisted(self): + """换凭证重放与写超时开关都要能在系统设置里改完立即生效，不需要重启进程。""" + current = self.client.get("/admin/settings", headers=self.headers).json() + items = {item["key"]: item for item in current["items"]} + for key, kind, default in (("failover_max", "integer", 0), ("retry_write_timeout", "boolean", False)): + with self.subTest(key=key): + spec = items[key] + self.assertEqual((spec["mode"], spec["locked"], spec["type"]), ("hot", False, kind)) + self.assertEqual(spec["value"], default, "默认必须与上游行为一致：关闭") + response = self.client.patch("/admin/settings", headers=self.headers, json={ + "revision": current["revision"], "values": {"failover_max": 2, "retry_write_timeout": True}}) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual((self.config["failover_max"], self.config["retry_write_timeout"]), (2, True)) + self.assertEqual(self.store.snapshot()["settings"]["failover_max"], 2) + self.assertIs(self.store.snapshot()["settings"]["retry_write_timeout"], True) + self.gateway.admin_apply_settings.assert_called_once_with( + {"failover_max": 2, "retry_write_timeout": True}) + applied = {item["key"]: item for item in response.json()["items"]} + for key, value in (("failover_max", 2), ("retry_write_timeout", True)): + self.assertEqual((applied[key]["value"], applied[key]["source"], applied[key]["locked"]), + (value, "management", False), applied[key]) + def test_settings_revision_locked_sources_and_secret_redaction(self): self.config["auth_dir"] = "/private-directory" self.config["settings_sources"] = {"max_images": "environment"} From d045bf125eb74441e00324bfb4c0893a20d23797 Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 04:26:21 +0000 Subject: [PATCH 5/8] Keep replays on the client's model, and make the preflight cancellable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the re-review. Each comes with a regression that is red on the previous commit and green on this one. - Re-route with the client's model, not the rewritten one. `_route_chat` rewrites `auto` into the site's `default-model` before sending, and the failover loop fed that rewritten body straight back into `_route_chat`. The second round then looked up the policy for `default-model`, so a rule binding `auto` to one credential or one site stopped applying exactly when the gateway starts choosing a new backend. `_routed_stream`/`_routed_fetch` now take the canonical (pre-rewrite) body separately from the per-round wire body, and all three endpoints pass what the client actually asked for. - Do the preflight inside the ASGI lifecycle. It was awaited in the endpoint, i.e. before `StreamingResponse.__call__` installs `listen_for_disconnect`, so across that whole window nobody consumed `http.disconnect`: a stuck first segment plus a client that already left held a `ConcurrencyLimitMiddleware` slot until the 300s read timeout -- at `max_concurrent=1` one stranded client took the whole gateway down. Streaming replies are now a `StreamingResponse` subclass that runs the preflight (and any credential failover) inside `stream_response`, so the disconnect cancels the pending upstream read while the response headers stay deferred. Failures still raise `HTTPException` from inside that call, below `ExceptionMiddleware`, so the status-code contract and the protocol error bodies are unchanged. Getting the cancellation to land is not enough on its own: anyio re-delivers it at every checkpoint, which aborts the generator's own `finally` -- measured directly, the httpx teardown there never finished. Each segment is therefore read in a child task (`_first_segment`), so the frame is never unwound by the outer scope, and the close runs in a shielded scope. Covered at the ASGI level with the real gate layer: disconnect before the first segment, disconnect after streaming started, and "the next request still gets a slot". - Tag the same-credential write replay. `--retry-write-timeout` applies to the connection retry as well, but only the credential-switch log line carried `上游可能已处理该请求`, so a first WriteTimeout followed by a same-credential success was a possibly-billed replay with no marker -- the docs already promised the tag for both layers. The retry callback now separates the two classes, logs the note for write timeouts, and records a `write_timeout_retry` attempt stage so the audit says which kind of replay it was. anyio becomes a declared direct dependency: same 4.14.2 that starlette/httpx already pinned, so the locked hash line is unchanged and only the `via` block moves. 781 passed, 2724 subtests. --- converter.py | 183 ++++++++++++++++++++++----- requirements.in | 1 + requirements.txt | 1 + tests/test_stream_failover.py | 121 ++++++++++++++++++ tests/test_stream_status_contract.py | 140 ++++++++++++++++++++ 5 files changed, 414 insertions(+), 32 deletions(-) diff --git a/converter.py b/converter.py index 89b8ab1..41ad6fb 100644 --- a/converter.py +++ b/converter.py @@ -23,6 +23,7 @@ from __future__ import annotations import argparse +import asyncio import hashlib import json import os @@ -39,6 +40,7 @@ from pathlib import Path from typing import Optional +import anyio # 断连收尾要屏蔽外层取消：用的正是发起取消的那套机制 import httpx from fastapi import FastAPI, Header, HTTPException, Request from fastapi.exception_handlers import http_exception_handler as _default_http_exception_handler @@ -2345,6 +2347,7 @@ async def chat_completions(request: Request, + (f" | tools={tool_names}" if tool_names else "") + (f" | last_user={_truncate(last_user, 60)!r}" if last_user else "")) # 凭据选择/到期刷新持线程锁与文件锁并可能同步访问网络：放到受限线程池，不占事件循环 + prepared = body # 改写前的规范请求体，换凭证重放按它判定绑定 body, cred, headers, url = await run_in_threadpool(_route_chat, payload, body, rid) _log_json(f"[{rid}] REQUEST BODY (发往后端，预览)", body) t0 = time.time() @@ -2352,14 +2355,14 @@ async def chat_completions(request: Request, if client_wants_stream: def attempt(routed, cred, headers, url): return _stream_upstream(url, headers, routed, model_name, t0, rid, cred=cred) - return await _routed_stream(payload, body, model_name, rid, t0, attempt, - body, cred, headers, url) + return _routed_stream(payload, prepared, model_name, rid, t0, attempt, + body, cred, headers, url) # 非流式：后端只支持流式，这里把后端 SSE 聚合成单个 chat.completion 响应 async def fetch(routed, cred, headers, url): return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, filter_retry=True) - collected = await _routed_fetch(payload, body, model_name, rid, t0, fetch, + collected = await _routed_fetch(payload, prepared, model_name, rid, t0, fetch, body, cred, headers, url) _log_finish(model_name, t0, collected, rid) if CONFIG.get("control_store") is not None: @@ -2538,9 +2541,18 @@ def _public_sse_line(line, model_name): async def _backend_stream(url, headers, body, *, timeout=300, rid="", model_name="?"): started, opened = time.monotonic(), False def retry(error): - observe_attempt("connect_retry", error_code=type(error).__name__, + """同一连接上的底层重放：换凭证那条日志到不了这里，风险标记得自己带上。 + + 建连失败/建连超时上游手里没有正文，标出来反而是噪音；写超时按 opt-in 参与重放时， + 「正文没写完」证不了上游没动过账，所以必须和换凭证重放同一口径标注（评审 P2）。 + `stage` 分开记，审计里能一眼看出是哪一类重放。 + """ + timeout_on_write = isinstance(error, WRITE_TIMEOUT_TRANSPORT) + observe_attempt("write_timeout_retry" if timeout_on_write else "connect_retry", + error_code=type(error).__name__, duration_ms=(time.monotonic() - started) * 1000) - _log(f"[{rid}] 建连失败，重试 1/1 | {model_name} | {_network_error_text(error)}") + _log(f"[{rid}] {'写超时重放' if timeout_on_write else '建连失败'}，重试 1/1 | {model_name}" + f" | {_network_error_text(error)}{_replay_cost_note(error)}") try: async with open_backend_stream(url, headers, body, read_timeout=timeout, on_retry=retry, retry_write_timeout=bool(CONFIG.get("retry_write_timeout"))) as response: @@ -2773,6 +2785,40 @@ def __init__(self, status, raw, error=None): super().__init__(f"stream failed before first byte (HTTP {status})") +async def _first_segment(agen): + """取生成器的第一段输出，但把「我们的等待」和「生成器自己的收尾」分开放。 + + 直接在当前任务里 `await agen.__anext__()` 也有问题：下游断连时取消打在生成器帧内部的 + await 上，而 anyio 的取消作用域会在每个检查点重复取消 —— 帧自己的 `finally` 做到一半就 + 被打断（`httpx` 正是在这里关连接），实测那段清理根本跑不完，连接留到读超时。放进子任务 + 之后，外层取消打断的是我们的 `await`，我们只取消子任务一次，再屏蔽着等它把 `finally` + 走完。结果不取：取消语义下这一轮已经作废，交给调用方关流。 + """ + task = asyncio.ensure_future(agen.__anext__()) + try: + return await asyncio.shield(task) + except BaseException: + task.cancel() + with anyio.CancelScope(shield=True): + await asyncio.wait([task]) + if not task.cancelled(): + task.exception() # 取回异常，别留给 asyncio 报「never retrieved」 + raise + + +async def _stream_segments(agen): + """逐段读上游，语义等同 `async for chunk in agen`，但每一段都可被干净打断。 + + 复用 `_first_segment`：断连落在「两段之间」还是「正等下一段」都无所谓，生成器自己的 + `finally` 都能走完。 + """ + while True: + try: + yield await _first_segment(agen) + except StopAsyncIteration: + return + + async def _preflight_stream(agen, model_name, t0, rid): """取到第一段输出之后再决定怎么回 200。 @@ -2784,7 +2830,7 @@ async def _preflight_stream(agen, model_name, t0, rid): （那时状态码已经收不回来了）。 """ try: - return await agen.__anext__() + return await _first_segment(agen) except StopAsyncIteration: empty = UpstreamResponseError(502, b'{"error":{"message":"upstream returned an empty stream",' b'"type":"upstream_error","code":"empty_response"}}') @@ -2795,35 +2841,84 @@ async def _preflight_stream(agen, model_name, t0, rid): raise _StreamFailure(status, raw, error) from None -def _stream_response(agen, first): - """把预取到的第一段接回流，之后保持原有生成器语义。""" - async def body(): - yield first - async for chunk in agen: - yield chunk - return StreamingResponse(body(), media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) +async def _close_stream(agen) -> None: + """显式收尾上游生成器，并屏蔽外层取消。 + + 下游断连时取消正在反复投递（anyio 的取消作用域会在每个检查点再取消一次），不屏蔽的话 + `httpx` 的关闭做到一半就被打断，上游连接和它的读超时一起留在原地。清理失败不改变已经 + 定型的响应，所以这里只吞掉异常，不吞掉取消。 + """ + if agen is None: + return + with anyio.CancelScope(shield=True): + try: + await agen.aclose() + except Exception: + pass + + +def _chunk_bytes(chunk, charset: str = "utf-8"): + return chunk if isinstance(chunk, (bytes, memoryview)) else chunk.encode(charset) + + +class _DeferredStreamResponse(StreamingResponse): + """把「预取第一段 + 必要的换凭证重放」放进 ASGI 生命周期里做的流式响应。 + + 预取不能就在端点里 `await`：`StreamingResponse.__call__` 是把 `stream_response` 和 + `listen_for_disconnect` 放进同一个任务组跑的，端点返回之前根本没有谁在消费 + `http.disconnect`。上游首段一旦卡住而客户端已经走了，这个 await 会一直挂到读超时， + `ConcurrencyLimitMiddleware` 的名额也跟着占满 —— 表现为整个网关 503。搬进 + `stream_response` 之后，断连取消的就是我们此刻的 await，挂起的上游读被打断，生成器的 + finally 跑得完，名额立刻归还。 + + 响应头仍然等到确实有字节可发时才发出，所以「把失败还原成真实状态码」的能力不受影响： + 失败以 `HTTPException` 抛出，由 ExceptionMiddleware 成形（`/v1/*` 走协议化错误体）， + 那一刻一个字节都还没出去。客户端中途断连则按普通流式断连处理 —— 取消穿出 `__call__`， + 和响应已经开始之后的行为一致；两种窗口里的读取都走 `_first_segment`， + 取消之后生成器的收尾仍然跑得完。 + """ + + def __init__(self, plan): + self._plan = plan # async callable -> (上游生成器, 已预取的第一段) + super().__init__(content=(), media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + async def stream_response(self, send) -> None: + agen, first = await self._plan() + try: + await send({"type": "http.response.start", "status": self.status_code, + "headers": self.raw_headers}) + await send({"type": "http.response.body", "body": _chunk_bytes(first, self.charset), + "more_body": True}) + async for chunk in _stream_segments(agen): + await send({"type": "http.response.body", "body": _chunk_bytes(chunk, self.charset), + "more_body": True}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + finally: + await _close_stream(agen) def _failover_limit() -> int: return int(CONFIG.get("failover_max") or 0) -async def _routed_stream(payload, body, model_name, rid, t0, make, routed, cred, headers, url): - """流式端点：预取失败时按策略换凭证重打，全部失败才把真实状态码回给下游。 +async def _stream_plan(payload, canonical, model_name, rid, t0, make, routed, cred, headers, url): + """预取第一段，失败就按策略换凭证重打；返回 (生成器, 首段)，全线失败才抛 `HTTPException`。 重放只发生在「一个字节都没发给下游」的时候（`_preflight_stream` 保证了这点），所以下游 看到的仍然是一次正常请求。`make(routed, cred, headers, url)` 每轮只建一个生成器。 + + `canonical` 与 `routed` 必须分开：`_route_chat` 会把逻辑模型（`auto`）改写成该站点的 + 默认模型再发出去，所以 `routed` 是「本轮的真实报文」，而重路由只能拿改写前的 `canonical` + 去问绑定规则 —— 否则第二轮查的是默认模型，客户端原来说的 `auto` 的账号/站点限制就丢了。 """ tried = [] while True: stream = make(routed, cred, headers, url) try: first = await _preflight_stream(stream, model_name, t0, rid) - if tried: - observe_recovery() # 重放救回来的请求对下游是正常响应，不该记成失败 - return _stream_response(stream, first) except _StreamFailure as failure: + await _close_stream(stream) # 本轮的上游已经终止，关掉只是兜底，不留半开的连接 tried.append(cred) limit = _failover_limit() surface = HTTPException(status_code=failure.status, @@ -2831,7 +2926,7 @@ async def _routed_stream(payload, body, model_name, rid, t0, make, routed, cred, if limit <= 0 or len(tried) > limit or not _failover_safe(failure.error, failure.raw): raise surface from None try: - attempt = await run_in_threadpool(_route_chat, payload, body, rid, + attempt = await run_in_threadpool(_route_chat, payload, canonical, rid, tried={_cred_manager(item) for item in tried}) except HTTPException: raise surface from None # 换不出别的凭证，就如实回第一次的错 @@ -2841,10 +2936,32 @@ async def _routed_stream(payload, body, model_name, rid, t0, make, routed, cred, _log(f"[{rid}] ↻ 换凭证重放 {len(tried)}/{limit} | {model_name} | 上游 HTTP " f"{failure.status} → {profile_for_headers(headers)}" f"{_replay_cost_note(failure.error)}") + continue # 换一个凭证，再预取一次 + except BaseException: + # 下游断连（取消）或没预料到的错误：先把本轮上游收掉，再把异常原样交出去 + await _close_stream(stream) + raise + if tried: + observe_recovery() # 重放救回来的请求对下游是正常响应，不该记成失败 + return stream, first + +def _routed_stream(payload, canonical, model_name, rid, t0, make, routed, cred, headers, url): + """流式端点入口：返回一个把预取与重放留待 ASGI 生命周期内执行的响应。 -async def _routed_fetch(payload, body, model_name, rid, t0, fetch, routed, cred, headers, url): - """非流式请求：失败时按同一策略换凭证重打（此时一个字节都还没回给下游）。""" + 这里刻意「什么都不做就返回」：预取必须发生在 `_DeferredStreamResponse.stream_response` + 里，那里才有下游断连监听（见该类的说明）。 + """ + return _DeferredStreamResponse( + lambda: _stream_plan(payload, canonical, model_name, rid, t0, make, + routed, cred, headers, url)) + + +async def _routed_fetch(payload, canonical, model_name, rid, t0, fetch, routed, cred, headers, url): + """非流式请求：失败时按同一策略换凭证重打（此时一个字节都还没回给下游）。 + + `canonical` 同 `_routed_stream`：重路由用改写前的规范请求体，判定才落在客户端模型上。 + """ tried = [] while True: try: @@ -2860,7 +2977,7 @@ async def _routed_fetch(payload, body, model_name, rid, t0, fetch, routed, cred, if limit <= 0 or len(tried) > limit or not _failover_safe(error, raw): raise surface from None try: - attempt = await run_in_threadpool(_route_chat, payload, body, rid, + attempt = await run_in_threadpool(_route_chat, payload, canonical, rid, tried={_cred_manager(item) for item in tried}) except HTTPException: raise surface from None @@ -2948,6 +3065,7 @@ async def create_response(request: Request, f"| anchor_user={projection_stats.get('anchor_user_preserved', False)}" ) # 同上：凭据选择/刷新是阻塞操作，移出事件循环 + prepared = chat_body # 改写前的规范请求体，见 `_routed_stream` chat_body, cred, headers, url = await run_in_threadpool(_route_chat, payload, chat_body, rid) _log_json(f"[{rid}] RESPONSES → CHAT BODY (预览)", chat_body) t0 = time.time() @@ -2955,23 +3073,23 @@ async def create_response(request: Request, if client_wants_stream: def attempt(routed, cred, headers, url): return _stream_responses(url, headers, routed, model_name, t0, rid, cred=cred) - return await _routed_stream(payload, chat_body, model_name, rid, t0, attempt, - chat_body, cred, headers, url) + return _routed_stream(payload, prepared, model_name, rid, t0, attempt, + chat_body, cred, headers, url) return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, - payload=payload) + payload=payload, canonical=prepared) async def _nonstream_adapted(url, headers, body, model_name, t0, rid, cred, *, anthropic=False, - payload=None): + payload=None, canonical=None): converter = (AnthropicStreamConverter(model=model_name) if anthropic else ResponsesStreamConverter(model=model_name, parallel_tool_calls=body.get("parallel_tool_calls", True))) async def fetch(routed, cred, headers, url): return await _fetch_checked_chat(url, headers, routed, model_name, rid, cred, filter_retry=True) try: - collected = await _routed_fetch(payload, body, model_name, rid, t0, fetch, - body, cred, headers, url) + collected = await _routed_fetch(payload, body if canonical is None else canonical, + model_name, rid, t0, fetch, body, cred, headers, url) for line in _chat_result_to_sse_lines(_completion_to_merged(collected)): converter.feed_line(_public_sse_line(line, model_name)) converter.finish() @@ -3054,18 +3172,19 @@ async def create_message(request: Request, rid = os.urandom(4).hex() _log(f"[{rid}] ▶ ANTHROPIC {model_name} | msgs={len(chat_messages)} | anthropic_msgs={len(messages)}") # 同上：凭据选择/刷新是阻塞操作，移出事件循环 + prepared = chat_body # 改写前的规范请求体，见 `_routed_stream` chat_body, cred, headers, url = await run_in_threadpool(_route_chat, payload, chat_body, rid) _log_json(f"[{rid}] ANTHROPIC → CHAT BODY (预览)", chat_body) t0 = time.time() if not _client_wants_stream(payload): return await _nonstream_adapted(url, headers, chat_body, model_name, t0, rid, cred, - anthropic=True, payload=payload) + anthropic=True, payload=payload, canonical=prepared) def attempt(routed, cred, headers, url): return _stream_anthropic(url, headers, routed, model_name, t0, rid, cred=cred) - return await _routed_stream(payload, chat_body, model_name, rid, t0, attempt, - chat_body, cred, headers, url) + return _routed_stream(payload, prepared, model_name, rid, t0, attempt, + chat_body, cred, headers, url) async def _stream_anthropic(url: str, headers: dict, body: dict, diff --git a/requirements.in b/requirements.in index e2305ba..c481ec7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,6 @@ # Direct dependencies; requirements.txt is the generated, hash-locked install file. fastapi +anyio # 取消屏蔽：下游断连时仍要把上游连接的收尾做完 uvicorn[standard] httpx pytest diff --git a/requirements.txt b/requirements.txt index 930c914..e09ff06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ anyio==4.14.2 \ --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f # via + # -r requirements.in # httpx # starlette # watchfiles diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py index d04bfb1..27d0f41 100644 --- a/tests/test_stream_failover.py +++ b/tests/test_stream_failover.py @@ -62,6 +62,7 @@ def setUp(self): self.arm_next = False self.stream = True self.poison_uid = None + self.poison_once = False self.poison = lambda request: httpx.Response(429, json=error_body()) self.logs = [] self.enterContext(patch.object(converter, "_log", side_effect=self.capture_log)) @@ -82,6 +83,8 @@ def handle_upstream(self, request): self.poison_uid = uid # 不依赖轮询顺序：下一个被选中的凭证开始失败 if self.poison_uid is not None and uid == self.poison_uid: self.requests.append(request) + if self.poison_once: + self.poison_uid = None # 只掐第一枪：后面那一枪是同一凭证上的重放 return self.poison(request) # 可以返回错误状态，也可以直接抛传输层异常 return super().handle_upstream(request) @@ -99,6 +102,11 @@ def raise_transport(request): raise error_type("synthetic transport failure") self.poison = raise_transport + def poison_transport_once(self, error_type): + """只让第一枪失败：测「同一连接/同一凭证」的底层重放，换凭证那条日志压根到不了。""" + self.poison_with_transport(error_type) + self.poison_once = True + def stream_post(self, endpoint="chat/completions"): self.allowed_profiles = set(fixtures.PROFILES) before = len(self.requests) @@ -198,6 +206,119 @@ def test_reroute_cannot_loop_back_to_the_same_credential(self): self.assertEqual(len(set(self.uids(self.requests))), len(self.requests), "同一凭证不得被打两次") + # --- 重路由必须沿用客户端请求的模型，不得被上游改写名绕过 --- + AUTO_PROFILES = ("intl-work", "intl-cli") + + def arm_auto(self, profiles=None): + """账号目录都含 default-model：国际站会把 `auto` 改写成它，国内站不会 —— 所以放宽只有 + 在「改写后的名字」上查规则才会发生，站点绑定那条用例要靠国内站账号当靶子。""" + self.auto_profiles = tuple(profiles or self.AUTO_PROFILES) + self.configure(profiles=self.auto_profiles, + tables={profile: [fixtures.model("default-model")] for profile in self.auto_profiles}) + + def bind_auto(self, name, **rule): + """建一个管理库并给 `auto` 下一条路由策略。""" + from app import model_policy + from app.control_store import ControlStore + + store = ControlStore(self.root / name) + self.addCleanup(store.close) + converter.CONFIG["control_store"] = store + store.update_model("auto", dict(model_policy.default_rule("auto"), **rule), 0) + return store + + def auto_post(self): + """发一次 model=auto 的流式请求，返回下游响应与真正打出去的上游请求。""" + self.allowed_profiles = set(self.auto_profiles) + before = len(self.requests) + response = self.client.post("/v1/chat/completions", + json=self.payload(stream=True, selected_model="auto")) + return response, self.requests[before:] + + def test_auto_is_rewritten_on_the_international_site(self): + """夹具自检：国际站确实把 auto 发成了 default-model，否则下面几条等于没测。""" + self.arm_auto() + response, sent = self.auto_post() + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(sent), 1) + import json as _json + self.assertEqual(_json.loads(sent[0].content)["model"], "default-model") + + def test_failover_cannot_widen_the_credential_binding(self): + """把 `auto` 只绑到 A：A 失败后不许放宽给 B。 + + `_route_chat` 会把 auto 改写成 default-model 再发出去；若拿改写后的正文重新选 + 凭证，策略查的是 default-model（没有规则），绑定在 auto 上的限制整个失效。 + """ + self.arm_auto() + self.bind_auto("bind.sqlite3", credential_ids=[self.entries["intl-work"]["account_key"]]) + with allow_failover(1): + self.poison_with_status(503) + response, sent = self.auto_post() + self.assertEqual(response.status_code, 503, response.text) + self.assertEqual(len(sent), 1, + f"绑定 auto 的账号失败后不得放宽给别的账号：{self.uids(sent)}") + + def test_failover_cannot_widen_the_site_binding(self): + """同上，按站点绑定：`auto` 限定 intl 时，重放不许跑到国内站。""" + self.arm_auto(("intl-work", "cn-work")) + self.bind_auto("region.sqlite3", region="intl") + with allow_failover(3): + self.poison_with_status(503) + response, sent = self.auto_post() + self.assertEqual(response.status_code, 503, response.text) + self.assertEqual(len(sent), 1, f"站点绑定被放宽：{self.uids(sent)}") + + def test_reroute_uses_the_pristine_body_model(self): + """直接钉住重路由入参：每一轮选凭证看的都必须是客户端请求的模型名。""" + self.arm_auto() + self.bind_auto("spy.sqlite3", region="intl") + seen = [] + real = converter._route_chat + + def spy(payload, body, rid, **kwargs): + seen.append(body.get("model")) + return real(payload, body, rid, **kwargs) + + with allow_failover(1), patch.object(converter, "_route_chat", side_effect=spy): + self.poison_with_status(503) + response, sent = self.auto_post() + self.assertEqual(len(seen), 2, f"应当恰好发生一次重路由：{seen}") + self.assertEqual([name for name in seen if name != "auto"], [], + f"重路由拿到了被改写的模型名：{seen}") + + def test_same_credential_write_timeout_replay_carries_the_risk_note(self): + """评审 P2：底层连接上的写超时重放也必须带代价标记，两层同一口径。 + + 第一次写超时、同一凭证第二次就成 —— 这条路径到不了换凭证那行日志，`failover_max=0` + 时更是完全不经过它，所以标注只能由 `open_backend_stream` 的重试回调自己带上。 + """ + store, client = self.audited_client() + with patch.dict(converter.CONFIG, {"retry_write_timeout": True, "failover_max": 0}): + self.poison_transport_once(httpx.WriteTimeout) + response = client.post("/v1/chat/completions", json=self.payload(stream=True)) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(self.requests), 2, self.uids(self.requests)) + self.assertEqual(len(set(self.uids(self.requests))), 1, + f"底层重放不许换凭证：{self.uids(self.requests)}") + self.assertEqual(self.failover_lines(), [], "不该出现换凭证重放的日志") + lines = [line for line in self.logs if "写超时重放" in line] + self.assertEqual(len(lines), 1, self.logs) + self.assertIn("上游可能已处理该请求", lines[0]) + stages = [attempt.get("stage") for attempt in self.only_record(store)["attempts"]] + self.assertIn("write_timeout_retry", stages, stages) + + def test_connect_retry_stays_untagged(self): + """建连失败不标记风险：上游手里没有正文，重放确定不重复计费。""" + self.audited_client() + with patch.dict(converter.CONFIG, {"failover_max": 0}): + self.poison_transport_once(httpx.ConnectError) + response = self.client.post("/v1/chat/completions", json=self.payload(stream=True)) + self.assertEqual(response.status_code, 200, response.text) + lines = [line for line in self.logs if "建连失败" in line] + self.assertEqual(len(lines), 1, self.logs) + self.assertNotIn("上游可能已处理该请求", lines[0]) + # --- 默认关闭：与上游一致，一次都不多重放 --- def test_disabled_by_default_replays_nothing(self): self.assertEqual(converter.CONFIG["failover_max"], 0, "默认必须关闭，行为与上游一致") diff --git a/tests/test_stream_status_contract.py b/tests/test_stream_status_contract.py index c542bfb..f90beb2 100644 --- a/tests/test_stream_status_contract.py +++ b/tests/test_stream_status_contract.py @@ -14,15 +14,19 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # 仓库根：允许直接运行本文件 +import asyncio import json +import time import unittest from unittest.mock import patch +import anyio import httpx from fastapi.testclient import TestClient import converter from app import upstream_io +from app.inbound_limits import ConcurrencyLimitMiddleware ROUTES = ("/v1/chat/completions", "/v1/responses", "/v1/messages") TOOLS = [{"type": "function", "function": {"name": "synthetic_tool", "parameters": {"type": "object"}}}] @@ -170,5 +174,141 @@ def test_happy_stream_still_delivers_every_event_once(self): self.assertIn("data: [DONE]", response.text) +class PreflightDisconnectTests(unittest.IsolatedAsyncioTestCase): + """预取窗口里的下游断连：取消要打得断挂起的上游读，并发名额当场归还。 + + 用真实的中间件顺序手工驱动 ASGI。评审 P1 的场景：预取曾在端点里直接 `await`，而 + `StreamingResponse.__call__` 是把 `stream_response` 和 `listen_for_disconnect` 放进同一个 + 任务组跑的 —— 端点返回之前没有谁在消费 `http.disconnect`。于是「上游首段卡住 + 客户端已经 + 走了」会一路挂到读超时，`ConcurrencyLimitMiddleware` 的名额跟着陪葬，单并发部署整个网关 + 变 503。这里钉住：断连必须当场收尾（首段之前、以及已经开始流式之后两种），并且下一次请求 + 拿得到名额。 + """ + + PAYLOAD = {"model": "auto", "stream": True, "messages": [{"role": "user", "content": "hi"}]} + + def setUp(self): + self.enterContext(patch.dict(converter.CONFIG, { + "api_key": "", "cred": None, "cred_pool": None, "model_guard": False, + "max_images": 16, "image_policy": "truncate", "max_request_bytes": 32 * 1024 * 1024, + "log_body_limit": 65536, "log_path": None, "desensitize": False, "no_compact": False, + "max_concurrent": 1, "max_collect_bytes": 0})) + self.enterContext(patch.object(converter, "_cred_for", return_value=(None, {}))) + self.enterContext(patch.object(converter, "_log")) + self.enterContext(patch.object(converter, "_note_cred_status")) + self.enterContext(patch.object(converter, "_note_cred_model_ok")) + # 名额层套在真实 app 外面（与 runtime_management.install 的层次一致）。 + # 每次新建实例：信号量挂在中间件实例上，测试之间不能互相借位。 + self.app = ConcurrencyLimitMiddleware(converter.app.build_middleware_stack(), + converter.CONFIG) + self.stuck = asyncio.Event() + self.closed = 0 + self.mode = "before-first-segment" + + async def upstream(self, url, headers, body, model_name="?", t0=0.0, rid="", cred=None): + """假上游：按模式卡在首段之前或之后；收尾一定要 await，跟真实 httpx 一样。""" + try: + if self.mode != "before-first-segment": + yield "data: " + json.dumps({"choices": [{"index": 0, "delta": {"content": "ok"}}], + "model": model_name}) + "\n\n" + self.stuck.set() # 「已经挂在上游上」/「首段已经交出去」的信号 + if self.mode == "done": + yield "data: [DONE]\n\n" + return + await asyncio.Event().wait() + finally: + await asyncio.sleep(0.01) + self.closed += 1 + + async def drive(self, *, disconnect): + """跑一次请求；返回 (响应状态码或 None, 断连后是否在超时内收尾)。""" + sent, queue = [], asyncio.Queue() + + async def receive(): + return await queue.get() + + async def send(message): + sent.append(message) + + scope = {"type": "http", "method": "POST", "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", "query_string": b"", "headers": [], + "scheme": "http", "http_version": "1.1", "server": ("test", 80), + "client": ("127.0.0.1", 1234), "asgi": {"version": "3.0", "spec_version": "2.3"}} + await queue.put({"type": "http.request", "body": json.dumps(self.PAYLOAD).encode(), + "more_body": False}) + with patch.object(converter, "_stream_upstream", new=self.upstream): + task = asyncio.create_task(self.app(scope, receive, send)) + try: + await asyncio.wait_for(self.stuck.wait(), 2) + if self.mode == "after-first-segment": # 等响应头真的发出去 + for _ in range(400): + if any(m["type"] == "http.response.start" for m in sent): + break + await asyncio.sleep(0.005) + if disconnect: + await queue.put({"type": "http.disconnect"}) + await asyncio.wait_for(task, 2) # 收尾不干净就会在这里超时（= 名额被占） + finally: + task.cancel() + start = next((m for m in sent if m["type"] == "http.response.start"), None) + return start["status"] if start else None, sent + + async def test_disconnect_before_first_segment_aborts_without_a_response(self): + """首段还没来就断连：一个字节都不该发出去，上游要当场关掉。""" + status, sent = await self.drive(disconnect=True) + self.assertIsNone(status, f"客户端已经走了， yet 发出了响应头：{sent}") + self.assertEqual(self.closed, 1) + + async def test_slot_is_returned_so_the_next_request_still_runs(self): + """名额归还：断连之后紧接着的请求必须是正常响应，而不是「并发已满」的 503。""" + await self.drive(disconnect=True) + self.stuck = asyncio.Event() + self.mode = "done" + status, _ = await self.drive(disconnect=False) + self.assertEqual(status, 200) + + async def test_disconnect_after_streaming_started_closes_the_upstream(self): + """已经开始流式之后断连：取消照常打到挂起的上游读，生成器被关干净。""" + self.mode = "after-first-segment" + status, sent = await self.drive(disconnect=True) + self.assertEqual(status, 200, sent) + self.assertEqual(self.closed, 1) + + +class ShieldedCloseTests(unittest.IsolatedAsyncioTestCase): + """`_close_stream` 必须扛得住「正在被取消」这件事本身。 + + 断连时 anyio 的取消作用域会在每个检查点重复取消；不屏蔽的话，生成器 finally 里那个 + await（httpx 在这里关连接）做到一半就被打断，连接和它的读超时一起留在原地。实测： + 同样的作用域里不加屏蔽，收尾 await 从来跑不完。 + """ + + async def test_cleanup_await_completes_inside_a_cancelled_scope(self): + done = [] + + async def upstream(): + try: + yield "data: x\n\n" # 停在 yield 上被关：真实场景是「两段之间」 + finally: + await asyncio.sleep(0.01) + done.append("closed") + + agen = upstream() + await agen.__anext__() + + async def worker(): + try: + await asyncio.Event().wait() + finally: + await converter._close_stream(agen) + done.append("cleanup-survived") + + async with anyio.create_task_group() as group: + group.start_soon(worker) + await asyncio.sleep(0) + group.cancel_scope.cancel() + self.assertEqual(done, ["closed", "cleanup-survived"]) + + if __name__ == "__main__": unittest.main(verbosity=2) From f63285cb93c5b953123be055daf9b4c2ebc570b9 Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 04:40:07 +0000 Subject: [PATCH 6/8] Do the disconnect teardown without a new dependency Follow-up to d045bf1, which shielded the teardown with `anyio.CancelScope` and declared anyio as a direct dependency. Since the base moved to a pyproject + `uv.lock` regime where `requirements.in` is generated and `tests/test_build_lock.py` pins every transitive to the lock, growing the direct-dependency set for one `finally` is the wrong trade. The guarantee is kept with stdlib only: - anyio re-delivers cancellation to the host task once per event loop cycle (`_deliver_cancellation` re-arms with `call_soon`), which is why a bare `await agen.aclose()`/`await task` in that context never finishes: the fix is not to wait harder, it is to run the teardown in a task that is *not* a member of the cancelled scope. `_first_segment` already moved the generator frame there, so `child.cancel()` is delivered exactly once and the frame's own `finally` (where httpx closes the connection) completes. - `_teardown_finished()` then waits up to `TEARDOWN_GRACE_CYCLES` poll cycles for that task so the common case stays synchronous, and otherwise leaves it to run in the background with its result drained. The bound is cycles, not wall time: under re-cancellation every `await` returns immediately, so a time bound would busy-wait instead. Measured with the same regressions: the ASGI disconnect tests and the cancel-scope teardown test all still pass, repeatedly. --- converter.py | 61 +++++++++++++++++++++------- tests/test_stream_status_contract.py | 14 ++++--- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/converter.py b/converter.py index 41ad6fb..1ca673b 100644 --- a/converter.py +++ b/converter.py @@ -40,7 +40,6 @@ from pathlib import Path from typing import Optional -import anyio # 断连收尾要屏蔽外层取消：用的正是发起取消的那套机制 import httpx from fastapi import FastAPI, Header, HTTPException, Request from fastapi.exception_handlers import http_exception_handler as _default_http_exception_handler @@ -2785,24 +2784,54 @@ def __init__(self, status, raw, error=None): super().__init__(f"stream failed before first byte (HTTP {status})") +# 断连收尾的等法：轮数而非墙上时间做上界。被反复取消时每次 await 都会立刻抛回来，用时间做 +# 上界就变成忙等；100 轮足够走完一次正常的关闭（实测个位数轮次），走完不成就交给后台。 +TEARDOWN_GRACE_CYCLES = 100 +TEARDOWN_POLL_SECONDS = 0.01 + + +def _drain_teardown(future) -> None: + """后台收尾任务的异常只取走、不重抛：它跑在没人再取消它的任务里，最终会做完。""" + if not future.cancelled(): + future.exception() + + +async def _teardown_finished(task) -> None: + """尽量当场等收尾任务结束；等不到就挂个回调让它后台做完，绝不因此拖住取消本身。 + + 为什么不能老实 `await task`：下游断连时 anyio 的取消作用域**每个事件循环周期**重投一次 + 取消（`_deliver_cancellation` 用 `call_soon` 自循环），当前任务里的任何 await 都会被反复 + 打断。收尾因此放在独立任务里 —— 它不属于那个作用域，没人再取消它 —— 这里只是尽量把结果 + 等成同步的，等不到也不影响它最终跑完。 + """ + for _ in range(TEARDOWN_GRACE_CYCLES): + if task.done(): + _drain_teardown(task) + return + try: + await asyncio.wait([task], timeout=TEARDOWN_POLL_SECONDS) + except asyncio.CancelledError: + pass + if not task.done(): + task.add_done_callback(_drain_teardown) + + async def _first_segment(agen): """取生成器的第一段输出，但把「我们的等待」和「生成器自己的收尾」分开放。 - 直接在当前任务里 `await agen.__anext__()` 也有问题：下游断连时取消打在生成器帧内部的 - await 上，而 anyio 的取消作用域会在每个检查点重复取消 —— 帧自己的 `finally` 做到一半就 - 被打断（`httpx` 正是在这里关连接），实测那段清理根本跑不完，连接留到读超时。放进子任务 - 之后，外层取消打断的是我们的 `await`，我们只取消子任务一次，再屏蔽着等它把 `finally` - 走完。结果不取：取消语义下这一轮已经作废，交给调用方关流。 + 直接在当前任务里 `await agen.__anext__()` 有个实测问题：断连的取消打在生成器帧内部的 + await 上，帧自己的 `finally` 做到一半就被反复投进来的取消打断 —— `httpx` 正是在那里关 + 连接，于是清理根本跑不完，连接留到读超时。放进子任务之后，外层取消打断的是我们的 + `await`，子任务只被取消一次，它的 `finally` 能自己走完。 + + 取消语义下这一轮已经作废，所以子任务的结果不取；异常交给 `_teardown_finished` 收尾时取走。 """ task = asyncio.ensure_future(agen.__anext__()) try: return await asyncio.shield(task) except BaseException: task.cancel() - with anyio.CancelScope(shield=True): - await asyncio.wait([task]) - if not task.cancelled(): - task.exception() # 取回异常，别留给 asyncio 报「never retrieved」 + await _teardown_finished(task) raise @@ -2842,20 +2871,22 @@ async def _preflight_stream(agen, model_name, t0, rid): async def _close_stream(agen) -> None: - """显式收尾上游生成器，并屏蔽外层取消。 + """显式收尾上游生成器，收尾跑在不受当前取消作用域影响的任务里。 - 下游断连时取消正在反复投递（anyio 的取消作用域会在每个检查点再取消一次），不屏蔽的话 - `httpx` 的关闭做到一半就被打断，上游连接和它的读超时一起留在原地。清理失败不改变已经 - 定型的响应，所以这里只吞掉异常，不吞掉取消。 + 覆盖「取消落在两段之间、帧还停在 yield 上」这种情况：直接 `await agen.aclose()` 会被 + 反复投递的取消打断在 `httpx` 关连接的半途。清理失败不改变已经定型的响应，所以只吞异常。 """ if agen is None: return - with anyio.CancelScope(shield=True): + + async def close() -> None: try: await agen.aclose() except Exception: pass + await _teardown_finished(asyncio.ensure_future(close())) + def _chunk_bytes(chunk, charset: str = "utf-8"): return chunk if isinstance(chunk, (bytes, memoryview)) else chunk.encode(charset) diff --git a/tests/test_stream_status_contract.py b/tests/test_stream_status_contract.py index f90beb2..541eae9 100644 --- a/tests/test_stream_status_contract.py +++ b/tests/test_stream_status_contract.py @@ -275,12 +275,14 @@ async def test_disconnect_after_streaming_started_closes_the_upstream(self): self.assertEqual(self.closed, 1) -class ShieldedCloseTests(unittest.IsolatedAsyncioTestCase): - """`_close_stream` 必须扛得住「正在被取消」这件事本身。 - - 断连时 anyio 的取消作用域会在每个检查点重复取消；不屏蔽的话，生成器 finally 里那个 - await（httpx 在这里关连接）做到一半就被打断，连接和它的读超时一起留在原地。实测： - 同样的作用域里不加屏蔽，收尾 await 从来跑不完。 +class TeardownCloseTests(unittest.IsolatedAsyncioTestCase): + """`_close_stream` 要扛得住「当前任务正在被反复取消」这件事。 + + 直接 `await agen.aclose()` 是不行的：anyio 的取消作用域用 `call_soon` 自循环，每个事件 + 循环周期重投一次取消，生成器 finally 里那个 await（httpx 在这里关连接）做到一半就被打断 + —— 实测要么永远等不到 `closed`，要么半关。收尾因此放进独立任务（不属于那个作用域，没人再 + 取消它），再尽量当场等它做完。这里钉住「停在 yield 上被取消」这一种：帧没在自己内部被撕开， + 正是 `_close_stream` 负责的那一段。 """ async def test_cleanup_await_completes_inside_a_cancelled_scope(self): From dc938172bf579ea00bfa5f45069b9638deeb4460 Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:19:50 +0000 Subject: [PATCH 7/8] Drop the stale anyio attribution left in the generated requirements.txt `requirements.txt` is exported by `scripts/export_requirements.py` (`uv export --locked --no-dev --no-emit-project`). With the revert to a stdlib-only teardown, anyio is no longer a direct requirement, so the `# -r requirements.in` attribution under it is dead: direct deps are exported as `# via codebuddy2api`, and anyio's own block already names httpx/starlette/watchfiles. Removing the line makes this file identical to upstream again (no diff), so the PR touches nothing dependency-related. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a83953..64fa7c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ anyio==4.14.2 \ --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f # via - # -r requirements.in # httpx # starlette # watchfiles From 23fcf35e579542dac04bc741d65bdc11ff28813b Mon Sep 17 00:00:00 2001 From: szbfwdy <233512983+szbfwdy@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:01:56 +0000 Subject: [PATCH 8/8] Keep a replay from clearing a failure it did not cause Codex's re-review of dc93817 (P2, converter.py:3054) is right: the recovery marker was cleared by sequence position, not by identity. `observe_recovery()` unset whatever `error_code` happened to be current, so a replay could swallow a failure recorded *after* it. The reachable case is the non-streaming aggregate. `_fetch_checked_chat` returns normally for a filtered completion -- it calls `observe_failure("content_filter")` and hands the refusal back -- and the `if tried: observe_recovery()` right after it then erased that, persisting a blocked request as `outcome=success` with no error code. The `failover_recovered` attempt even named `content_filter` as the failure it had recovered from. The aggregate-inside-preflight stream (a `tools` request on `/v1/chat/completions`) had the same ordering, and a plain streamed refusal did not: there the filter lands after the preflight, so it survived. Recovery is now scoped to the failure that triggered the replay. `_Observation` keeps a per-request `failure_seq`, bumped on every `fail()`; the failover loops snapshot it when a round fails and pass it as `observe_recovery(through=...)`, which only clears while that is still the newest failure. A newer one stands on its own, so the filtered row reads `outcome=error` + `error_code=content_filter` + HTTP 200 -- exactly what the same refusal records with `--failover-max=0`. The default `through=None` keeps the old semantics for any future caller that has no sequence to name. Three regressions in tests/test_stream_failover.py, all red on dc93817: the non-streaming chat row, the aggregated stream row, and a parity case asserting a filtered request audits identically with and without a replay. The existing recovery test now also pins the marker's code (`upstream_429`), so a recovery can never again be credited with clearing something else. --- app/observability.py | 39 ++++++++++++----- converter.py | 15 +++++-- tests/test_stream_failover.py | 79 ++++++++++++++++++++++++++++++++++- 3 files changed, 118 insertions(+), 15 deletions(-) diff --git a/app/observability.py b/app/observability.py index 70eaac9..d91bd5a 100644 --- a/app/observability.py +++ b/app/observability.py @@ -65,6 +65,7 @@ class _Observation: monotonic_start: float attempts: list = field(default_factory=list) failed: bool = False + failure_seq: int = 0 terminal: bool = False body_finished: bool = False status: int | None = None @@ -76,6 +77,7 @@ class _Observation: def fail(self, code): self.failed = True + self.failure_seq += 1 self.record["error_code"] = safe_label(code, 80) or "upstream_error" def usage(self, value, source, priority): @@ -159,25 +161,42 @@ def observe_attempt(stage, **safe_metadata): def observe_failure(code): + """记下失败并返回本次请求的失败序号，供 `observe_recovery(through=…)` 界定撤销范围。""" observation = _current.get() - if observation is not None: - observation.fail(code) + if observation is None: + return None + observation.fail(code) + return observation.failure_seq + +def observe_failure_seq(): + """当前失败序号的快照；没有失败时为 0，调用方原样传给 `observe_recovery` 即可。""" + observation = _current.get() + return observation.failure_seq if observation is not None else None -def observe_recovery(): - """标记「先前记录的失败已经被就地重放救回」：请求对下游是完整正常响应。 + +def observe_recovery(through=None): + """标记「`through` 那一次失败已经被就地重放救回」：请求对下游是完整正常响应。 失败尝试仍留在 `attempts` 里（另加一条 `failover_recovered` 标记），只是不再决定 outcome —— 否则一次成功的换凭证重放会留下 `outcome=error` + `status_code=200` 这种自相矛盾的 审计记录，看板和排障都会把它读成失败。 + + `through` 是重放前那次失败的序号，只有它仍然是最新一次失败时才撤销：序号对不上说明 + 重放之后的响应自己又记了新失败（换到的账号回了内容审核拒绝就是这种），那次失败必须留下， + 否则一个被审核拦截的请求会被持久化成 `outcome=success` 且没有 `error_code`。默认 `None` + 保持旧的「清掉当前失败」语义，给没有序号概念的调用方兜底。 """ observation = _current.get() - if observation is not None and observation.failed: - code = observation.record.get("error_code") or "upstream_error" - observation.failed = False - observation.record["error_code"] = None - if len(observation.attempts) < 32: - observation.attempts.append(safe_attempt({"stage": "failover_recovered", "code": code})) + if observation is None or not observation.failed: + return + if through is not None and observation.failure_seq != through: + return + code = observation.record.get("error_code") or "upstream_error" + observation.failed = False + observation.record["error_code"] = None + if len(observation.attempts) < 32: + observation.attempts.append(safe_attempt({"stage": "failover_recovered", "code": code})) class _Parser: diff --git a/converter.py b/converter.py index d343ef6..43e829a 100644 --- a/converter.py +++ b/converter.py @@ -70,7 +70,8 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app import checkin as checkin_service, model_policy, travel from app.model_blocks import ModelBlocks from app.observability import (AuditMiddleware, observe_recovery, observe_route, - observe_usage, observe_attempt, observe_failure) + observe_usage, observe_attempt, observe_failure, + observe_failure_seq) from app.credential_io import (CredentialFileError, read_import_file, atomic_write_credential, credential_file_lock) from app.upstream_io import (ChatSSEAccumulator, UpstreamHTTPError, UpstreamResponseError, @@ -2997,12 +2998,14 @@ async def _stream_plan(payload, canonical, model_name, rid, t0, make, routed, cr 去问绑定规则 —— 否则第二轮查的是默认模型，客户端原来说的 `auto` 的账号/站点限制就丢了。 """ tried = [] + recovered = None while True: stream = make(routed, cred, headers, url) try: first = await _preflight_stream(stream, model_name, t0, rid) except _StreamFailure as failure: await _close_stream(stream) # 本轮的上游已经终止，关掉只是兜底，不留半开的连接 + recovered = observe_failure_seq() # 这一枪记的失败，才是重放有权撤销的那一次 tried.append(cred) limit = _failover_limit() surface = HTTPException(status_code=failure.status, @@ -3026,7 +3029,9 @@ async def _stream_plan(payload, canonical, model_name, rid, t0, make, routed, cr await _close_stream(stream) raise if tried: - observe_recovery() # 重放救回来的请求对下游是正常响应，不该记成失败 + # 只撤销重放对应的那一次失败：序号对不上说明换到手的响应自己又记了新失败 + # （最典型是内容审核拒绝），那次失败要如实留在审计里。 + observe_recovery(recovered) # 重放救回来的请求对下游是正常响应，不该记成失败 return stream, first @@ -3047,14 +3052,18 @@ async def _routed_fetch(payload, canonical, model_name, rid, t0, fetch, routed, `canonical` 同 `_routed_stream`：重路由用改写前的规范请求体，判定才落在客户端模型上。 """ tried = [] + recovered = None while True: try: collected = await fetch(routed, cred, headers, url) if tried: - observe_recovery() # 同上：换凭证后成功的请求不该记成失败 + # 同 `_stream_plan`：聚合路径里 `_fetch_checked_chat` 会在返回前就记上审核拒绝， + # 无差别撤销会把被拦截的请求写成一次成功。 + observe_recovery(recovered) # 换凭证后成功的请求不该记成失败 return collected except (httpx.HTTPError, UpstreamResponseError) as error: status, raw = _upstream_failure(error, model_name, t0, rid) + recovered = observe_failure_seq() tried.append(cred) limit = _failover_limit() surface = HTTPException(status_code=status, detail=_safe_err_raw(raw, status)) diff --git a/tests/test_stream_failover.py b/tests/test_stream_failover.py index 27d0f41..2bc0372 100644 --- a/tests/test_stream_failover.py +++ b/tests/test_stream_failover.py @@ -47,6 +47,23 @@ def error_body(message="synthetic rejection", code="rate_limit"): return {"error": {"message": message, "type": "upstream_error", "code": code}} +def filtered_sse(): + """上游正常回 200、结果却是审核拒绝：聚合路径会在 `fetch` 返回**之前**就记上这次失败。 + + 只用 `finish_reason` 触发检测（`ContentFilterDetector.feed` 认 `content_filter`）， + 不依赖任何拒绝文案，免得上游改措辞就把测试带崩。 + """ + chunks = [ + {"id": "synthetic-completion", "choices": [{"index": 0, + "delta": {"role": "assistant", "content": "blocked"}, "finish_reason": None}]}, + {"id": "synthetic-completion", "choices": [{"index": 0, + "delta": {}, "finish_reason": "content_filter"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}}, + ] + return ("".join("data: " + json.dumps(chunk) + "\n\n" for chunk in chunks) + + "data: [DONE]\n\n").encode() + + @contextmanager def allow_failover(times: int): """打开换凭证重放开关（等价于 --failover-max N）。""" @@ -337,8 +354,8 @@ def test_disabled_by_default_replays_nothing(self): self.stream = True # --- 审计口径：重放救回来的请求不得留在 error --- - def audited_client(self): - store = AuditStore(self.root / "failover-audit.sqlite3") + def audited_client(self, name="failover-audit"): + store = AuditStore(self.root / f"{name}.sqlite3") # 一条用例要对比两行审计时分开落盘 self.addCleanup(store.close) application = FastAPI() application.router.routes = list(converter.app.router.routes) @@ -363,6 +380,64 @@ def test_replayed_request_is_audited_as_success(self): statuses = [attempt.get("status_code") for attempt in record["attempts"]] self.assertEqual(statuses[:2], [429, 200], record["attempts"]) self.assertIn("failover_recovered", json.dumps(record["attempts"], ensure_ascii=False)) + markers = [attempt for attempt in record["attempts"] + if attempt.get("stage") == "failover_recovered"] + self.assertEqual([attempt.get("code") for attempt in markers], ["upstream_429"], + "恢复标记必须点名「被撤销的那一次失败」本身") + + def test_content_filter_after_replay_is_still_audited_as_filtered(self): + """换到的账号回了审核拒绝：那是这一枪的真实结果，不能被上一枪 429 的重放抹掉。 + + 没有修复前的样子：`outcome=success` + `error_code` 为空 + 恢复标记写着 + `content_filter` —— 等于把一次被拦截的请求记成了一次干净的成功。 + """ + store, client = self.audited_client() + with allow_failover(1), patch.object(fixtures, "success_sse", filtered_sse): + self.poison_once = True + self.poison_with_status(429) + response = client.post("/v1/chat/completions", json=self.payload(stream=False)) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(self.failover_lines()), 1, self.logs) + record = self.only_record(store) + self.assertEqual(record["outcome"], "error", record) + self.assertEqual(record["error_code"], "content_filter", record) + self.assertNotIn("failover_recovered", + json.dumps(record["attempts"], ensure_ascii=False), record["attempts"]) + + def test_content_filter_audit_matches_the_unreplayed_request(self): + """同一次审核拒绝，走没走过重放必须是同一行审计：重放不该改变账单口径。""" + with patch.object(fixtures, "success_sse", filtered_sse): + store, client = self.audited_client("filter-without-replay") + plain = client.post("/v1/chat/completions", json=self.payload(stream=False)) + self.assertEqual(plain.status_code, 200, plain.text) + baseline = self.only_record(store) + self.assertEqual(baseline["outcome"], "error", baseline) + self.assertEqual(baseline["error_code"], "content_filter", baseline) + store, client = self.audited_client("filter-with-replay") + with allow_failover(1), patch.object(fixtures, "success_sse", filtered_sse): + self.poison_once = True + self.poison_with_status(429) + replayed = client.post("/v1/chat/completions", json=self.payload(stream=False)) + self.assertEqual(replayed.status_code, 200, replayed.text) + record = self.only_record(store) + for key in ("outcome", "error_code", "status_code"): + self.assertEqual(record[key], baseline[key], (key, record, baseline)) + + def test_content_filter_after_replay_survives_the_aggregated_stream(self): + """带 tools 的流式在预取里就跑完整聚合，`_stream_plan` 那条顺序陷阱一模一样。""" + store, client = self.audited_client() + payload = self.payload(stream=True) + payload["tools"] = [{"type": "function", "function": {"name": "synthetic_tool", + "parameters": {"type": "object"}}}] + with allow_failover(1), patch.object(fixtures, "success_sse", filtered_sse): + self.poison_once = True + self.poison_with_status(429) + response = client.post("/v1/chat/completions", json=payload) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(self.failover_lines()), 1, self.logs) + record = self.only_record(store) + self.assertEqual(record["outcome"], "error", record) + self.assertEqual(record["error_code"], "content_filter", record) def test_unreplayed_failure_is_still_audited_as_error(self): store, client = self.audited_client()