diff --git a/benchmark/hosted_grounder_probe/RESULTS.md b/benchmark/hosted_grounder_probe/RESULTS.md new file mode 100644 index 00000000..aba77107 --- /dev/null +++ b/benchmark/hosted_grounder_probe/RESULTS.md @@ -0,0 +1,162 @@ +# Hosted grounder probe — real model, real wire (Together AI) + +**Date: 2026-08-12.** First run of `OpenAICompatibleGrounder` against a real +served model. Endpoint: `https://api.together.xyz/v1` (Together AI serverless). +Driver: `scripts/run_hosted_probe.sh` -> `scripts/probe_local_grounder.py`; +the adapter under test is the shipped +`openadapt_flow.runtime.grounder.OpenAICompatibleGrounder` (an injected +recording HTTP client observes token usage; request construction and reply +parsing are the adapter's own). + +**Caveat (read first).** Hosted weights are NOT an on-prem deployment. This +probe measures (a) the adapter's wire-protocol correctness against a real +commercial OpenAI-compatible endpoint and (b) the capability of the served +open-weights model on our dense-list surfaces. It says nothing about on-prem +latency, quantized local builds, or appliance behaviour; the local/on-prem run +remains a separate scheduled artifact (`benchmark/local_grounder_probe/`, +still UNRUN). + +## Model availability on Together (measured 2026-08-12, models API) + +The classic Qwen VL family is listed but **dedicated-endpoint only** — every +serverless chat call returns `model_not_available`: + +| model id | serverless? | listed price in/out (USD per 1M) | +|---|---|---| +| `Qwen/Qwen2-VL-72B-Instruct` | no (dedicated only) | 1.20 / 1.20 | +| `Qwen/Qwen2.5-VL-72B-Instruct` | no (dedicated only) | 1.95 / 8.00 | +| `Qwen/Qwen3-VL-8B-Instruct` | no (dedicated only) | 0.18 / 0.68 | +| `Qwen/Qwen3-VL-32B-Instruct` | no (dedicated only) | 0.50 / 1.50 | +| `Qwen/Qwen3-VL-235B-A22B-Instruct-FP8` | no (dedicated only) | n/a | +| `Qwen/Qwen3.5-9B` | **yes** (image-capable, reasoning) | 0.17 / 0.25 | +| `Qwen/Qwen3.7-Plus` | streaming-only (image-capable) | 0.32 / 1.28 | + +Creating a dedicated GPU endpoint for the Qwen VL pair (8B at $0.09/min, +32B at $0.18/min) was prepared but the session's permission system denied the +endpoint-creation call, and per standing instructions no workaround was +attempted. The probed pair is therefore the two hosted Qwen models the shipped +adapter can reach: `Qwen/Qwen3.5-9B` (smallest image-capable serverless Qwen) +and `Qwen/Qwen3.7-Plus` (strongest hosted Qwen reachable at all — which turns +out to be unreachable by a non-streaming client; measured below). + +## Setup + +- Surfaces (both rendered by `openadapt_flow.validation.dense_surface`, the + committed fixture source of `benchmark/dense_surface/record_seed1.png`): + - `record_seed1`: seed 1, RECORD_CONDITION — 51 rows, 2240x3726 px + (device_scale_factor 2). Identical to `benchmark/grounding_eval`, so + directly comparable with the July baselines. + - `small_dense_seed2`: seed 2, `small_dense` condition — 51 rows, + 1120x1401 px (12 px font, tighter rows, device_scale_factor 1). +- Targets: 6 per surface, the deterministic spread `indices[::step][:6]` + (rows 0, 8, 16, 24, 32, 40). Truth = DOM centre of each row's Open button. + Tolerances: hit@40 px (headline) and hit@60 px. +- Prompt (the adapter's `_PROMPT`, verbatim template): + + ``` + You are grounding a UI automation target on a screenshot. + Target intent: {intent} + Target text label (may be stale): {ocr_text} + + Reply with ONLY a JSON object of pixel coordinates for the point to + click, e.g. {"x": 123, "y": 45}. If the target is not visible, reply + with ONLY {"x": null, "y": null}. + ``` + + with `intent = "click Open in the row for patient {name} (MRN {mrn})"` and + `ocr_text = "Open"`. + +## Run 1 — `Qwen/Qwen3.5-9B`, adapter default `max_tokens=256` + +**hit@40 0/12 · hit@60 0/12 · abstain 11/12 · 1 miss @ 2594 px · $0.0108** + +Every abstain shows `completion_tokens: 256` — the hosted reasoning model +spends the whole fixed budget on reasoning and returns EMPTY content, which +the adapter (correctly, fail-safe) treats as an abstain. The single completed +reply (219 tokens) proposed (957, 623) for a truth of (2059, 2971). + +| surface | row | MRN | proposed | truth | err px | result | latency s | +|---|---|---|---|---|---|---|---| +| record_seed1 | 0 | MG584224 | — | (2059, 251) | — | abstain (truncated) | 7.2 | +| record_seed1 | 8 | MG129724 | — | (2059, 795) | — | abstain (truncated) | 5.8 | +| record_seed1 | 16 | MG499721 | — | (2059, 1339) | — | abstain (truncated) | 8.3 | +| record_seed1 | 24 | MG536396 | — | (2059, 1883) | — | abstain (truncated) | 5.3 | +| record_seed1 | 32 | PLl9181 | — | (2059, 2427) | — | abstain (truncated) | 4.3 | +| record_seed1 | 40 | 2OO633 | (957, 623) | (2059, 2971) | 2593.7 | miss | 3.6 | +| small_dense_seed2 | 0 | RC903088 | — | (1015, 118) | — | abstain (truncated) | 5.0 | +| small_dense_seed2 | 8 | MG551589 | — | (1015, 318) | — | abstain (truncated) | 3.2 | +| small_dense_seed2 | 16 | MG687737 | — | (1015, 518) | — | abstain (truncated) | 3.5 | +| small_dense_seed2 | 24 | MG899943 | — | (1015, 718) | — | abstain (truncated) | 5.0 | +| small_dense_seed2 | 32 | PLl9444 | — | (1015, 918) | — | abstain (truncated) | 2.6 | +| small_dense_seed2 | 40 | 5OO675 | — | (1015, 1118) | — | abstain (truncated) | 3.7 | + +This run motivated the `max_tokens` constructor parameter on +`OpenAICompatibleGrounder` (pinned by `tests/test_grounder_max_tokens.py`): +the fixed 256 was a 100% availability loss against a hosted reasoning model. +Safety was never at risk — truncation abstains. + +## Run 2 — `Qwen/Qwen3.7-Plus` (strongest hosted Qwen), `max_tokens=256` + +**hit@40 0/12 · hit@60 0/12 · abstain 12/12 · $0.00 · ~1 s/call** + +Together serves this model streaming-only: every non-streaming +`chat/completions` request is rejected (`This model only supports streaming`), +the adapter sees a 4xx and abstains — 12/12, no crash, no spend. The shipped +adapter cannot measure this model's grounding capability; a streaming client +would be a new adapter feature, not a probe change. + +## Run 3 — `Qwen/Qwen3.5-9B`, `max_tokens=2048` (the capability number) + +**hit@40 0/12 · hit@60 0/12 · abstain 6/12 · 6 misses, median err 1792 px · +$0.0137** + +| surface | row | MRN | proposed | truth | err px | result | latency s | +|---|---|---|---|---|---|---|---| +| record_seed1 | 0 | MG584224 | (958, 76) | (2059, 251) | 1114.8 | miss | 8.5 | +| record_seed1 | 8 | MG129724 | — | (2059, 795) | — | abstain (truncated @2048) | 19.2 | +| record_seed1 | 16 | MG499721 | (684, 190) | (2059, 1339) | 1791.9 | miss | 3.9 | +| record_seed1 | 24 | MG536396 | — | (2059, 1883) | — | abstain (truncated @2048) | 24.3 | +| record_seed1 | 32 | PLl9181 | (929, 608) | (2059, 2427) | 2141.4 | miss | 4.3 | +| record_seed1 | 40 | 2OO633 | (942, 758) | (2059, 2971) | 2478.9 | miss | 19.4 | +| small_dense_seed2 | 0 | RC903088 | — | (1015, 118) | — | abstain (701 tok, model declined) | 11.3 | +| small_dense_seed2 | 8 | MG551589 | (925, 458) | (1015, 318) | 166.4 | miss | 10.3 | +| small_dense_seed2 | 16 | MG687737 | — | (1015, 518) | — | abstain (truncated @2048) | 20.0 | +| small_dense_seed2 | 24 | MG899943 | — | (1015, 718) | — | abstain (truncated @2048) | 27.2 | +| small_dense_seed2 | 32 | PLl9444 | (870, 768) | (1015, 918) | 208.6 | miss | 14.3 | +| small_dense_seed2 | 40 | 5OO675 | — | (1015, 1118) | — | abstain (466 tok, model declined) | 4.8 | + +Reading the misses: on `record_seed1` (device_scale_factor 2) the proposals +cluster around x≈940 against a truth of x=2059 — consistent with the model +answering roughly in CSS-pixel space (1120 CSS px wide) on a 2240-device-px +screenshot — and even after a hypothetical 2x rescale the y is still several +hundred px (many rows) off. On `small_dense_seed2` (scale factor 1) the errors +drop to 166–209 px but remain 5–7 rows away from the right patient. This is +the July failure mode again — column roughly right, row wrong — now +reproduced on hosted, unquantized open weights, so it is not an artifact of +the local 4-bit build. + +## Prompt-token footprint (from the API `usage` field) + +- `record_seed1` (2240x3726): ~8,233 prompt tokens/call. +- `small_dense_seed2` (1120x1401): ~1,652 prompt tokens/call. + +## Total spend + +$0.0245 across the three runs (24 billed calls + 12 free-failing calls), plus +under $0.01 of model-discovery smoke calls. Cap was $5; nothing approached it. + +## Comparison (same `record_seed1` surface and truth) + +| grounder | hit@40 | median err | +|---|---|---| +| OCR row-anchoring, local, $0 (`benchmark/grounding_eval`, July) | 4/6 (88–100% over 50 targets) | ~3 px | +| Bespoke remote VLM, local 4-bit Qwen3-VL-4B (July) | 0/6 | ~472 px | +| `OpenAICompatibleGrounder` -> hosted `Qwen/Qwen3.5-9B` @2048 (this run) | 0/6 | ~2141 px | + +**Bottom line.** The wire protocol works end-to-end against a real commercial +endpoint (auth, image transport, parsing, fail-safe abstain on truncation, +4xx, and refusal all behaved exactly as the contract tests pin). The +single-shot VLM grounding approach stays broken on dense lists with real +hosted open weights — every wrong proposal was a wrong ROW, which is precisely +the silent-wrong-record class the ladder's OCR anchoring + identity band exist +to prevent. The grounder rung remains bottom-of-ladder and risk-gated. diff --git a/benchmark/hosted_grounder_probe/results_qwen3_5_9b.json b/benchmark/hosted_grounder_probe/results_qwen3_5_9b.json new file mode 100644 index 00000000..a87dcaba --- /dev/null +++ b/benchmark/hosted_grounder_probe/results_qwen3_5_9b.json @@ -0,0 +1,298 @@ +{ + "meta": { + "endpoint": "https://api.together.xyz/v1", + "model": "Qwen/Qwen3.5-9B", + "surfaces": { + "record_seed1": { + "viewport_px": [ + 2240, + 3726 + ] + }, + "small_dense_seed2": { + "viewport_px": [ + 1120, + 1401 + ] + } + }, + "surface_spec": "dense_surface.render_frame(build_dense_table(seed=S, n_rows=18), COND, top_offset_px=12); record_seed1: S=1, COND=RECORD_CONDITION; small_dense_seed2: S=2, COND=small_dense", + "truth": "DOM centre of each row's Open button (frame.points[i][1])", + "tol_px": { + "strict": 40, + "loose": 60 + }, + "adapter": "openadapt_flow.runtime.grounder.OpenAICompatibleGrounder", + "price_usd_per_1m": { + "input": 0.17, + "output": 0.25 + }, + "timestamp": "2026-08-12T13:29:23+0200" + }, + "summary": { + "n_targets": 12, + "hits@40": 0, + "hits@60": 0, + "abstained": 11, + "median_error_px": 2593.7, + "est_spend_usd": 0.0108, + "spend_capped": false + }, + "rows": [ + { + "surface": "record_seed1", + "row": 0, + "name": "Halloran, Karen", + "mrn": "MG584224", + "truth": [ + 2059, + 251 + ], + "latency_s": 7.21, + "usage": { + "prompt_tokens": 8233, + "completion_tokens": 256 + }, + "est_cost_usd": 0.001464, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG129724", + "truth": [ + 2059, + 795 + ], + "latency_s": 5.82, + "usage": { + "prompt_tokens": 8234, + "completion_tokens": 256 + }, + "est_cost_usd": 0.001464, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG499721", + "truth": [ + 2059, + 1339 + ], + "latency_s": 8.29, + "usage": { + "prompt_tokens": 8234, + "completion_tokens": 256 + }, + "est_cost_usd": 0.001464, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 24, + "name": "Fitzgerald, Gregory", + "mrn": "MG536396", + "truth": [ + 2059, + 1883 + ], + "latency_s": 5.31, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 256 + }, + "est_cost_usd": 0.001463, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 32, + "name": "Whitfield, Daniel", + "mrn": "PLl9181", + "truth": [ + 2059, + 2427 + ], + "latency_s": 4.26, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 256 + }, + "est_cost_usd": 0.001463, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 40, + "name": "Bianchi, Gregory", + "mrn": "2OO633", + "truth": [ + 2059, + 2971 + ], + "latency_s": 3.56, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 219 + }, + "est_cost_usd": 0.001454, + "proposed": [ + 957, + 623 + ], + "abstained": false, + "error_px": 2593.7, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 0, + "name": "Ferreira, Philip", + "mrn": "RC903088", + "truth": [ + 1015, + 118 + ], + "latency_s": 4.95, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG551589", + "truth": [ + 1015, + 318 + ], + "latency_s": 3.15, + "usage": { + "prompt_tokens": 1654, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG687737", + "truth": [ + 1015, + 518 + ], + "latency_s": 3.49, + "usage": { + "prompt_tokens": 1654, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 24, + "name": "Montgomery, Angela", + "mrn": "MG899943", + "truth": [ + 1015, + 718 + ], + "latency_s": 4.96, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 32, + "name": "Underwood, Thomas", + "mrn": "PLl9444", + "truth": [ + 1015, + 918 + ], + "latency_s": 2.61, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 40, + "name": "Whitfield, Gregory", + "mrn": "5OO675", + "truth": [ + 1015, + 1118 + ], + "latency_s": 3.69, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 256 + }, + "est_cost_usd": 0.000345, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + } + ] +} diff --git a/benchmark/hosted_grounder_probe/results_qwen3_5_9b_mt2048.json b/benchmark/hosted_grounder_probe/results_qwen3_5_9b_mt2048.json new file mode 100644 index 00000000..5120f678 --- /dev/null +++ b/benchmark/hosted_grounder_probe/results_qwen3_5_9b_mt2048.json @@ -0,0 +1,314 @@ +{ + "meta": { + "endpoint": "https://api.together.xyz/v1", + "model": "Qwen/Qwen3.5-9B", + "surfaces": { + "record_seed1": { + "viewport_px": [ + 2240, + 3726 + ] + }, + "small_dense_seed2": { + "viewport_px": [ + 1120, + 1401 + ] + } + }, + "surface_spec": "dense_surface.render_frame(build_dense_table(seed=S, n_rows=18), COND, top_offset_px=12); record_seed1: S=1, COND=RECORD_CONDITION; small_dense_seed2: S=2, COND=small_dense", + "truth": "DOM centre of each row's Open button (frame.points[i][1])", + "tol_px": { + "strict": 40, + "loose": 60 + }, + "adapter": "openadapt_flow.runtime.grounder.OpenAICompatibleGrounder", + "max_tokens": 2048, + "price_usd_per_1m": { + "input": 0.17, + "output": 0.25 + }, + "timestamp": "2026-08-12T13:35:19+0200" + }, + "summary": { + "n_targets": 12, + "hits@40": 0, + "hits@60": 0, + "abstained": 6, + "median_error_px": 1791.9, + "est_spend_usd": 0.0137, + "spend_capped": false + }, + "rows": [ + { + "surface": "record_seed1", + "row": 0, + "name": "Halloran, Karen", + "mrn": "MG584224", + "truth": [ + 2059, + 251 + ], + "latency_s": 8.5, + "usage": { + "prompt_tokens": 8233, + "completion_tokens": 454 + }, + "est_cost_usd": 0.001513, + "proposed": [ + 958, + 76 + ], + "abstained": false, + "error_px": 1114.8, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG129724", + "truth": [ + 2059, + 795 + ], + "latency_s": 19.19, + "usage": { + "prompt_tokens": 8234, + "completion_tokens": 2048 + }, + "est_cost_usd": 0.001912, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG499721", + "truth": [ + 2059, + 1339 + ], + "latency_s": 3.87, + "usage": { + "prompt_tokens": 8234, + "completion_tokens": 155 + }, + "est_cost_usd": 0.001439, + "proposed": [ + 684, + 190 + ], + "abstained": false, + "error_px": 1791.9, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 24, + "name": "Fitzgerald, Gregory", + "mrn": "MG536396", + "truth": [ + 2059, + 1883 + ], + "latency_s": 24.28, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 2048 + }, + "est_cost_usd": 0.001911, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 32, + "name": "Whitfield, Daniel", + "mrn": "PLl9181", + "truth": [ + 2059, + 2427 + ], + "latency_s": 4.32, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 222 + }, + "est_cost_usd": 0.001455, + "proposed": [ + 929, + 608 + ], + "abstained": false, + "error_px": 2141.4, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 40, + "name": "Bianchi, Gregory", + "mrn": "2OO633", + "truth": [ + 2059, + 2971 + ], + "latency_s": 19.37, + "usage": { + "prompt_tokens": 8232, + "completion_tokens": 1758 + }, + "est_cost_usd": 0.001839, + "proposed": [ + 942, + 758 + ], + "abstained": false, + "error_px": 2478.9, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 0, + "name": "Ferreira, Philip", + "mrn": "RC903088", + "truth": [ + 1015, + 118 + ], + "latency_s": 11.33, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 701 + }, + "est_cost_usd": 0.000456, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG551589", + "truth": [ + 1015, + 318 + ], + "latency_s": 10.26, + "usage": { + "prompt_tokens": 1654, + "completion_tokens": 1022 + }, + "est_cost_usd": 0.000537, + "proposed": [ + 925, + 458 + ], + "abstained": false, + "error_px": 166.4, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG687737", + "truth": [ + 1015, + 518 + ], + "latency_s": 20.02, + "usage": { + "prompt_tokens": 1654, + "completion_tokens": 2048 + }, + "est_cost_usd": 0.000793, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 24, + "name": "Montgomery, Angela", + "mrn": "MG899943", + "truth": [ + 1015, + 718 + ], + "latency_s": 27.18, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 2048 + }, + "est_cost_usd": 0.000793, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 32, + "name": "Underwood, Thomas", + "mrn": "PLl9444", + "truth": [ + 1015, + 918 + ], + "latency_s": 14.33, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 1376 + }, + "est_cost_usd": 0.000625, + "proposed": [ + 870, + 768 + ], + "abstained": false, + "error_px": 208.6, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 40, + "name": "Whitfield, Gregory", + "mrn": "5OO675", + "truth": [ + 1015, + 1118 + ], + "latency_s": 4.77, + "usage": { + "prompt_tokens": 1652, + "completion_tokens": 466 + }, + "est_cost_usd": 0.000397, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + } + ] +} diff --git a/benchmark/hosted_grounder_probe/results_qwen3_7_plus.json b/benchmark/hosted_grounder_probe/results_qwen3_7_plus.json new file mode 100644 index 00000000..43ae324e --- /dev/null +++ b/benchmark/hosted_grounder_probe/results_qwen3_7_plus.json @@ -0,0 +1,259 @@ +{ + "meta": { + "endpoint": "https://api.together.xyz/v1", + "model": "Qwen/Qwen3.7-Plus", + "surfaces": { + "record_seed1": { + "viewport_px": [ + 2240, + 3726 + ] + }, + "small_dense_seed2": { + "viewport_px": [ + 1120, + 1401 + ] + } + }, + "surface_spec": "dense_surface.render_frame(build_dense_table(seed=S, n_rows=18), COND, top_offset_px=12); record_seed1: S=1, COND=RECORD_CONDITION; small_dense_seed2: S=2, COND=small_dense", + "truth": "DOM centre of each row's Open button (frame.points[i][1])", + "tol_px": { + "strict": 40, + "loose": 60 + }, + "adapter": "openadapt_flow.runtime.grounder.OpenAICompatibleGrounder", + "price_usd_per_1m": { + "input": 0.32, + "output": 1.28 + }, + "timestamp": "2026-08-12T13:30:12+0200" + }, + "summary": { + "n_targets": 12, + "hits@40": 0, + "hits@60": 0, + "abstained": 12, + "median_error_px": null, + "est_spend_usd": 0.0, + "spend_capped": false + }, + "rows": [ + { + "surface": "record_seed1", + "row": 0, + "name": "Halloran, Karen", + "mrn": "MG584224", + "truth": [ + 2059, + 251 + ], + "latency_s": 1.41, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG129724", + "truth": [ + 2059, + 795 + ], + "latency_s": 0.98, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG499721", + "truth": [ + 2059, + 1339 + ], + "latency_s": 1.28, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 24, + "name": "Fitzgerald, Gregory", + "mrn": "MG536396", + "truth": [ + 2059, + 1883 + ], + "latency_s": 0.96, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 32, + "name": "Whitfield, Daniel", + "mrn": "PLl9181", + "truth": [ + 2059, + 2427 + ], + "latency_s": 0.91, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "record_seed1", + "row": 40, + "name": "Bianchi, Gregory", + "mrn": "2OO633", + "truth": [ + 2059, + 2971 + ], + "latency_s": 1.15, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 0, + "name": "Ferreira, Philip", + "mrn": "RC903088", + "truth": [ + 1015, + 118 + ], + "latency_s": 1.04, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 8, + "name": "Ngyuen, Anh", + "mrn": "MG551589", + "truth": [ + 1015, + 318 + ], + "latency_s": 0.75, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 16, + "name": "Okafor, Janet", + "mrn": "MG687737", + "truth": [ + 1015, + 518 + ], + "latency_s": 1.02, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 24, + "name": "Montgomery, Angela", + "mrn": "MG899943", + "truth": [ + 1015, + 718 + ], + "latency_s": 0.52, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 32, + "name": "Underwood, Thomas", + "mrn": "PLl9444", + "truth": [ + 1015, + 918 + ], + "latency_s": 0.58, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + }, + { + "surface": "small_dense_seed2", + "row": 40, + "name": "Whitfield, Gregory", + "mrn": "5OO675", + "truth": [ + 1015, + 1118 + ], + "latency_s": 1.59, + "usage": {}, + "est_cost_usd": 0.0, + "proposed": null, + "abstained": true, + "error_px": null, + "hit@40": false, + "hit@60": false + } + ] +} diff --git a/benchmark/local_grounder_probe/RESULTS.md b/benchmark/local_grounder_probe/RESULTS.md new file mode 100644 index 00000000..6d5c97ae --- /dev/null +++ b/benchmark/local_grounder_probe/RESULTS.md @@ -0,0 +1,55 @@ +# Local grounder probe — UNRUN + +**Status: UNRUN locally. There are no local results.** No measurement of +`OpenAICompatibleGrounder` against a locally served model has completed with +this probe. Nothing in this directory may be cited as local evidence. + +**Hosted results exist.** The same probe DID run against a real hosted model +(Together AI, 2026-08-12): see `benchmark/hosted_grounder_probe/RESULTS.md`. +That run measures model capability and wire-protocol correctness on hosted +open weights; the on-prem/local measurement this directory is reserved for +remains outstanding. + +## What this probe is + +`scripts/probe_local_grounder.py` points the shipped, unmodified +`openadapt_flow.runtime.grounder.OpenAICompatibleGrounder` at a real +OpenAI-compatible endpoint serving a real vision model, on two dense-list +surfaces with DOM ground truth (the `benchmark/grounding_eval` surface — +seed 1, ~51 rows — plus a distinct seed-2 `small_dense` render; 6 +deterministic targets each, truth = the DOM centre of each target row's Open +button, tolerances 40/60 px). It records per-target proposals, errors, hits, +misses, abstentions, latency, and token usage verbatim to +`results.json` in this directory — no retries, no downscaling, no massaging. + +The wire protocol itself is proven in CI without any model by +`tests/test_grounder_openai_compatible_contract.py` (in-process loopback +server, real httpx transport). This probe adds only the real-model evidence, +which CI never needs: the probe is env-gated behind +`OPENADAPT_GROUNDER_BASE_URL` / `OPENADAPT_GROUNDER_MODEL` and exits UNRUN +when they are unset. + +## Why it is unrun + +The first run attempt (2026-08-12, Ollama serving `qwen3-vl:8b` on loopback) +was aborted: the machine kernel-panicked under concurrent load, with local +model inference a likely contributor. + +## Requirements for the first real run (hard) + +- A **dedicated solo run window**: nothing else on the machine — no builds, + no test suites, no parallel agents, no other model servers, no VMs. +- **Never concurrently with builds** or CI jobs anywhere on the host. +- An **explicitly small model** (8B-class or below, quantized — e.g. + `qwen3-vl:8b` / `qwen2.5vl:7b`). Never a large local model. + +## Prior comparable numbers (different setups, recorded elsewhere) + +- Bespoke remote-VLM grounder (served `mlx-community/Qwen3-VL-4B-Instruct-4bit`), + same surface: **0/6 hits @ 40 px, ~472 px median error** + (`benchmark/appliance_validation/REPORT.md`). +- OCR row-anchoring (local, $0), same surface: **4/6 @ 40 px, ~3.1 px median** + (`benchmark/grounding_eval/REPORT.md`). + +A single-shot VLM grounder is measured BROKEN on dense lists in those +baselines. This probe exists to measure, not to vindicate. diff --git a/openadapt_flow/runtime/grounder.py b/openadapt_flow/runtime/grounder.py index 49763bcd..e498e8b5 100644 --- a/openadapt_flow/runtime/grounder.py +++ b/openadapt_flow/runtime/grounder.py @@ -665,6 +665,7 @@ def __init__( *, api_key: str = "", timeout: float = 10.0, + max_tokens: int = 256, client: Any = None, ) -> None: """Create the grounder. @@ -677,20 +678,35 @@ def __init__( => no auth header (a loopback vLLM/Ollama needs none). The caller resolves this from a named env var; it is never stored here. timeout: Per-call timeout in seconds. + max_tokens: Completion-token budget sent as ``max_tokens``. The + default (256) is ample for the coordinate JSON, but a hosted + REASONING model spends its budget on reasoning before emitting + content: measured on Together's ``Qwen/Qwen3.5-9B`` + (2026-08-12, ``benchmark/hosted_grounder_probe/``), 256 makes + it truncate mid-reasoning with EMPTY content on 11/12 dense- + list targets — every one a (safe) abstain. Raise this for such + a model. Truncation can only ever lower availability, never + safety: an empty or cut-off reply abstains. client: Optional pre-built ``httpx.Client``-like object exposing ``post(url, json=..., headers=..., timeout=...)`` (for tests / custom transports). None => module-level ``httpx.post``. Raises: - ValueError: If ``base_url`` or ``model`` is empty/blank. + ValueError: If ``base_url`` or ``model`` is empty/blank, or + ``max_tokens`` is not a positive integer. """ if not base_url or not str(base_url).strip(): raise ValueError("OpenAICompatibleGrounder requires a non-empty base_url") if not model or not str(model).strip(): raise ValueError("OpenAICompatibleGrounder requires a non-empty model") + if isinstance(max_tokens, bool) or not isinstance(max_tokens, int): + raise ValueError("OpenAICompatibleGrounder max_tokens must be an int") + if max_tokens <= 0: + raise ValueError("OpenAICompatibleGrounder max_tokens must be positive") self._url = base_url.rstrip("/") + "/chat/completions" self._model = model.strip() self._timeout = timeout + self._max_tokens = max_tokens self._headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} self._client = client @@ -711,7 +727,7 @@ def locate( image_b64 = base64.standard_b64encode(screen_png).decode("utf-8") body = { "model": self._model, - "max_tokens": 256, + "max_tokens": self._max_tokens, "messages": [ { "role": "user", diff --git a/public-artifacts.json b/public-artifacts.json index 9fe9bb54..bcaf1d36 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -283,6 +283,18 @@ "path": "benchmark/grounding_eval/results_sweep.json", "sha256": "580eef16facc7bb7fc9b225daabded6bb7344c2bf2e9aa070364a3c1215ed9b7" }, + { + "path": "benchmark/hosted_grounder_probe/results_qwen3_5_9b.json", + "sha256": "9548ec8f10f5b2ae082ce439b433477821d8317eedf55e8a31406f17b54a8e5b" + }, + { + "path": "benchmark/hosted_grounder_probe/results_qwen3_5_9b_mt2048.json", + "sha256": "7e867904c4c59b23db0727e403a478832ea944f193eebaa70500d567ed6c0fb2" + }, + { + "path": "benchmark/hosted_grounder_probe/results_qwen3_7_plus.json", + "sha256": "227e3956f6c913a425ea88b591f7be270b155fcaa97561f77d16b25597a673a2" + }, { "path": "benchmark/hybrid/results.json", "sha256": "cd4e736884b70d0f7723c7ece17bbfdd83d1553922ee576a374ff85114997984" diff --git a/scripts/probe_local_grounder.py b/scripts/probe_local_grounder.py new file mode 100644 index 00000000..699ebc9f --- /dev/null +++ b/scripts/probe_local_grounder.py @@ -0,0 +1,324 @@ +"""Probe OpenAICompatibleGrounder against a REAL served vision model. + +Works against any OpenAI-compatible endpoint: a locally served model (Ollama, +vLLM, LM Studio) or a hosted provider (e.g. Together AI at +``https://api.together.xyz/v1``). A hosted run measures model capability and +wire-protocol correctness with the real open weights; it is NOT evidence about +an on-prem deployment, which is a separate scheduled artifact. + +LOCAL-RUN STATUS: UNRUN. A first local run attempt (2026-08-12, Ollama serving +qwen3-vl:8b on loopback) was aborted when the machine kernel-panicked under +concurrent load with local model inference as a likely contributor. Treat every +local number as nonexistent until a run under the conditions below produces +``benchmark/local_grounder_probe/results.json``. Hosted results live in +``benchmark/hosted_grounder_probe/``. + +OPERATIONAL REQUIREMENTS for a LOCAL run (hard — irrelevant to a hosted +endpoint, which puts no inference load on this machine): + +* **Dedicated solo run window.** Run this with NOTHING else on the machine: + no builds, no test suites, no parallel agents, no other model servers, no + VMs. Local VLM inference plus concurrent load starved the watchdog and + kernel-panicked the host on the first attempt. +* **Never concurrently with builds.** Do not launch this while any compile, + CI job, or packaging step runs anywhere on the host. +* **Explicitly small model.** Serve a small vision model (8B-class or below, + quantized — e.g. ``qwen3-vl:8b`` / ``qwen2.5vl:7b``). Do not point this at + a large local model. + +Evidence artifact, not a CI gate. ``OpenAICompatibleGrounder`` had only ever +been exercised against mocked HTTP (``tests/test_byo_grounding_model.py``) and +an in-process fake server (``tests/test_grounder_openai_compatible_contract.py``). +This probe closes the last gap: the same adapter, unmodified, pointed at a real +OpenAI-compatible endpoint serving a real vision model (e.g. Ollama's +``/v1``), on the same dense-EMR surface and ground truth as +``benchmark/grounding_eval`` — so the numbers are directly comparable with the +July baseline there (bespoke remote-VLM grounder: 0/6 @ ~472 px median on this +surface; OCR row-anchoring: 4/6 @ ~3 px). + +ENV-GATED — CI never needs a model and never runs this. Set: + + OPENADAPT_GROUNDER_BASE_URL e.g. http://127.0.0.1:11434/v1 (required) + OPENADAPT_GROUNDER_MODEL e.g. qwen3-vl:8b (required) + OPENADAPT_GROUNDER_API_KEY optional bearer token (loopback needs none) + +Run (a local run only inside a solo window): + + uv run python scripts/probe_local_grounder.py \ + [--out benchmark/local_grounder_probe/results.json] \ + [--price-in USD_PER_1M --price-out USD_PER_1M] [--max-spend USD] + +Surfaces and truth. Two deterministic dense-list surfaces, both rendered by +``openadapt_flow.validation.dense_surface`` (the repo's committed fixture +source — ``benchmark/dense_surface/record_seed1.png`` is the committed render +of the first): + +* ``record_seed1`` — ``render_frame(build_dense_table(seed=1, n_rows=18), + RECORD_CONDITION, top_offset_px=12)`` -> ~51 rows at 2240x3726 px. Identical + to benchmark/grounding_eval/harness.py, so numbers are directly comparable + with the July baseline (bespoke remote-VLM grounder: 0/6 @ ~472 px median; + OCR row-anchoring: 4/6 @ ~3 px). +* ``small_dense_seed2`` — ``render_frame(build_dense_table(seed=2, n_rows=18), + REPLAY_CONDITIONS small_dense, top_offset_px=12)`` -> a visually and + textually distinct screenshot (different names/MRNs, 12 px font, tighter + rows, device_scale_factor 1). + +Targets are the same deterministic spread on each surface +(``indices[::step][:6]``); truth is the DOM centre of each target row's Open +button (``frame.points[i][1]``). Requires the ``dev`` extra (playwright, with +its Chromium runtime installed) for the render. + +The probe records EXACTLY what the grounder returns: hits, misses, and +abstentions. It does not retry, downscale, crop, or massage. It also records +the token ``usage`` of every call (via an injected recording HTTP client — the +grounder itself is unmodified) and, when per-1M-token prices are given, the +estimated spend; it stops before a call that would break ``--max-spend``. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time +from pathlib import Path +from typing import Any + +TOL_STRICT = 40 # px — the grounding_eval REPORT.md headline tolerance +TOL_LOOSE = 60 # px — run_validation.run_grounder default +N_TARGETS = 6 # per surface; same deterministic spread as the July baseline +N_ROWS = 18 # floor; renders ~51 rows, exactly as the baseline harness + + +class _UsageRecordingClient: + """httpx-backed client that records each call's token ``usage``. + + Injected into ``OpenAICompatibleGrounder`` via its ``client`` parameter so + the adapter under test stays byte-for-byte the shipped one; this wrapper + only observes the response on the way through. + """ + + def __init__(self) -> None: + import httpx + + self._client = httpx.Client() + self.last_usage: dict = {} + + def post( + self, url: str, *, json: Any = None, headers: Any = None, timeout: Any = None + ): # noqa: ANN401,E501 + self.last_usage = {} + resp = self._client.post(url, json=json, headers=headers, timeout=timeout) + try: + usage = resp.json().get("usage") + if isinstance(usage, dict): + self.last_usage = { + "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": usage.get("completion_tokens"), + } + except ValueError: + pass + return resp + + +def _build_surfaces() -> list[tuple[str, Any, Any]]: + """Render the two probe surfaces; return (name, table, frame) triples.""" + from openadapt_flow.validation.dense_surface import ( + RECORD_CONDITION, + REPLAY_CONDITIONS, + build_dense_table, + render_frame, + ) + + small_dense = next(c for c in REPLAY_CONDITIONS if c.name == "small_dense") + out: list[tuple[str, Any, Any]] = [] + for name, seed, cond in ( + ("record_seed1", 1, RECORD_CONDITION), + ("small_dense_seed2", 2, small_dense), + ): + table = build_dense_table(seed=seed, n_rows=N_ROWS) + frame = render_frame(table, cond, top_offset_px=12) + vw, vh = frame.viewport + print(f"rendered surface {name}: {len(frame.points)} rows at {vw}x{vh} px") + out.append((name, table, frame)) + return out + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--out", + default="benchmark/local_grounder_probe/results.json", + help="where to write the raw per-target JSON", + ) + parser.add_argument( + "--price-in", + type=float, + default=0.0, + help="endpoint price in USD per 1M input tokens (0 = unknown/local)", + ) + parser.add_argument( + "--price-out", + type=float, + default=0.0, + help="endpoint price in USD per 1M output tokens (0 = unknown/local)", + ) + parser.add_argument( + "--max-spend", + type=float, + default=5.0, + help="hard cap in USD; the probe stops once the estimate reaches it", + ) + parser.add_argument( + "--max-tokens", + type=int, + default=256, + help=( + "completion-token budget per call (adapter default 256; raise for " + "a hosted reasoning model, which spends the budget on reasoning " + "before emitting the coordinate JSON)" + ), + ) + args = parser.parse_args() + + base_url = os.environ.get("OPENADAPT_GROUNDER_BASE_URL", "").strip() + model = os.environ.get("OPENADAPT_GROUNDER_MODEL", "").strip() + if not base_url or not model: + print( + "UNRUN: set OPENADAPT_GROUNDER_BASE_URL and OPENADAPT_GROUNDER_MODEL " + "to point at a served OpenAI-compatible vision model " + "(e.g. ollama serve -> http://127.0.0.1:11434/v1, qwen3-vl:8b)." + ) + return 2 + api_key = os.environ.get("OPENADAPT_GROUNDER_API_KEY", "") + + from openadapt_flow.runtime.grounder import OpenAICompatibleGrounder + + print(f"endpoint: {base_url} model: {model}") + surfaces = _build_surfaces() + + recording_client = _UsageRecordingClient() + grounder = OpenAICompatibleGrounder( + base_url=base_url, + model=model, + api_key=api_key, + timeout=600.0, + max_tokens=args.max_tokens, + client=recording_client, + ) + + spend_usd = 0.0 + capped = False + rows_out: list[dict] = [] + for surface_name, table, frame in surfaces: + indices = sorted(frame.points.keys()) + step = max(1, len(indices) // N_TARGETS) + chosen = indices[::step][:N_TARGETS] + for i in chosen: + if spend_usd >= args.max_spend: + capped = True + print( + f"SPEND CAP: estimate ${spend_usd:.2f} >= ${args.max_spend:.2f}; stopping." + ) + break + row = table.rows[i] + truth = frame.points[i][1] # DOM centre of the row's Open button + intent = f"click Open in the row for patient {row.name} (MRN {row.mrn})" + t0 = time.time() + match = grounder.locate(frame.png, intent, "Open") + latency = time.time() - t0 + usage = dict(recording_client.last_usage) + pt = usage.get("prompt_tokens") or 0 + ct = usage.get("completion_tokens") or 0 + call_usd = (pt * args.price_in + ct * args.price_out) / 1e6 + spend_usd += call_usd + rec = { + "surface": surface_name, + "row": i, + "name": row.name, + "mrn": row.mrn, + "truth": list(truth), + "latency_s": round(latency, 2), + "usage": usage, + "est_cost_usd": round(call_usd, 6), + } + if match is None: + rec.update( + proposed=None, + abstained=True, + error_px=None, + **{"hit@40": False, "hit@60": False}, + ) + print( + f"{surface_name} row {i:3d} {row.mrn} ABSTAIN " + f"({latency:.1f}s)" + ) + else: + px, py = match.point + err = math.dist((px, py), truth) + rec.update( + proposed=[px, py], + abstained=False, + error_px=round(err, 1), + **{"hit@40": err <= TOL_STRICT, "hit@60": err <= TOL_LOOSE}, + ) + print( + f"{surface_name} row {i:3d} {row.mrn} proposed ({px}, {py}) " + f"truth {truth} err {err:7.1f} px " + f"{'HIT ' if err <= TOL_STRICT else 'miss'} ({latency:.1f}s)" + ) + rows_out.append(rec) + if capped: + break + + n = len(rows_out) + hits40 = sum(r["hit@40"] for r in rows_out) + hits60 = sum(r["hit@60"] for r in rows_out) + abstained = sum(r["abstained"] for r in rows_out) + errors = sorted(r["error_px"] for r in rows_out if r["error_px"] is not None) + median_err = errors[len(errors) // 2] if errors else None + out = { + "meta": { + "endpoint": base_url, + "model": model, + "surfaces": { + name: {"viewport_px": list(frame.viewport)} + for name, _table, frame in surfaces + }, + "surface_spec": ( + "dense_surface.render_frame(build_dense_table(seed=S, n_rows=18), " + "COND, top_offset_px=12); record_seed1: S=1, COND=RECORD_CONDITION; " + "small_dense_seed2: S=2, COND=small_dense" + ), + "truth": "DOM centre of each row's Open button (frame.points[i][1])", + "tol_px": {"strict": TOL_STRICT, "loose": TOL_LOOSE}, + "adapter": "openadapt_flow.runtime.grounder.OpenAICompatibleGrounder", + "max_tokens": args.max_tokens, + "price_usd_per_1m": {"input": args.price_in, "output": args.price_out}, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + }, + "summary": { + "n_targets": n, + "hits@40": hits40, + "hits@60": hits60, + "abstained": abstained, + "median_error_px": median_err, + "est_spend_usd": round(spend_usd, 4), + "spend_capped": capped, + }, + "rows": rows_out, + } + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out, indent=2) + "\n") + print( + f"\nhit@40 {hits40}/{n} hit@60 {hits60}/{n} abstain {abstained}/{n} " + f"median err {median_err} px est spend ${spend_usd:.4f}\nwrote {out_path}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_hosted_probe.sh b/scripts/run_hosted_probe.sh new file mode 100755 index 00000000..5814d09f --- /dev/null +++ b/scripts/run_hosted_probe.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Run scripts/probe_local_grounder.py against a HOSTED OpenAI-compatible +# endpoint (default: Together AI) without ever exposing the API key. +# +# The key is sourced at RUNTIME from an operator-named env file and travels +# only via the environment into the probe process. This script never prints +# it; do not add any echo of the environment here. +# +# Usage: +# OPENADAPT_ENV_FILE=/path/to/.env \ +# scripts/run_hosted_probe.sh MODEL_ID OUT_JSON [PRICE_IN PRICE_OUT [MAX_TOKENS]] +# +# MODEL_ID e.g. Qwen/Qwen3.5-9B +# OUT_JSON e.g. benchmark/hosted_grounder_probe/results_qwen3_5_9b.json +# PRICE_IN USD per 1M input tokens (optional, for the spend estimate) +# PRICE_OUT USD per 1M output tokens (optional) +# MAX_TOKENS completion budget per call (optional; adapter default 256 — +# raise for a hosted reasoning model) +# +# The env file must define the key under the name in OPENADAPT_KEY_VAR +# (default: TOGETHERAI_API_KEY). +set -eu + +MODEL="${1:?usage: run_hosted_probe.sh MODEL_ID OUT_JSON [PRICE_IN PRICE_OUT [MAX_TOKENS]]}" +OUT="${2:?usage: run_hosted_probe.sh MODEL_ID OUT_JSON [PRICE_IN PRICE_OUT [MAX_TOKENS]]}" +PRICE_IN="${3:-0}" +PRICE_OUT="${4:-0}" +MAX_TOKENS="${5:-256}" + +ENV_FILE="${OPENADAPT_ENV_FILE:?set OPENADAPT_ENV_FILE to the .env file holding the API key}" +KEY_VAR="${OPENADAPT_KEY_VAR:-TOGETHERAI_API_KEY}" + +# Source the env file; auto-export everything it defines. +set -a +# shellcheck disable=SC1090 +. "$ENV_FILE" +set +a + +# Re-export the named key under the name the probe expects. eval only ever +# expands the VARIABLE NAME here; the value itself is never interpolated into +# a command line or printed. +eval "OPENADAPT_GROUNDER_API_KEY=\"\${${KEY_VAR}:-}\"" +export OPENADAPT_GROUNDER_API_KEY +if [ -z "$OPENADAPT_GROUNDER_API_KEY" ]; then + echo "error: $KEY_VAR is not defined in $ENV_FILE" >&2 + exit 1 +fi + +export OPENADAPT_GROUNDER_BASE_URL="${OPENADAPT_GROUNDER_BASE_URL:-https://api.together.xyz/v1}" +export OPENADAPT_GROUNDER_MODEL="$MODEL" + +exec uv run python scripts/probe_local_grounder.py \ + --out "$OUT" --price-in "$PRICE_IN" --price-out "$PRICE_OUT" \ + --max-tokens "$MAX_TOKENS" diff --git a/tests/test_grounder_max_tokens.py b/tests/test_grounder_max_tokens.py new file mode 100644 index 00000000..ce5aa0dc --- /dev/null +++ b/tests/test_grounder_max_tokens.py @@ -0,0 +1,102 @@ +"""``OpenAICompatibleGrounder.max_tokens`` — the completion-token budget knob. + +Added after the first hosted real-model run (2026-08-12, Together AI, +``benchmark/hosted_grounder_probe/``): the previously fixed ``max_tokens: 256`` +made the hosted reasoning model ``Qwen/Qwen3.5-9B`` truncate mid-reasoning with +EMPTY content on 11/12 dense-list targets — every one a (safe) abstain, but a +100% availability loss. The budget is now a constructor parameter; these tests +pin it without touching the existing wire contract +(``tests/test_grounder_openai_compatible_contract.py``): + +* the default stays exactly 256 (the historical wire value); +* an explicit budget rides the wire verbatim; +* a non-positive / non-int budget is rejected at construction (fail loud at + wiring time, not with a silent per-call 4xx => abstain at run time); +* SAFETY UNCHANGED: a truncated (empty-content) reply abstains — the knob can + raise availability, never risk. +""" + +from __future__ import annotations + +import json +from typing import Any, Optional + +import pytest + +from openadapt_flow.runtime.grounder import OpenAICompatibleGrounder + + +class _FakeResponse: + def __init__(self, payload: dict) -> None: + self.status_code = 200 + self._payload = payload + + def json(self) -> dict: + return self._payload + + +class _FakeClient: + """Minimal client capturing the request body (the wire payload).""" + + def __init__(self, content: str) -> None: + self._content = content + self.bodies: list[dict] = [] + + def post( + self, + url: str, + *, + json: Any = None, # noqa: A002 - mirrors httpx signature + headers: Any = None, + timeout: Any = None, + ) -> _FakeResponse: + self.bodies.append(json) + return _FakeResponse({"choices": [{"message": {"content": self._content}}]}) + + +def _locate(client: _FakeClient, **kwargs: Any) -> Optional[Any]: + g = OpenAICompatibleGrounder( + base_url="http://127.0.0.1:1/v1", model="m", client=client, **kwargs + ) + return g.locate(b"png", "click Open for patient X (MRN 1)", "Open") + + +class TestMaxTokensOnTheWire: + def test_default_budget_is_exactly_256(self) -> None: + client = _FakeClient('{"x": 1, "y": 2}') + assert _locate(client) is not None + assert client.bodies[0]["max_tokens"] == 256 + + def test_explicit_budget_rides_the_wire_verbatim(self) -> None: + client = _FakeClient('{"x": 1, "y": 2}') + assert _locate(client, max_tokens=2048) is not None + assert client.bodies[0]["max_tokens"] == 2048 + # And it is valid JSON on the wire, not merely a Python object. + assert json.loads(json.dumps(client.bodies[0]))["max_tokens"] == 2048 + + +class TestMaxTokensValidation: + @pytest.mark.parametrize("bad", [0, -1, -256]) + def test_non_positive_budget_is_rejected_at_construction(self, bad: int) -> None: + with pytest.raises(ValueError, match="max_tokens"): + OpenAICompatibleGrounder( + base_url="http://127.0.0.1:1/v1", model="m", max_tokens=bad + ) + + @pytest.mark.parametrize("bad", [256.0, "256", None, True]) + def test_non_int_budget_is_rejected_at_construction(self, bad: Any) -> None: + with pytest.raises(ValueError, match="max_tokens"): + OpenAICompatibleGrounder( + base_url="http://127.0.0.1:1/v1", model="m", max_tokens=bad + ) + + +class TestTruncationStaysFailSafe: + def test_truncated_empty_content_reply_abstains(self) -> None: + # The measured hosted failure shape: finish_reason=length, content "". + client = _FakeClient("") + assert _locate(client, max_tokens=1) is None + + def test_truncated_partial_json_reply_abstains(self) -> None: + client = _FakeClient('{"x": 12') # cut off mid-object by the budget + assert _locate(client, max_tokens=8) is None diff --git a/tests/test_grounder_openai_compatible_contract.py b/tests/test_grounder_openai_compatible_contract.py new file mode 100644 index 00000000..b9953df0 --- /dev/null +++ b/tests/test_grounder_openai_compatible_contract.py @@ -0,0 +1,283 @@ +"""Wire-protocol contract for ``OpenAICompatibleGrounder`` — real HTTP. + +``tests/test_byo_grounding_model.py`` covers the adapter against a *fake +client object*, which proves the parsing logic but never exercises the real +transport. This module stands up an in-process loopback HTTP server speaking +the OpenAI chat-completions shape (the same stdlib pattern as +``tests/test_effect_verifier_transport.py``) and drives the grounder through +its REAL default path — module-level ``httpx.post`` over TCP — so the exact +bytes on the wire are pinned: + +* the request is a well-formed ``POST {base_url}/chat/completions`` JSON body + carrying the ``model`` id, the screenshot as a ``data:image/png;base64,`` + URL that round-trips byte-for-byte, and a text prompt containing the target + intent and label; +* a valid coordinate reply resolves to exactly that point; +* a refusing, malformed, ``{"x": null}``, or non-JSON reply => None (abstain); +* an unreachable endpoint or non-2xx => None, never an exception; +* WEAKEST AUTHORITY: the model's reply can only ever *propose*. The resolver + files a grounder proposal on the ``grounder`` rung — the bottom of + ``RUNG_ORDER``, below ``ocr`` — so the risk gate (``is_below_ocr``) refuses + it for irreversible steps; the reply cannot raise its own confidence, add + fields, or reach any authority-bearing surface. + +CI-safe: loopback only, stdlib server, no model, no network egress. +""" + +from __future__ import annotations + +import base64 +import json +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Iterator + +import pytest + +from openadapt_flow.runtime.grounder import ( + GrounderMatch, + OpenAICompatibleGrounder, + component_may_egress, +) +from openadapt_flow.runtime.resolver import RUNG_ORDER, is_below_ocr + +# A tiny but real PNG (1x1 white pixel) so the data: URL carries genuine +# image/png bytes end to end. +PNG_1PX = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4" + "z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + +INTENT = "click Open in the row for patient Halloran, Karen MRN MG584224" +LABEL = "Open" + + +class _Server: + """Loopback OpenAI-compatible chat-completions endpoint. + + Records every request (path, headers, parsed JSON body) and returns the + configured status/body, so tests assert on what actually crossed the wire. + """ + + def __init__(self) -> None: + self.requests: list[dict[str, Any]] = [] + self.status = 200 + self.body: Any = {} + self.raw_body: bytes | None = None # overrides ``body`` when set + + outer = self + + class _Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length) + try: + parsed = json.loads(raw) + except ValueError: + parsed = None + outer.requests.append( + { + "path": self.path, + "headers": dict(self.headers), + "json": parsed, + } + ) + payload = ( + outer.raw_body + if outer.raw_body is not None + else json.dumps(outer.body).encode() + ) + self.send_response(outer.status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args: Any) -> None: + pass + + self._httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=self._httpd.serve_forever, daemon=True).start() + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self._httpd.server_address[1]}/v1" + + def reply(self, content: Any) -> None: + """Configure a 200 chat-completions reply with ``content``.""" + self.status = 200 + self.raw_body = None + self.body = {"choices": [{"message": {"content": content}}]} + + def close(self) -> None: + self._httpd.shutdown() + self._httpd.server_close() + + +@pytest.fixture() +def server() -> Iterator[_Server]: + srv = _Server() + try: + yield srv + finally: + srv.close() + + +def _grounder(server: _Server, **kwargs: Any) -> OpenAICompatibleGrounder: + kwargs.setdefault("timeout", 5.0) + return OpenAICompatibleGrounder( + base_url=server.base_url, model="qwen3-vl:8b", **kwargs + ) + + +# -------------------------------------------------------------------------- +# (a) The request on the wire is well-formed. +# -------------------------------------------------------------------------- + + +class TestRequestShape: + def test_request_is_a_wellformed_chat_completions_post( + self, server: _Server + ) -> None: + server.reply('{"x": 10, "y": 20}') + _grounder(server).locate(PNG_1PX, INTENT, LABEL) + + assert len(server.requests) == 1 + req = server.requests[0] + assert req["path"] == "/v1/chat/completions" + assert req["headers"]["Content-Type"].startswith("application/json") + + body = req["json"] + assert body["model"] == "qwen3-vl:8b" + assert isinstance(body["max_tokens"], int) + (message,) = body["messages"] + assert message["role"] == "user" + + parts = {p["type"]: p for p in message["content"]} + assert set(parts) == {"image_url", "text"} + + # The screenshot rides as a data: URL and round-trips byte-for-byte. + url = parts["image_url"]["image_url"]["url"] + prefix = "data:image/png;base64," + assert url.startswith(prefix) + assert base64.b64decode(url[len(prefix) :]) == PNG_1PX + + # The prompt names the target: intent and label both present. + assert INTENT in parts["text"]["text"] + assert LABEL in parts["text"]["text"] + + def test_no_api_key_sends_no_authorization_header(self, server: _Server) -> None: + server.reply('{"x": 1, "y": 2}') + _grounder(server).locate(PNG_1PX, INTENT, LABEL) + assert "Authorization" not in server.requests[0]["headers"] + + def test_api_key_rides_as_bearer_on_the_wire(self, server: _Server) -> None: + server.reply('{"x": 1, "y": 2}') + _grounder(server, api_key="sekret").locate(PNG_1PX, INTENT, LABEL) + assert server.requests[0]["headers"]["Authorization"] == "Bearer sekret" + + +# -------------------------------------------------------------------------- +# (b) A valid coordinate reply resolves to exactly that point. +# -------------------------------------------------------------------------- + + +class TestValidReply: + def test_coordinate_reply_resolves_to_the_point(self, server: _Server) -> None: + server.reply('{"x": 123, "y": 45}') + match = _grounder(server).locate(PNG_1PX, INTENT, LABEL) + assert isinstance(match, GrounderMatch) + assert match.point == (123, 45) + x0, y0, w, h = match.region + assert x0 <= 123 <= x0 + w and y0 <= 45 <= y0 + h + + def test_content_parts_list_shape_is_accepted(self, server: _Server) -> None: + server.reply([{"type": "text", "text": 'sure: {"x": 7, "y": 9}'}]) + match = _grounder(server).locate(PNG_1PX, INTENT, LABEL) + assert match is not None and match.point == (7, 9) + + +# -------------------------------------------------------------------------- +# (c) A refusing / malformed reply abstains — never a guessed point. +# -------------------------------------------------------------------------- + + +class TestBadReplyAbstains: + @pytest.mark.parametrize( + "content", + [ + "I cannot locate that control on this screen.", # NL refusal + '{"x": null, "y": null}', # explicit not-visible + '{"x": "twelve", "y": 4}', # non-numeric + '{"x": NaN, "y": 4}', # not JSON at all + "", # empty content + ], + ) + def test_unusable_content_abstains(self, server: _Server, content: str) -> None: + server.reply(content) + assert _grounder(server).locate(PNG_1PX, INTENT, LABEL) is None + + def test_non_json_response_body_abstains(self, server: _Server) -> None: + server.raw_body = b"gateway error" + assert _grounder(server).locate(PNG_1PX, INTENT, LABEL) is None + + def test_wrong_response_shape_abstains(self, server: _Server) -> None: + server.body = {"choices": []} + assert _grounder(server).locate(PNG_1PX, INTENT, LABEL) is None + + def test_non_200_abstains(self, server: _Server) -> None: + server.status = 503 + server.body = {"error": "overloaded"} + assert _grounder(server).locate(PNG_1PX, INTENT, LABEL) is None + + +# -------------------------------------------------------------------------- +# (d) Transport failure abstains — never a crash. +# -------------------------------------------------------------------------- + + +class TestTransportFailure: + def test_unreachable_endpoint_returns_none(self) -> None: + # Grab an ephemeral port with nothing listening on it. + probe = socket.socket() + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + probe.close() + g = OpenAICompatibleGrounder( + base_url=f"http://127.0.0.1:{port}/v1", model="m", timeout=2.0 + ) + assert g.locate(PNG_1PX, INTENT, LABEL) is None # and does not raise + + +# -------------------------------------------------------------------------- +# (e) Weakest authority: the reply can only ever PROPOSE. +# -------------------------------------------------------------------------- + + +class TestWeakestAuthority: + def test_reply_cannot_inflate_its_own_confidence(self, server: _Server) -> None: + # Even a reply claiming certainty and extra authority fields yields the + # fixed grounder confidence and nothing else. + server.reply('{"x": 5, "y": 6, "confidence": 0.99, "authorized": true}') + match = _grounder(server).locate(PNG_1PX, INTENT, LABEL) + assert match is not None + assert match.confidence == 0.5 + + def test_match_carries_no_authority_bearing_fields(self, server: _Server) -> None: + server.reply('{"x": 5, "y": 6}') + match = _grounder(server).locate(PNG_1PX, INTENT, LABEL) + assert match is not None + assert set(GrounderMatch.model_fields) == {"point", "region", "confidence"} + + def test_grounder_rung_is_bottom_of_the_ladder_and_risk_gated(self) -> None: + # The resolver files a grounder proposal on the "grounder" rung: the + # weakest evidence class. The risk gate refuses it for irreversible + # steps, so a model reply can never authorize an irreversible action. + assert RUNG_ORDER[-1] == "grounder" + assert is_below_ocr("grounder") is True + + def test_grounder_is_declared_egress_capable(self, server: _Server) -> None: + # The egress gate keys off MAY_EGRESS: the replayer refuses to wire + # this grounder unless the operator explicitly opts in. + assert component_may_egress(_grounder(server)) is True