feat: read the engine's unfulfilled-keys report instead of diffing declared vs delivered

packages/opencode/src/altimate/workspace/engine-overlay.ts

@@ -25,6 +25,7 @@
 // that turn's start.
 import { DATAMATE_KEY } from "@/altimate/datamate-transport"
 import { MCP } from "@/mcp"
+import { sanitize } from "@/mcp/catalog"
 import { Config } from "@/config/config"
 import {
   currentDirectory,
@@ -45,6 +46,8 @@ import {
   clearsFloor,
   describeExtensionServed,
   describeMissing,
+  parseUnfulfilled,
+  reportedMissing,
   describeRefusal,
   engineEntry,
   engineToolKeys,
@@ -338,6 +341,9 @@ function mcp() {
       add: (name: string, cfg: LocalMcpConfig | McpEntry) => MCP.add(name, cfg as Parameters<typeof MCP.add>[1]),
       remove: (name: string) => MCP.remove(name),
       tools: () => MCP.tools() as Promise<Record<string, unknown>>,
+      listMeta: (name: string) => MCP.listMeta(name),
+      snapshot: (name: string) =>
+        MCP.snapshot(name) as Promise<{ tools: Record<string, unknown>; meta: Record<string, unknown> | undefined }>,
     }
   )
 }
@@ -645,43 +651,81 @@ async function reconcile(sessionID: string, directory: string, state: DirectoryS
     return
   }
 
-  const present = engineToolKeys(await mcp().tools())
-  const missing = declared ? declared.keys.filter((k) => !present.has(k)) : undefined
+  // One read for both: a tools/list refresh that completes between two separate
+  // reads would pair one listing's tools with another's report. (multi-model review)
+  const { tools, meta } = await mcp().snapshot(DATAMATE_KEY)
+  const present = engineToolKeys(tools)
+  // The gaps come from the engine's own report, with reasons; this client no
+  // longer diffs the allowlist against what arrived. No report (nothing at or
+  // above the floor omits it) means no gap is claimed, not that there is none.
+  const unfulfilled = parseUnfulfilled(meta)
+  const missingReport = unfulfilled === undefined ? undefined : reportedMissing(unfulfilled)
+  const missing = missingReport?.map((u) => u.key)
   // `available` is everything the engine serves under the key. The engine adds
   // tools beyond the allowlist (knowledge, memory) when the workspace enables
   // them, so the "N of M declared" line counts only the declared ones present.
-  const served = declared ? declared.keys.length - (missing?.length ?? 0) : present.size
+  // Compared in the catalog's key space: `present` holds tool names as the MCP
+  // layer sanitised them (`[a-zA-Z0-9_-]`), while the declaration carries the
+  // raw keys, so a raw key with any other character would never count as served
+  // and the headline would disagree with a report that names no gap. (multi-model review)
+  // And never a key the engine itself reports as unfulfilled: two raw keys can
+  // sanitise to one catalog name, and the report is the authority on which of
+  // them the served tool stands for. (codex)
+  // And counted per catalog entry, not per declaration: two raw keys that both
+  // sanitise to `foo_bar` are one callable tool however many the engine lists.
+  // Consumed across both groups: an ordinary key and an extension key that
+  // collide are still one entry, counted where it is met first — with the
+  // ordinary keys, which are counted first.
+  const reported = new Set((unfulfilled ?? []).map((u) => u.key))
+  const consumed = new Set<string>()
+  const servedEntries = (keys: string[]) => {
+    let n = 0
+    for (const k of keys) {
+      const entry = sanitize(k)
+      if (!present.has(entry) || reported.has(k) || consumed.has(entry)) continue
+      consumed.add(entry)
+      n += 1
+    }
+    return n
+  }
+  const served = declared ? servedEntries(declared.keys) : present.size
   // Extension-declared tools appear in `present` only while the engine holds a
   // live IDE bridge; when they do they are real capability and the line names
   // them, but their absence is the normal no-IDE case, never `missing`.
-  const extServed = declared ? declared.extensionKeys.filter((k) => present.has(k)).length : 0
+  const extServed = declared ? servedEntries(declared.extensionKeys) : 0
   const outcome: Outcome = {
     kind: "attached",
     available: present.size,
-    ...(declared ? { declared: declared.keys.length, missing } : {}),
+    ...(declared ? { declared: declared.keys.length } : {}),
+    ...(missing === undefined ? {} : { missing }),
+    ...(unfulfilled === undefined ? {} : { unfulfilled }),
   }
   const rec = record(sessionID, outcome)
   // Keyed on the workspace too: a re-link with an identical inventory is still
   // a new verdict the user should hear.
   // extServed is part of what the user hears, so it is part of the signature:
   // an equal-count tool swap that changes only the extension share must still
   // re-announce. (bot review)
-  const signature = `attached:${workspace.key}:${outcome.available}:${outcome.declared ?? "?"}:${(missing ?? []).join(",")}:${extServed}`
+  // A gap whose reason changed (a connection fixed, a binary still absent)
+  // is a new verdict too, so the reasons are in the signature.
+  const gaps = (missingReport ?? []).map((u) => `${u.key}=${u.reason}`).join(",")
+  const signature = `attached:${workspace.key}:${outcome.available}:${outcome.declared ?? "?"}:${gaps}:${extServed}`
   if (rec.announced === signature) return
   rec.announced = signature
   log.info("workspace engine attached", {
     workspaceId: workspace.id,
     available: outcome.available,
     declared: outcome.declared,
-    missing,
+    unfulfilled,
   })
   if (isHeadless()) return
+  const headline = declared
+    ? `${served} of ${declared.keys.length} declared integration tools available.`
+    : `${outcome.available} integration tools available.`
   await notify({
     title: `Workspace "${workspace.name}"`,
-    message: declared
-      ? `${served} of ${declared.keys.length} declared integration tools available.${describeMissing(missing ?? [])}${describeExtensionServed(extServed)}`
-      : `${outcome.available} integration tools available.`,
-    variant: missing && missing.length > 0 ? "warning" : "info",
+    message: `${headline}${describeMissing(missingReport ?? [])}${describeExtensionServed(extServed)}`,
+    variant: missingReport !== undefined && missingReport.length > 0 ? "warning" : "info",
   })
 }
 

packages/opencode/src/altimate/workspace/engine-seams.ts

@@ -50,6 +50,8 @@ export const syncInternals: {
     add: (name: string, cfg: LocalMcpConfig | McpEntry) => Promise<unknown>
     remove: (name: string) => Promise<unknown>
     tools: () => Promise<Record<string, unknown>>
+    listMeta: (name: string) => Promise<Record<string, unknown> | undefined>
+    snapshot: (name: string) => Promise<{ tools: Record<string, unknown>; meta: Record<string, unknown> | undefined }>
   }
   config?: {
     invalidate: () => Promise<void>

packages/opencode/src/altimate/workspace/engine-types.ts

@@ -14,9 +14,11 @@ import { DATAMATE_KEY } from "@/altimate/datamate-transport"
  * workspace promise rests on: integrations configured purely in the workspace
  * UI must produce working tools with no local files. It also passes the
  * resolved connection to MCP-type handlers, so their credential placeholders
- * resolve. A 0.7.0 engine holds the pin but serves none of those tools, which
- * is why the floor is 0.7.1. */
-export const MIN_ENGINE_VERSION = "0.7.1"
+ * resolve. A 0.7.0 engine holds the pin but serves none of those tools. 0.7.2
+ * is the first that reports, on every tools/list, the allowlist keys it could
+ * not serve and why (`UNFULFILLED_META_KEY`); this client no longer diffs the
+ * allowlist itself, so below 0.7.2 it would announce no gaps at all. */
+export const MIN_ENGINE_VERSION = "0.7.2"
 export const ENGINE_PACKAGE = "@altimateai/datamate"
 export const ENGINE_BINARY = "datamate"
 export const INSTALL_COMMAND = `npm i -g ${ENGINE_PACKAGE}@${MIN_ENGINE_VERSION}`
@@ -29,7 +31,16 @@ export const TOOL_PREFIX = `${DATAMATE_KEY}_`
 export type Outcome =
   | { kind: "disabled" }
   | { kind: "unbound" }
-  | { kind: "attached"; available: number; declared?: number; missing?: string[] }
+  | {
+      kind: "attached"
+      available: number
+      declared?: number
+      /** Keys of `unfulfilled` that count as gaps (see `reportedMissing`). */
+      missing?: string[]
+      /** The engine's full report, `no-bridge` entries included; absent when
+       * the engine sent none. */
+      unfulfilled?: Unfulfilled[]
+    }
   | { kind: "engine-missing"; declared?: number }
   /** `found` is null when the binary ran but printed nothing usable — broken
    * rather than old; the message says so. */
@@ -207,11 +218,102 @@ export function describeRefusal(
   )
 }
 
-export function describeMissing(missing: string[]): string {
+/** Where the engine (0.7.2+) reports the allowlist keys it could not serve,
+ * on every tools/list response, so the client never diffs the allowlist
+ * against what arrived: a diff can name the keys, never the reason. */
+export const UNFULFILLED_META_KEY = "ai.altimate/unfulfilled"
+
+export type UnfulfilledReason =
+  | "catalog-missing"
+  | "invalid-connection"
+  | "spawn-failed"
+  | "no-bridge"
+  | "unknown-key"
+  | "exception"
+
+/** One declared key the engine did not serve, in the engine's own words. A
+ * reason outside the known set is kept verbatim: a newer engine may add one. */
+export type Unfulfilled = {
+  key: string
+  integrationId: string
+  reason: UnfulfilledReason | (string & {})
+  detail?: string
+}
+
+/** The engine's report out of a tools/list `_meta`. Undefined when there is
+ * none, or it is malformed: the caller then knows nothing about gaps, which
+ * is not the same as knowing there are none. */
+export function parseUnfulfilled(meta: Record<string, unknown> | undefined): Unfulfilled[] | undefined {
+  const raw = meta?.[UNFULFILLED_META_KEY]
+  if (!Array.isArray(raw)) return undefined
+  const out: Unfulfilled[] = []
+  for (const item of raw) {
+    if (typeof item !== "object" || item === null) return undefined
+    const { key, integrationId, reason, detail } = item as Record<string, unknown>
+    // Custom (tenant-created) integrations carry numeric ids; take them as strings.
+    const id = typeof integrationId === "number" ? String(integrationId) : integrationId
+    if (typeof key !== "string" || typeof id !== "string" || typeof reason !== "string") return undefined
+    // A present `detail` must be a string: an entry with a malformed one is a
+    // malformed report, not a report with one field dropped. Fails closed like
+    // the fields above. (codex)
+    if (detail !== undefined && typeof detail !== "string") return undefined
+    out.push({ key, integrationId: id, reason, ...(detail ? { detail } : {}) })
+  }
+  return out
+}
+
+/** Absent extension tools without an IDE window are expected, not missing:
+ * `no-bridge` entries never join the "declared but not available" line.
+ * Everything else the engine reports is a real gap. */
+export function reportedMissing(unfulfilled: Unfulfilled[]): Unfulfilled[] {
+  return unfulfilled.filter((u) => u.reason !== "no-bridge")
+}
+
+const REASON_PHRASE: Record<UnfulfilledReason, string> = {
+  "invalid-connection": "no usable connection",
+  // The engine records transport construction, connect AND list failures under
+  // this one reason, so the phrase must not claim more than "could not be reached".
+  "spawn-failed": "server could not be started or reached",
+  "catalog-missing": "no longer in the catalog",
+  "unknown-key": "not offered by the integration",
+  exception: "failed to load",
+  "no-bridge": "needs a VS Code window",
+}
+
+const MISSING_SHOWN = 5
+const DETAIL_CHARS = 60
+
+/** The gaps, grouped by reason AND integration in report order, at most
+ * `MISSING_SHOWN` keys across the groups; a group's first detail (the engine's
+ * error text, e.g. `spawn docker ENOENT`) stands for the group. Grouped per
+ * integration so one integration's error is never printed as another's — two
+ * servers that both failed to start failed for their own reasons. (multi-model review) */
+export function describeMissing(missing: Unfulfilled[]): string {
   if (missing.length === 0) return ""
-  const shown = missing.slice(0, 5).join(", ")
-  const more = missing.length > 5 ? ` (+${missing.length - 5} more)` : ""
-  return ` Declared but not available: ${shown}${more}.`
+  const groups = new Map<string, { reason: string; keys: string[]; detail?: string }>()
+  for (const u of missing) {
+    const id = `${u.reason}${u.integrationId}`
+    const group = groups.get(id) ?? { reason: u.reason, keys: [] }
+    group.keys.push(u.key)
+    if (group.detail === undefined && u.detail) group.detail = u.detail
+    groups.set(id, group)
+  }
+  let budget = MISSING_SHOWN
+  const parts: string[] = []
+  for (const { reason, ...group } of groups.values()) {
+    if (budget <= 0) break
+    const shown = group.keys.slice(0, budget)
+    budget -= shown.length
+    const phrase = (REASON_PHRASE as Record<string, string>)[reason] ?? reason
+    const detail = group.detail === undefined ? "" : ` (${truncate(group.detail, DETAIL_CHARS)})`
+    parts.push(`${phrase}${detail}: ${shown.join(", ")}`)
+  }
+  const more = missing.length > MISSING_SHOWN ? ` (+${missing.length - MISSING_SHOWN} more)` : ""
+  return ` Declared but not available — ${parts.join("; ")}${more}.`
+}
+
+function truncate(text: string, max: number): string {
+  return text.length <= max ? text : `${text.slice(0, max - 1)}…`
 }
 
 /** Extension-declared tools a connected IDE bridge is actually serving. Zero

packages/opencode/src/mcp/catalog.ts

@@ -15,6 +15,22 @@ import z from "zod/v4"
 const DEFAULT_TIMEOUT = 30_000
 const MAX_LIST_PAGES = 1_000
 
+// altimate_change start — keep the `_meta` of a server's last tools/list.
+// `paginate` keeps only each page's items, so the result object — the sole
+// carrier of `_meta` — is dropped. The workspace engine reports the allowlist
+// keys it could not serve there (altimate/workspace/engine-types). Kept per
+// client and committed only when a listing COMPLETES: the last page that
+// carries a `_meta` wins, a listing with none clears it, and a listing that
+// is still pending or that failed leaves the previous value standing — so the
+// tools and their report, which the caller commits together, never describe
+// two different listings. (multi-model review)
+const listMetaByClient = new WeakMap<Client, Record<string, unknown>>()
+
+export function listMeta(client: Client): Record<string, unknown> | undefined {
+  return listMetaByClient.get(client)
+}
+// altimate_change end
+
 // altimate_change start — Microsoft Fabric Core MCP returns `null` (instead of
 // omitting the field) for `tool.annotations.{readOnlyHint,destructiveHint,
 // idempotentHint,openWorldHint}`, which the SDK's strict schema (boolean,
@@ -58,9 +74,18 @@ export async function paginate<T, R extends { nextCursor?: string }>(
   throw new Error(`MCP list exceeded ${MAX_LIST_PAGES} pages`)
 }
 
+// altimate_change start — `defs` is the tools half of `defsWithMeta`: a listing
+// and its own `_meta` as one value. The caller commits the pair; reading the
+// per-client `listMeta` after the fact could hand it another listing's `_meta`
+// when two refreshes overlap. (codex)
 export function defs(client: Client, timeout?: number) {
+  return defsWithMeta(client, timeout).pipe(Effect.map((listing) => listing?.tools))
+}
+
+export function defsWithMeta(client: Client, timeout?: number) {
   return listTools(client, timeout ?? DEFAULT_TIMEOUT).pipe(Effect.catch(() => Effect.void))
 }
+// altimate_change end
 
 export function convertTool(mcpTool: MCPToolDef, client: Client, timeout?: number): Tool {
   const inputSchema: JSONSchema7 = {
@@ -150,8 +175,11 @@ export function resources(client: Client, timeout?: number) {
 
 function listTools(client: Client, timeout: number) {
   return Effect.tryPromise({
-    try: () =>
-      paginate(
+    // altimate_change start — `_meta` is committed with the completed listing (see listMeta).
+    try: async () => {
+      let meta: Record<string, unknown> | undefined
+      const tools = await paginate(
+        // altimate_change end
         async (cursor) => {
           const params = cursor === undefined ? undefined : { cursor }
           try {
@@ -169,8 +197,17 @@ function listTools(client: Client, timeout: number) {
             // altimate_change end
           }
         },
-        (result) => result.tools,
-      ),
+        // altimate_change start — the last page that carries a `_meta` wins.
+        (result) => {
+          if (result._meta !== undefined) meta = result._meta as Record<string, unknown>
+          return result.tools
+        },
+      )
+      if (meta === undefined) listMetaByClient.delete(client)
+      else listMetaByClient.set(client, meta)
+      return { tools, meta }
+    },
+    // altimate_change end
     catch: (error) => (error instanceof Error ? error : new Error(String(error))),
   })
 }