From c2f2b1a2adf8616c99426ac96cff2b46782fcac2 Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:03:46 +0000 Subject: [PATCH 1/3] Improve agent feedback signal --- README.md | 3 +- src/lib/mcp/analytics.test.ts | 159 ++++++++++++- src/lib/mcp/analytics.ts | 291 +++++++++++++++--------- src/lib/mcp/register.test.ts | 15 ++ src/lib/mcp/tool-names.ts | 46 ++++ src/lib/mcp/tools/feedback.test.ts | 116 ++++++++++ src/lib/mcp/tools/feedback.ts | 125 +++++++++- src/lib/mcp/tools/missing-capability.ts | 191 ++++++++++++++++ 8 files changed, 828 insertions(+), 118 deletions(-) create mode 100644 src/lib/mcp/tool-names.ts create mode 100644 src/lib/mcp/tools/missing-capability.ts diff --git a/README.md b/README.md index e15b2c6..4c350dd 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,8 @@ See [Vault payments](docs/vault-payments.md) for both provider flows, safety rul - `webmcp` - List native page tools across every tab and frame in a browser, then synchronously invoke an exact opaque `tool_ref` with structured input. - `exec_command` - Run shell commands inside a browser VM. Returns decoded stdout/stderr. - `search_docs` - Search Kernel platform documentation and guides. -- `submit_feedback` - send product, bot-detection, config-registry, mcp, or documentation feedback directly to the KERNEL team without interrupting the current task. Config-registry reports connect the observed site outcome to the browser session, recommendation metadata and evidence, and exact browser and proxy settings applied unchanged; general bot-detection reports remain available for outcomes not tied to a registry recommendation. +- `get_more_tools` - Report a structured KERNEL capability or external-integration gap after checking the available tools. Existing-tool failures, transient capacity errors, and client permission restrictions are rejected from capability-demand analytics. Accepted requests emit `mcp_capability_requested`; historical unstructured requests remain under `$mcp_missing_capability`. +- `submit_feedback` - Send product, bot-detection, config-registry, MCP, or documentation feedback directly to the KERNEL team without interrupting the current task. Reports include a normalized task outcome; MCP reports identify one KERNEL-owned tool. Config-registry reports connect exactly one observed outcome to the browser session, recommendation metadata and evidence, and unchanged browser and proxy settings. - `open_auth_login` - Open a secure interactive Managed Auth MCP App after user consent. Registered only for clients that declare MCP Apps support; credentials and MFA never enter MCP/model traffic. ## Resources diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index d478fcd..3ad2e84 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -11,10 +11,12 @@ import { import { captureMcpConnectionScopeFailure, captureMcpFeedback, + captureMissingCapabilityReport, captureOAuthTokenExchange, clientCapabilityAnalyticsFromInitialize, enrichMcpAnalyticsEvent, instrumentMcpAnalytics, + MCP_CAPABILITY_REQUESTED_EVENT, MCP_CLIENT_ELICITATION_MODE_PROPERTY, MCP_CLIENT_SUPPORTS_APPS_PROPERTY, MCP_CLIENT_SUPPORTS_ENTERPRISE_AUTH_PROPERTY, @@ -370,10 +372,10 @@ describe("sanitizeMcpAnalyticsEvent", () => { expect(result?.properties.$set).toBeUndefined(); }); - test("redacts emails, URLs, and tokens from intent", async () => { + test("redacts identifiable values from analytics text", async () => { const event = toolCallEvent({ [PostHogMCPAnalyticsProperty.Intent]: - "Checking out on https://shop.example.com/cart for buyer@example.com with key sk_abc123DEF456", + "Checking /tmp/private/cart.json on shop.example.com at 192.0.2.1 for buyer@example.com using https://shop.example.com/cart and key sk_abc123DEF456", }); const result = await sanitizeMcpAnalyticsEvent(event); @@ -384,8 +386,14 @@ describe("sanitizeMcpAnalyticsEvent", () => { expect(intent).toContain("[url]"); expect(intent).toContain("[email]"); expect(intent).toContain("[token]"); + expect(intent).toContain("[domain]"); + expect(intent).toContain("[ip]"); + expect(intent).toContain("[path]"); expect(intent).not.toContain("buyer@example.com"); expect(intent).not.toContain("sk_abc123DEF456"); + expect(intent).not.toContain("shop.example.com"); + expect(intent).not.toContain("192.0.2.1"); + expect(intent).not.toContain("/tmp/private/cart.json"); }); test("deletes non-string intents", async () => { @@ -576,6 +584,59 @@ describe("captureMcpConnectionScopeFailure", () => { }); }); +describe("captureMissingCapabilityReport", () => { + test("routes structured Kernel demand and redacts sensitive text", async () => { + const captured: unknown[] = []; + const analytics = { + capture: async (event: unknown) => { + captured.push(event); + }, + } as McpAnalytics; + + await captureMissingCapabilityReport( + { + context: + "Uploading /tmp/private/image.png to files.example.com requires a browser filesystem transfer capability.", + gap_reason: "kernel_capability_missing", + capability_area: "browser_files", + capability: "browser filesystem upload", + requested_action: "transfer", + task_outcome: "blocked", + tools_checked: ["manage_browsers"], + }, + { + authInfo: { + extra: { + connectionContext: { + scope: { organizationId: "org_analytics" }, + }, + }, + }, + }, + analytics, + ); + + expect(captured).toEqual([ + { + event: MCP_CAPABILITY_REQUESTED_EVENT, + properties: expect.objectContaining({ + $groups: { organization: "org_analytics" }, + [PostHogMCPAnalyticsProperty.Intent]: + "Uploading [path] to [domain] requires a browser filesystem transfer capability.", + missing_capability_gap_reason: "kernel_capability_missing", + missing_capability_destination: "kernel_product_demand", + missing_capability_area: "browser_files", + missing_capability_name: "browser filesystem upload", + missing_capability_requested_action: "transfer", + missing_capability_task_outcome: "blocked", + missing_capability_tools_checked: ["manage_browsers"], + missing_capability_privacy_redacted: true, + }), + }, + ]); + }); +}); + describe("captureMcpFeedback", () => { test("routes redacted feedback through contextual MCP analytics", async () => { const captured: unknown[] = []; @@ -619,9 +680,11 @@ describe("captureMcpFeedback", () => { feedback_type: "product", feedback_sentiment: "mixed", feedback_product_area: "browsers", - feedback_destination: undefined, + feedback_destination: "product_feedback", feedback_category: undefined, + feedback_task_outcome: "completed", feedback_task_completed: true, + feedback_privacy_redacted: true, feedback_tools_used: ["manage_browsers"], feedback_friction_points: "- The response did not say when to retry.", feedback_suggested_improvement: @@ -819,14 +882,46 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { await enabled.client.listTools() ).tools.find(({ name }) => name === "get_more_tools"); expect(missingCapabilityTool?.description).toContain( - "only after checking the available tools", + "after checking the tool list", ); expect(missingCapabilityTool?.description).toContain( - "transient failure or capacity limit", + "transient or capacity failure", ); expect(missingCapabilityTool?.description).toContain( "client-side permission restriction", ); + expect(missingCapabilityTool?.inputSchema.required).toEqual([ + "context", + "gap_reason", + "capability_area", + "capability", + "requested_action", + "task_outcome", + ]); + const disabledMissingCapabilityTool = ( + await disabled.client.listTools() + ).tools.find(({ name }) => name === "get_more_tools"); + expect(disabledMissingCapabilityTool?.inputSchema).toEqual( + missingCapabilityTool?.inputSchema, + ); + + const unavailableRequest = await disabled.client.callTool({ + name: "get_more_tools", + arguments: { + context: + "Transferring a binary into a browser requires a filesystem upload capability that no listed tool provides.", + gap_reason: "kernel_capability_missing", + capability_area: "browser_files", + capability: "browser filesystem upload", + requested_action: "transfer", + task_outcome: "blocked", + tools_checked: ["manage_browsers"], + }, + }); + expect(toolResultJSON(unavailableRequest)).toMatchObject({ + recorded: false, + status: "unavailable", + }); const result = await disabled.client.callTool({ name: KERNEL_FEEDBACK_TOOL_NAME, @@ -836,6 +931,9 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { summary: "Feedback analytics are unavailable", feedback_type: "mcp", sentiment: "negative", + task_outcome: "blocked", + affected_tool: "submit_feedback", + category: "tool_correctness", }, }); expect(toolResultJSON(result)).toMatchObject({ @@ -899,9 +997,10 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { )._requestHandlers; const handler = handlers.get(method); if (!handler) throw new Error(`no handler registered for ${method}`); - await handler(request, extra); + const result = await handler(request, extra); // The SDK's event sink captures fire-and-forget; give it a tick to flush. await new Promise((resolve) => setTimeout(resolve, 50)); + return result; } test("attributes initialize, tools/list, and tools/call to the organization", async () => { @@ -963,6 +1062,54 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { expect(byEvent.has("$identify")).toBe(false); }); + test("captures only structured capability demand", async () => { + const captured: { event?: string }[] = []; + + const capabilityResult = await simulateRequest(captured, "tools/call", { + name: "get_more_tools", + arguments: { + context: + "Transferring a local binary into a browser requires a filesystem upload capability that no listed tool provides.", + gap_reason: "kernel_capability_missing", + capability_area: "browser_files", + capability: "browser filesystem upload", + requested_action: "transfer", + task_outcome: "blocked", + tools_checked: ["kernel__manage_browsers"], + }, + }); + await simulateRequest(captured, "tools/call", { + name: "get_more_tools", + arguments: { + context: + "Retrying an existing browser tool after a capacity failure does not require a new server capability.", + gap_reason: "transient_or_capacity_failure", + capability_area: "browsers", + capability: "browser creation", + requested_action: "create", + task_outcome: "blocked", + tools_checked: ["manage_browsers"], + }, + }); + + expect(capabilityResult).toBeDefined(); + const requests = captured.filter( + ({ event }) => event === MCP_CAPABILITY_REQUESTED_EVENT, + ) as { properties: Record }[]; + expect(requests).toHaveLength(1); + expect(requests[0]?.properties).toMatchObject({ + $groups: { organization: ORG }, + [PostHogMCPAnalyticsProperty.SessionId]: "ses_integration", + missing_capability_gap_reason: "kernel_capability_missing", + missing_capability_destination: "kernel_product_demand", + missing_capability_area: "browser_files", + missing_capability_name: "browser filesystem upload", + missing_capability_requested_action: "transfer", + missing_capability_task_outcome: "blocked", + missing_capability_tools_checked: ["manage_browsers"], + }); + }); + test("captures feedback with the surrounding MCP session metadata", async () => { const captured: { event?: string }[] = []; diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index 6ca8360..7a277dd 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -1,5 +1,5 @@ +import { createHash } from "node:crypto"; import { - getMoreToolsResult, instrument, PostHogMCPAnalyticsEvent, PostHogMCPAnalyticsProperty, @@ -8,7 +8,6 @@ import { } from "@posthog/mcp"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { PostHog } from "posthog-node"; -import { z } from "zod"; import type { McpConnectionAnalyticsContext, McpConnectionContext, @@ -18,6 +17,10 @@ import { type KernelFeedback, registerFeedbackTool, } from "@/lib/mcp/tools/feedback"; +import { + type MissingCapabilityReport, + registerMissingCapabilityTool, +} from "@/lib/mcp/tools/missing-capability"; import { clientDeclaresExtension, clientElicitationModes, @@ -67,6 +70,7 @@ export type McpConnectionScopeFailureAnalytics = { export const MCP_CONNECTION_SCOPE_FAILURE_EVENT = "mcp_connection_scope_failure"; export const MCP_FEEDBACK_SUBMITTED_EVENT = "mcp_feedback_submitted"; +export const MCP_CAPABILITY_REQUESTED_EVENT = "mcp_capability_requested"; if (!projectToken && process.env.NODE_ENV !== "production") { console.error( @@ -163,6 +167,10 @@ const SENT_PROPERTIES = new Set([ "feedback_summary", "feedback_type", "feedback_sentiment", + "feedback_task_outcome", + "feedback_affected_tool", + "feedback_dedupe_key", + "feedback_privacy_redacted", "feedback_product_area", "feedback_destination", "feedback_bot_detection_registrable_domain", @@ -200,25 +208,34 @@ const SENT_PROPERTIES = new Set([ "feedback_suggested_improvement", "feedback_user_request", "feedback_details", + "missing_capability_gap_reason", + "missing_capability_destination", + "missing_capability_area", + "missing_capability_name", + "missing_capability_requested_action", + "missing_capability_task_outcome", + "missing_capability_tools_checked", + "missing_capability_dedupe_key", + "missing_capability_privacy_redacted", ]); -// Intent is the only free-form text this captures, and an agent writes it. Long enough for -// the 15-25 words asked for, short enough that a client ignoring the instruction can't -// stream a payload or a prompt into an event property. +// Free-form analytics text is agent-written. Intent stays long enough for the 15-25 words +// requested by the schema but short enough that a client ignoring the instruction cannot +// stream a payload or prompt into one event property. const INTENT_MAX_LENGTH = 300; -// The intent descriptions ask agents to leave specifics out, and the agent writing the string -// is the only thing holding them to it. These cover the shapes that are unambiguous when one -// does slip through. They are not a substitute for the instruction: no pattern can tell that a -// plain noun is a customer's name, so an intent still has to be treated as agent-written prose. -// -// A capability gap is often named as a long snake_case or kebab-case identifier, and that name -// is the whole point of the report, so length alone can't stand in for a credential. What marks -// one is either a vendor prefix (Kernel API keys are sk_*) or an unbroken high-entropy run: -// mixed case with a digit, or hex. Separator-joined lowercase names match none of those. +// Instructions remain the first privacy boundary, but feedback has enough free-form fields +// that recognizable identifiers must also be removed server-side. Plain organization and +// person names cannot be identified reliably without false positives, so schemas still tell +// agents to anonymize them. const INTENT_REDACTIONS: readonly [RegExp, string][] = [ [/[^\s@]+@[^\s@]+\.[^\s@]+/g, "[email]"], [/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "[url]"], + [/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, "[ip]"], + [/\b(?=[A-F0-9:]*[A-F])(?:[A-F0-9]{1,4}:){2,}[A-F0-9:]{1,}\b/gi, "[ip]"], + [/(?) { properties[MCP_USED_PROJECT_PROPERTY] = hasNonEmptyParam(args, "project"); } +function redactAnalyticsTextWithStatus(text: string) { + let value = text.trim(); + let redacted = false; + for (const [pattern, replacement] of INTENT_REDACTIONS) { + const next = value.replace(pattern, replacement); + redacted ||= next !== value; + value = next; + } + return { value, redacted }; +} + function redactAnalyticsText(text: string) { - return INTENT_REDACTIONS.reduce( - (redacted, [pattern, replacement]) => - redacted.replace(pattern, replacement), - text.trim(), - ); + return redactAnalyticsTextWithStatus(text).value; } function sanitizeIntent(intent: string) { return redactAnalyticsText(intent).slice(0, INTENT_MAX_LENGTH); } -// Must stay the SDK's default name: reportMissing advertises a tool under this name and -// dispatches calls to it as capability reports, and the name is what ties the two together. -const MISSING_CAPABILITY_TOOL_NAME = "get_more_tools"; - -const MISSING_CAPABILITY_CONTEXT_DESCRIPTION = - "The capability that is missing and what the user was trying to do, in 15-25 words, third " + - "person. Used for product analytics. Name the capability, not the specifics: never include " + - "credentials, tokens, URLs, domain or account names, file contents, or personal data. " + - 'Example: "Wanted to run one automation across several sessions at once; no tool exposes ' + - 'fan-out over a pool."'; - -/** - * Advertises `get_more_tools`, which agents call when no tool covers what they were asked - * to do. Registered here rather than left to `reportMissing` alone, which injects its own - * descriptor only when no tool of this name exists: the SDK asks for "a description of - * your goal" and skips the `context` description configured below, and this is the one - * intent field describing the user's original ask, so it's the likeliest to carry a target - * site or an account. The tool description itself is the SDK's — it's what gets an agent to - * report a gap instead of giving up. - * - * The callback is the fallback path. `instrument` answers a call to this tool itself, with - * this same result, and records `$mcp_missing_capability` instead of a tool call. - */ -function registerMissingCapabilityTool(server: McpServer) { - server.tool( - MISSING_CAPABILITY_TOOL_NAME, - "Report a genuine server capability gap only after checking the available tools and confirming none can complete the task. Do not call this for an existing fallback, a transient failure or capacity limit, or a client-side permission restriction; use the available tool or submit_feedback instead.", - { - context: z.string().describe(MISSING_CAPABILITY_CONTEXT_DESCRIPTION), - }, - { - title: "Get more tools", - readOnlyHint: true, - destructiveHint: false, - idempotentHint: true, - openWorldHint: true, - }, - async () => getMoreToolsResult(), - ); -} - const ANALYTICS_CONTEXT_PROPERTY = "__mcp_connection_analytics_context"; /** @@ -436,6 +419,54 @@ export function captureMcpCustomEvent( }); } +function analyticsDedupeKey(parts: (string | undefined)[]) { + return createHash("sha256") + .update(parts.filter(Boolean).join(":")) + .digest("hex") + .slice(0, 16); +} + +export function captureMissingCapabilityReport( + report: MissingCapabilityReport, + extra: unknown, + analytics: McpAnalytics, +) { + const context = report.context + ? redactAnalyticsTextWithStatus(report.context) + : { value: "", redacted: false }; + const capability = redactAnalyticsTextWithStatus(report.capability); + const destination = + report.gap_reason === "kernel_capability_missing" + ? "kernel_product_demand" + : "external_integration_demand"; + + return captureMcpCustomEvent( + analytics, + extra, + MCP_CAPABILITY_REQUESTED_EVENT, + { + [PostHogMCPAnalyticsProperty.Intent]: ( + context.value || capability.value + ).slice(0, INTENT_MAX_LENGTH), + missing_capability_gap_reason: report.gap_reason, + missing_capability_destination: destination, + missing_capability_area: report.capability_area, + missing_capability_name: capability.value, + missing_capability_requested_action: report.requested_action, + missing_capability_task_outcome: report.task_outcome, + missing_capability_tools_checked: report.tools_checked, + missing_capability_dedupe_key: analyticsDedupeKey([ + destination, + report.capability_area, + report.requested_action, + capability.value.toLowerCase(), + ]), + missing_capability_privacy_redacted: + context.redacted || capability.redacted, + }, + ); +} + export function enrichMcpAnalyticsEvent(event: { event: string; distinct_id: string; @@ -573,44 +604,96 @@ export function captureMcpFeedback( feedback.feedback_type === "config_registry" ? feedback.config_registry : undefined; + + let privacyRedacted = false; + const safeText = (value: string | undefined) => { + if (value === undefined) return undefined; + const sanitized = redactAnalyticsTextWithStatus(value); + privacyRedacted ||= sanitized.redacted; + return sanitized.value; + }; + + const summary = safeText(feedback.summary)!; + const productArea = safeText(feedback.product_area); + const suspectedVendor = safeText(botDetection?.suspected_vendor); + const region = safeText(botDetection?.region); + const browserVersion = safeText(botDetection?.browser_version); + const browserImageVersion = safeText(botDetection?.browser_image_version); + const browserSessionId = safeText(botDetection?.browser_session_id); + const analysisId = safeText(configRegistry?.analysis_id); + const toolsUsed = feedback.tools_used?.map((tool) => safeText(tool)!); + const frictionPoints = safeText(feedback.friction_points); + const suggestedImprovement = safeText(feedback.suggested_improvement); + const userRequest = safeText(feedback.user_request); + const details = safeText(feedback.details); + + const taskOutcome = + feedback.task_outcome ?? + (feedback.task_completed === true + ? "completed" + : feedback.task_completed === false + ? "blocked" + : "unknown"); + const destination = configRegistry + ? "config_registry_quality" + : botDetection + ? "config_registry_prioritization" + : feedback.feedback_type === "mcp" + ? feedback.category === "missing_tool" + ? "legacy_capability_feedback" + : feedback.affected_tool + ? "mcp_quality" + : "mcp_unclassified" + : feedback.feedback_type === "product" && + feedback.sentiment === "positive" + ? "product_praise" + : feedback.feedback_type === "product" && !productArea + ? "product_unclassified" + : `${feedback.feedback_type}_feedback`; + const dedupeKey = configRegistry + ? analyticsDedupeKey([ + "config_registry", + analysisId ?? configRegistry.request_method, + botDetection?.registrable_domain, + ]) + : botDetection + ? analyticsDedupeKey([ + "bot_detection", + botDetection.registrable_domain, + botDetection.observed_outcome, + botDetection.reproducibility, + ]) + : analyticsDedupeKey([ + feedback.feedback_type, + feedback.affected_tool, + productArea, + feedback.category, + summary.toLowerCase().replace(/[^a-z0-9]+/g, " "), + ]); + return captureMcpCustomEvent(analytics, extra, MCP_FEEDBACK_SUBMITTED_EVENT, { - feedback_summary: redactAnalyticsText(feedback.summary), + feedback_summary: summary, feedback_type: feedback.feedback_type, feedback_sentiment: feedback.sentiment, - feedback_product_area: feedback.product_area - ? redactAnalyticsText(feedback.product_area) - : undefined, - feedback_destination: configRegistry - ? "config_registry_quality" - : botDetection - ? "config_registry_prioritization" - : undefined, + feedback_task_outcome: taskOutcome, + feedback_affected_tool: feedback.affected_tool, + feedback_dedupe_key: dedupeKey, + feedback_privacy_redacted: privacyRedacted, + feedback_product_area: productArea, + feedback_destination: destination, feedback_bot_detection_registrable_domain: botDetection?.registrable_domain, feedback_bot_detection_observed_outcome: botDetection?.observed_outcome, - feedback_bot_detection_suspected_vendor: botDetection?.suspected_vendor - ? redactAnalyticsText(botDetection.suspected_vendor) - : undefined, + feedback_bot_detection_suspected_vendor: suspectedVendor, feedback_bot_detection_challenge_type: botDetection?.challenge_type, feedback_bot_detection_stealth: botDetection?.stealth, feedback_bot_detection_proxy_type: botDetection?.proxy_type, - feedback_bot_detection_region: botDetection?.region - ? redactAnalyticsText(botDetection.region) - : undefined, - feedback_bot_detection_browser_version: botDetection?.browser_version - ? redactAnalyticsText(botDetection.browser_version) - : undefined, - feedback_bot_detection_browser_image_version: - botDetection?.browser_image_version - ? redactAnalyticsText(botDetection.browser_image_version) - : undefined, + feedback_bot_detection_region: region, + feedback_bot_detection_browser_version: browserVersion, + feedback_bot_detection_browser_image_version: browserImageVersion, feedback_bot_detection_reproducibility: botDetection?.reproducibility, - feedback_bot_detection_browser_session_id: botDetection?.browser_session_id - ? redactAnalyticsText(botDetection.browser_session_id) - : undefined, + feedback_bot_detection_browser_session_id: browserSessionId, feedback_config_registry_request_method: configRegistry?.request_method, - feedback_config_registry_analysis_id: configRegistry?.analysis_id - ? redactAnalyticsText(configRegistry.analysis_id) - : undefined, + feedback_config_registry_analysis_id: analysisId, feedback_config_registry_recommendation_match_scope: configRegistry?.recommendation_match_scope, feedback_config_registry_recommendation_verification: @@ -646,19 +729,11 @@ export function captureMcpFeedback( : undefined, feedback_category: feedback.category, feedback_task_completed: feedback.task_completed, - feedback_tools_used: feedback.tools_used?.map(redactAnalyticsText), - feedback_friction_points: feedback.friction_points - ? redactAnalyticsText(feedback.friction_points) - : undefined, - feedback_suggested_improvement: feedback.suggested_improvement - ? redactAnalyticsText(feedback.suggested_improvement) - : undefined, - feedback_user_request: feedback.user_request - ? redactAnalyticsText(feedback.user_request) - : undefined, - feedback_details: feedback.details - ? redactAnalyticsText(feedback.details) - : undefined, + feedback_tools_used: toolsUsed, + feedback_friction_points: frictionPoints, + feedback_suggested_improvement: suggestedImprovement, + feedback_user_request: userRequest, + feedback_details: details, }); } @@ -671,15 +746,17 @@ export function instrumentMcpAnalytics( client: PostHog | null = posthog, ) { if (!client) { + registerMissingCapabilityTool(server); registerFeedbackTool(server); return; } const analytics = instrument(server, client, { - // Records a `$mcp_missing_capability` event, carrying the reported gap as $mcp_intent, - // when an agent calls the tool registered by registerMissingCapabilityTool. - reportMissing: true, - missingCapabilityToolName: MISSING_CAPABILITY_TOOL_NAME, + // The first-class get_more_tools handler validates and captures structured demand itself. + // Point the SDK's name-based interception at an unadvertised name so calls to the real + // tool reach its registered schema and callback even while reportMissing is disabled. + reportMissing: false, + missingCapabilityToolName: "__posthog_missing_capability_disabled", // Adds a required `context` argument to every advertised tool, which the agent fills // with why it is making the call. Recorded as $mcp_intent. The description replaces // the SDK default: it repeats per tool in every tools/list response, so it stays @@ -723,7 +800,9 @@ export function instrumentMcpAnalytics( beforeSend: sanitizeMcpAnalyticsEvent, }); - registerMissingCapabilityTool(server); + registerMissingCapabilityTool(server, (report, extra) => + captureMissingCapabilityReport(report, extra, analytics), + ); registerFeedbackTool(server, (feedback, extra) => captureMcpFeedback(feedback, extra, analytics), ); diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 85cb3ca..03de603 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { registerMcpCapabilities } from "@/lib/mcp/register"; +import { KERNEL_MCP_TOOL_NAMES } from "@/lib/mcp/tool-names"; const NON_AUTH_TOOLSETS = [ "profiles", @@ -52,6 +53,20 @@ function captureRegistration(mcpApps: boolean, vaults = false) { return { legacyTools, appTools, resources, schemas }; } +describe("MCP tool ownership", () => { + test("recognizes every registered KERNEL tool", () => { + const registration = captureRegistration(true, true); + const knownTools = new Set(KERNEL_MCP_TOOL_NAMES); + + for (const toolName of [ + ...registration.legacyTools, + ...registration.appTools, + ]) { + expect(knownTools.has(toolName)).toBe(true); + } + }); +}); + describe("MCP Apps additive registration", () => { test("keeps managed auth unchanged and only adds the App tools for capable clients", () => { const previous = process.env.KERNEL_MCP_DISABLED_TOOLSETS; diff --git a/src/lib/mcp/tool-names.ts b/src/lib/mcp/tool-names.ts new file mode 100644 index 0000000..dd2abeb --- /dev/null +++ b/src/lib/mcp/tool-names.ts @@ -0,0 +1,46 @@ +export const KERNEL_MCP_TOOL_NAMES = [ + "begin_auth_login", + "browser_curl", + "computer_action", + "exec_command", + "execute_playwright_code", + "get_connection_context", + "get_more_tools", + "manage_api_keys", + "manage_apps", + "manage_auth_connections", + "manage_browser_pools", + "manage_browsers", + "manage_credential_providers", + "manage_credentials", + "manage_extensions", + "manage_profiles", + "manage_projects", + "manage_proxies", + "manage_replays", + "manage_vault_cards", + "manage_vault_items", + "manage_vault_wallets", + "manage_vaults", + "open_auth_login", + "search_docs", + "submit_feedback", + "webmcp", +] as const; + +export type KernelMcpToolName = (typeof KERNEL_MCP_TOOL_NAMES)[number]; + +const kernelMcpToolNameSet: ReadonlySet = new Set( + KERNEL_MCP_TOOL_NAMES, +); + +export function normalizeKernelMcpToolName( + value: string, +): KernelMcpToolName | undefined { + let candidate = value.trim().toLowerCase().replace(/-/g, "_"); + if (candidate.includes("__")) candidate = candidate.split("__").at(-1) ?? ""; + candidate = candidate.replace(/^(?:mcp_)?kernel_/, ""); + return kernelMcpToolNameSet.has(candidate) + ? (candidate as KernelMcpToolName) + : undefined; +} diff --git a/src/lib/mcp/tools/feedback.test.ts b/src/lib/mcp/tools/feedback.test.ts index 06d1125..ed6a0ac 100644 --- a/src/lib/mcp/tools/feedback.test.ts +++ b/src/lib/mcp/tools/feedback.test.ts @@ -64,6 +64,7 @@ describe("submit_feedback", () => { friction_points: "- The timeout response did not suggest a retry.", suggested_improvement: "Include retry timing in browser creation timeout responses.", + task_outcome: "completed", task_completed: true, tools_used: ["manage_browsers"], }, @@ -73,6 +74,114 @@ describe("submit_feedback", () => { } }); + test("normalizes KERNEL tool ownership and rejects feedback for other servers", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + const accepted = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting that a KERNEL browser control returned no output while the user remained blocked.", + summary: "Playwright execution returned no output", + feedback_type: "mcp", + sentiment: "negative", + task_outcome: "blocked", + affected_tool: "kernel__execute_playwright_code", + category: "tool_output_format", + }, + }); + expect(accepted.isError).not.toBe(true); + expect(captured[0]?.affected_tool).toBe("execute_playwright_code"); + + const external = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting a schema problem in a tool owned by another MCP server instead of KERNEL.", + summary: "Another server exposed an incomplete schema", + feedback_type: "mcp", + sentiment: "negative", + task_outcome: "blocked", + affected_tool: "mcp_driftwood_install_task", + category: "tool_input_schema", + }, + }); + expect(external.isError).toBe(true); + expect(captured).toHaveLength(1); + + const missingCapability = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting an absent KERNEL capability through the wrong feedback channel instead of get_more_tools.", + summary: "A required browser operation is unavailable", + feedback_type: "mcp", + sentiment: "negative", + task_outcome: "blocked", + affected_tool: "manage_browsers", + category: "missing_tool", + }, + }); + expect(missingCapability.isError).not.toBe(true); + expect(captured).toHaveLength(2); + } finally { + await close(); + } + }); + + test("defaults legacy task outcomes and rejects conflicting fields", async () => { + const captured: KernelFeedback[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerFeedbackTool(server, (feedback) => { + captured.push(feedback); + }), + {}, + ); + + try { + const missing = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting product feedback without enough information to determine the task impact.", + summary: "Browser startup guidance was unclear", + feedback_type: "product", + sentiment: "mixed", + product_area: "browsers", + }, + }); + expect(missing.isError).not.toBe(true); + expect(captured[0]?.task_outcome).toBe("unknown"); + + const conflicting = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting product feedback with contradictory completion signals that cannot be prioritized reliably.", + summary: "Browser startup guidance was unclear", + feedback_type: "product", + sentiment: "mixed", + product_area: "browsers", + task_outcome: "blocked", + task_completed: true, + }, + }); + expect(conflicting.isError).toBe(true); + expect(captured).toHaveLength(1); + } finally { + await close(); + } + }); + test("records structured bot-detection outcomes for config registry prioritization", async () => { const captured: KernelFeedback[] = []; const { client, close } = await connectTestMcp( @@ -131,6 +240,7 @@ describe("submit_feedback", () => { summary: "Stealth sessions were consistently blocked", feedback_type: "bot_detection", sentiment: "negative", + task_outcome: "blocked", task_completed: false, tools_used: ["manage_browsers", "execute_playwright_code"], bot_detection: { @@ -219,6 +329,7 @@ describe("submit_feedback", () => { summary: "The recommended configuration remained blocked", feedback_type: "config_registry", sentiment: "negative", + task_outcome: "blocked", task_completed: false, bot_detection: { registrable_domain: "example.com", @@ -418,6 +529,9 @@ describe("submit_feedback", () => { summary: "The MCP response was easy to use", feedback_type: "mcp", sentiment: "positive", + task_outcome: "completed", + affected_tool: "kernel__execute_playwright_code", + category: "tool_output_format", }, }); @@ -447,6 +561,8 @@ describe("submit_feedback", () => { summary: "Browser feedback could not be delivered", feedback_type: "product", sentiment: "negative", + task_outcome: "blocked", + product_area: "browsers", }, }); diff --git a/src/lib/mcp/tools/feedback.ts b/src/lib/mcp/tools/feedback.ts index 21516c7..43e2aae 100644 --- a/src/lib/mcp/tools/feedback.ts +++ b/src/lib/mcp/tools/feedback.ts @@ -3,9 +3,37 @@ import { parse as parseDomain } from "tldts"; import { z } from "zod"; import { MCP_INTENT_ARGUMENT_DESCRIPTION } from "@/lib/mcp/analytics-context"; import { errorResponse, jsonResponse } from "@/lib/mcp/responses"; +import { + normalizeKernelMcpToolName, + type KernelMcpToolName, +} from "@/lib/mcp/tool-names"; export const KERNEL_FEEDBACK_TOOL_NAME = "submit_feedback"; +const taskOutcomeSchema = z.enum([ + "completed", + "completed_with_workaround", + "partially_completed", + "blocked", + "not_applicable", + "unknown", +]); + +const affectedToolSchema = z + .string() + .trim() + .min(1) + .max(100) + .transform((value, context): KernelMcpToolName => { + const toolName = normalizeKernelMcpToolName(value); + if (toolName) return toolName; + context.addIssue({ + code: "custom", + message: "must name a tool provided by the KERNEL MCP server", + }); + return z.NEVER; + }); + const configRegistryAppliedBrowserSchema = z.object({ stealth: z.boolean().describe("the applied browser stealth setting."), headless: z.boolean().describe("the applied browser headless setting."), @@ -218,7 +246,17 @@ const feedbackFields = { sentiment: z .enum(["positive", "neutral", "negative", "mixed"]) .describe( - 'the overall tone. use "negative" for something broken or blocking, "mixed" for mostly fine with a concrete problem, "neutral" for a suggestion or feature request with no strong sentiment, and "positive" for praise or something that worked well. all sentiments are welcome.', + 'the overall tone. use "negative" for something broken or blocking, "mixed" for mostly fine with a concrete problem, "neutral" for a suggestion or feature request with no strong sentiment, and "positive" for praise or something that worked well. all sentiments are welcome, but task_outcome—not sentiment—describes impact.', + ), + task_outcome: taskOutcomeSchema + .optional() + .describe( + 'the outcome of the user\'s task: "completed", "completed_with_workaround", "partially_completed", "blocked", "not_applicable" for feedback not tied to a task, or "unknown" only for legacy submissions without an outcome. preferred over task_completed.', + ), + affected_tool: affectedToolSchema + .optional() + .describe( + 'the single KERNEL MCP tool this report is primarily about. required for `feedback_type: "mcp"`. use the canonical tool name without a client namespace; namespaced forms are normalized when recognized. feedback about tools from another MCP server or the client itself belongs with that owner.', ), product_area: z .string() @@ -227,7 +265,7 @@ const feedbackFields = { .max(100) .optional() .describe( - 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). most useful for product feedback; use `feedback_type: "bot_detection"` instead of putting bot detection here, and for mcp feedback put the tool name in `details` or `friction_points`.', + 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). required for product feedback. use `feedback_type: "bot_detection"` instead of putting bot detection here, and use affected_tool for mcp feedback.', ), bot_detection: botDetectionReportSchema .optional() @@ -253,13 +291,13 @@ const feedbackFields = { ]) .optional() .describe( - 'for mcp feedback (`feedback_type: "mcp"`) only: the single category that best describes the dominant theme. use "missing_tool" when a capability is absent, "tool_description" when tool documentation is unclear, "tool_input_schema" when arguments are confusing, "tool_output_format" when a response is hard to consume, "instructions_clarity" when mcp instructions are unclear, "tool_correctness" when a tool returns wrong data, "error_message" when an error is unhelpful, and "performance" when latency is the issue. omit for product, docs, or other feedback.', + 'for mcp feedback (`feedback_type: "mcp"`) only: the single category that best describes the dominant theme. `missing_tool` remains accepted for compatibility but is routed outside MCP quality; use `get_more_tools` for new capability requests. use "tool_description" when tool documentation is unclear, "tool_input_schema" when arguments are confusing, "tool_output_format" when a response is hard to consume, "instructions_clarity" when mcp instructions are unclear, "tool_correctness" when a tool returns wrong data, "error_message" when an error is unhelpful, and "performance" when latency is the issue. omit for product, docs, or other feedback.', ), task_completed: z .boolean() .optional() .describe( - "whether the user's task was completed. be honest: `false` is useful signal. required for bot-detection and config-registry feedback, and also useful for mcp feedback.", + "legacy task completion signal retained for compatibility. prefer task_outcome, which distinguishes workarounds, partial completion, blockers, and feedback not tied to a task. task_completed remains required for bot-detection and config-registry feedback.", ), tools_used: z .array(z.string().trim().min(1).max(100)) @@ -316,7 +354,7 @@ export type KernelFeedbackCapture = ( ) => void | Promise; const TOOL_DESCRIPTION = - "send feedback about anything KERNEL to the KERNEL team. set `feedback_type` to route it: `product` for any KERNEL product or feature, `bot_detection` for a site-specific pass, challenge, block, or degraded result not tied to an unchanged registry recommendation, `config_registry` for the result after requesting and applying a config registry recommendation unchanged, `mcp` for this mcp server, `docs` for KERNEL documentation, or `other`. for bot detection, fill `bot_detection` with the public registrable domain, outcome, and reproducibility. when a config registry recommendation was requested and applied unchanged, choose `config_registry`; include the request metadata, recommendation evidence, exact browser and proxy settings used, and `bot_detection.browser_session_id`. report both passes and failures so recommendation quality can be measured. if any recommended setting was changed before testing, use `bot_detection` instead so the result is not attributed to the original recommendation. all sentiments are welcome through `sentiment`: praise and feature requests are useful, not just problems. use this for confusing or broken experiences, papercuts, missing capabilities, unhelpful errors, feature requests, and things that worked especially well. keep `summary` to one sentence and make the detail fields concise and actionable, quoting the product surface, tool name, parameter, or error text when possible. include a concrete `suggested_improvement` when one is clear. never include credentials, tokens, api keys, urls, paths, browser or page content, customer or account names, private hosts, IP addresses, or personal data. a public registrable domain is allowed only in `bot_detection.registrable_domain`; never include a subdomain or account-specific host. the user can also ask to send feedback directly. submitting feedback is a side report to KERNEL, not a reason to stop: continue and finish the user's task with the other available tools."; + "send feedback about a KERNEL product, this KERNEL MCP server, or KERNEL documentation. use get_more_tools—not this tool—for a genuinely absent capability. for mcp feedback, identify the single affected KERNEL tool and its category; do not report client behavior or tools owned by another server. describe task impact with task_outcome, while sentiment remains useful for tone and praise. set feedback_type to product, bot_detection, config_registry, mcp, docs, or other. for bot detection, fill bot_detection with the public registrable domain, outcome, and reproducibility. after applying a config registry recommendation unchanged, submit exactly one config_registry report for the tested recommendation, whether it passed or failed; include the recommendation metadata, evidence, exact browser and proxy settings, and bot_detection.browser_session_id. if any setting changed before testing, use bot_detection instead. keep summary to one sentence, make detail fields concise and actionable, and include a concrete suggested_improvement when one is clear. never include credentials, tokens, api keys, urls, paths, browser or page content, customer or account names, private hosts, IP addresses, or personal data. a public registrable domain is allowed only in bot_detection.registrable_domain. submitting feedback is a side report, not a reason to stop; continue the user's task with the other available tools."; const RESPONSE_MESSAGES = { recorded: @@ -330,6 +368,35 @@ const RESPONSE_MESSAGES = { type FeedbackCaptureStatus = keyof typeof RESPONSE_MESSAGES; +function taskCompletedForOutcome( + outcome: z.infer, +): boolean | undefined { + switch (outcome) { + case "completed": + case "completed_with_workaround": + return true; + case "partially_completed": + case "blocked": + return false; + case "not_applicable": + case "unknown": + return undefined; + } +} + +function kernelToolsUsed(toolsUsed: string[] | undefined) { + return new Set( + toolsUsed + ?.map(normalizeKernelMcpToolName) + .filter( + (toolName): toolName is KernelMcpToolName => + toolName !== undefined && + toolName !== KERNEL_FEEDBACK_TOOL_NAME && + toolName !== "get_more_tools", + ), + ); +} + export function registerFeedbackTool( server: McpServer, capture?: KernelFeedbackCapture, @@ -348,6 +415,54 @@ export function registerFeedbackTool( }, }, async ({ context: _context, ...feedback }, extra) => { + if (feedback.task_outcome === undefined) { + feedback.task_outcome = + feedback.task_completed === undefined + ? "unknown" + : feedback.task_completed + ? "completed" + : "blocked"; + } else { + const expectedTaskCompleted = taskCompletedForOutcome( + feedback.task_outcome, + ); + if ( + feedback.task_completed !== undefined && + feedback.task_completed !== expectedTaskCompleted + ) { + return errorResponse( + "task_outcome and task_completed describe conflicting outcomes.", + ); + } + feedback.task_completed ??= expectedTaskCompleted; + } + + if (feedback.feedback_type === "mcp") { + const candidates = kernelToolsUsed(feedback.tools_used); + if (!feedback.affected_tool && candidates.size === 1) { + feedback.affected_tool = [...candidates][0]; + } + if ( + !feedback.affected_tool && + feedback.tools_used && + feedback.tools_used.length > 0 && + candidates.size === 0 + ) { + return errorResponse( + "this feedback names no KERNEL MCP tool; report client or external-server feedback to its owner.", + ); + } + } else { + if (feedback.affected_tool) { + return errorResponse( + "affected_tool is only accepted for mcp feedback.", + ); + } + if (feedback.category) { + return errorResponse("category is only accepted for mcp feedback."); + } + } + const hasSiteOutcome = feedback.feedback_type === "bot_detection" || feedback.feedback_type === "config_registry"; diff --git a/src/lib/mcp/tools/missing-capability.ts b/src/lib/mcp/tools/missing-capability.ts new file mode 100644 index 0000000..74d6cbd --- /dev/null +++ b/src/lib/mcp/tools/missing-capability.ts @@ -0,0 +1,191 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { jsonResponse } from "@/lib/mcp/responses"; +import { + normalizeKernelMcpToolName, + type KernelMcpToolName, +} from "@/lib/mcp/tool-names"; + +export const KERNEL_MISSING_CAPABILITY_TOOL_NAME = "get_more_tools"; + +const gapReasonSchema = z.enum([ + "kernel_capability_missing", + "existing_tool_failed", + "transient_or_capacity_failure", + "client_permission_restriction", + "external_integration_unavailable", + "unknown", +]); + +const capabilityAreaSchema = z.enum([ + "browsers", + "browser_files", + "profiles", + "projects", + "apps", + "browser_pools", + "proxies", + "extensions", + "managed_auth", + "credentials", + "api_keys", + "replays", + "vaults", + "docs", + "mcp", + "external_integration", + "client_environment", + "other", +]); + +const requestedActionSchema = z.enum([ + "create", + "read", + "update", + "delete", + "execute", + "search", + "transfer", + "authenticate", + "inspect", + "other", +]); + +const taskOutcomeSchema = z.enum([ + "completed", + "completed_with_workaround", + "partially_completed", + "blocked", +]); + +const checkedKernelToolSchema = z + .string() + .trim() + .min(1) + .max(100) + .transform((value, context): KernelMcpToolName => { + const toolName = normalizeKernelMcpToolName(value); + if (toolName) return toolName; + context.addIssue({ + code: "custom", + message: "must name a tool provided by the KERNEL MCP server", + }); + return z.NEVER; + }); + +const missingCapabilityFields = { + context: z + .string() + .describe( + "The missing capability and the user's goal, in 15-25 words and third person. Never include credentials, URLs, domains, account names, file contents, paths, or personal data.", + ), + gap_reason: gapReasonSchema.describe( + "Why the task could not proceed. Only kernel_capability_missing and external_integration_unavailable are recorded as demand. For an existing tool failure, use submit_feedback instead; transient failures and client restrictions are not capability gaps.", + ), + capability_area: capabilityAreaSchema.describe( + "The single KERNEL product area that would own the capability, or external_integration/client_environment when Kernel does not own it.", + ), + capability: z + .string() + .trim() + .min(1) + .max(100) + .describe( + 'A short generic capability name, such as "browser filesystem upload". Do not include a site, customer, account, domain, path, or payload.', + ), + requested_action: requestedActionSchema.describe( + "The primary operation the missing capability needed to perform.", + ), + task_outcome: taskOutcomeSchema.describe( + "Whether the task was completed, completed through a workaround, partially completed, or blocked.", + ), + tools_checked: z + .array(checkedKernelToolSchema) + .max(10) + .optional() + .describe( + "The closest KERNEL MCP tools checked before confirming the gap. Omit when no existing tool is relevant.", + ), +}; + +export type MissingCapabilityReport = z.infer< + z.ZodObject +>; +export type MissingCapabilityCapture = ( + report: MissingCapabilityReport, + extra: unknown, +) => void | Promise; + +export function registerMissingCapabilityTool( + server: McpServer, + capture?: MissingCapabilityCapture, +) { + server.tool( + KERNEL_MISSING_CAPABILITY_TOOL_NAME, + "Report a capability that no available KERNEL tool can provide after checking the tool list. Classify disconnected third-party services as external integrations. Do not use this for an existing tool that failed, a transient or capacity failure, or a client-side permission restriction; use submit_feedback for an existing KERNEL tool failure. Reports never replace the original task, so continue with any available workaround.", + missingCapabilityFields, + { + title: "Get more tools", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + async (report, extra) => { + const externalIntegration = + report.gap_reason === "external_integration_unavailable"; + if ( + (externalIntegration && + report.capability_area !== "external_integration") || + (report.gap_reason === "kernel_capability_missing" && + (report.capability_area === "external_integration" || + report.capability_area === "client_environment")) + ) { + return jsonResponse({ + recorded: false, + status: "invalid_capability_owner", + message: + "gap_reason and capability_area identify different owners. Correct the classification, then continue the original task.", + }); + } + + const recordable = + report.gap_reason === "kernel_capability_missing" || + externalIntegration; + if (!recordable) { + return jsonResponse({ + recorded: false, + status: "not_a_capability_gap", + message: + report.gap_reason === "existing_tool_failed" + ? "Use submit_feedback for the existing KERNEL tool, then continue the original task." + : "This is not a missing capability request. Continue the original task using its normal recovery or client-permission path.", + }); + } + + let status: "recorded" | "unavailable" | "failed" = "unavailable"; + if (capture) { + try { + await capture(report, extra); + status = "recorded"; + } catch (error) { + status = "failed"; + console.error("Failed to capture MCP capability request", error); + } + } + return jsonResponse({ + recorded: status === "recorded", + status, + capability: report.capability, + destination: + report.gap_reason === "kernel_capability_missing" + ? "kernel_product_demand" + : "external_integration_demand", + message: + status === "recorded" + ? "The capability request was recorded. No additional KERNEL tools are available; continue the original task with any available workaround." + : "The capability request was not recorded. No additional KERNEL tools are available; continue the original task with any available workaround.", + }); + }, + ); +} From fb598f453f2003425811fccab771e53aaaa41c3c Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:12:01 +0000 Subject: [PATCH 2/3] Mark capability reports as writes --- src/lib/mcp/analytics.test.ts | 1 + src/lib/mcp/tools/missing-capability.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index 3ad2e84..0aede87 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -881,6 +881,7 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { const missingCapabilityTool = ( await enabled.client.listTools() ).tools.find(({ name }) => name === "get_more_tools"); + expect(missingCapabilityTool?.annotations?.readOnlyHint).toBe(false); expect(missingCapabilityTool?.description).toContain( "after checking the tool list", ); diff --git a/src/lib/mcp/tools/missing-capability.ts b/src/lib/mcp/tools/missing-capability.ts index 74d6cbd..40b275c 100644 --- a/src/lib/mcp/tools/missing-capability.ts +++ b/src/lib/mcp/tools/missing-capability.ts @@ -126,7 +126,7 @@ export function registerMissingCapabilityTool( missingCapabilityFields, { title: "Get more tools", - readOnlyHint: true, + readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: true, From e8e7135cc2473c690f8a767578a06a48fe317610 Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:29:44 +0000 Subject: [PATCH 3/3] Address feedback routing review --- src/lib/mcp/analytics.test.ts | 122 +++++++++++++++++++ src/lib/mcp/analytics.ts | 32 +++-- src/lib/mcp/register.test.ts | 28 +++-- src/lib/mcp/tool-names.ts | 14 ++- src/lib/mcp/tools/feedback.test.ts | 20 +++ src/lib/mcp/tools/feedback.ts | 4 +- src/lib/mcp/tools/missing-capability.test.ts | 111 +++++++++++++++++ src/lib/mcp/tools/missing-capability.ts | 2 +- 8 files changed, 309 insertions(+), 24 deletions(-) create mode 100644 src/lib/mcp/tools/missing-capability.test.ts diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index 0aede87..da77156 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -396,6 +396,24 @@ describe("sanitizeMcpAnalyticsEvent", () => { expect(intent).not.toContain("/tmp/private/cart.json"); }); + test("redacts compressed and digit-only IPv6 addresses", async () => { + const event = toolCallEvent({ + [PostHogMCPAnalyticsProperty.Intent]: + "Checking 2001:4860:4860::8888, fe80::1, and ::1 at 12:30:45.", + }); + + const result = await sanitizeMcpAnalyticsEvent(event); + const intent = result?.properties[ + PostHogMCPAnalyticsProperty.Intent + ] as string; + + expect(intent.match(/\[ip\]/g)).toHaveLength(3); + expect(intent).toContain("12:30:45"); + expect(intent).not.toContain("2001:4860:4860::8888"); + expect(intent).not.toContain("fe80::1"); + expect(intent).not.toContain("::1"); + }); + test("deletes non-string intents", async () => { const event = toolCallEvent({ [PostHogMCPAnalyticsProperty.Intent]: { goal: "payload" }, @@ -696,6 +714,35 @@ describe("captureMcpFeedback", () => { ]); }); + test("routes product feedback without an area as unclassified before praise", async () => { + const captured: { properties: Record }[] = []; + const analytics = { + capture: async (event: { properties: Record }) => { + captured.push(event); + }, + } as McpAnalytics; + + for (const summary of ["功能无法使用", "設定を保存できない"]) { + await captureMcpFeedback( + { + summary, + feedback_type: "product", + sentiment: "positive", + task_outcome: "unknown", + }, + {}, + analytics, + ); + } + + expect( + captured.map(({ properties }) => properties.feedback_destination), + ).toEqual(["product_unclassified", "product_unclassified"]); + expect(captured[0]?.properties.feedback_dedupe_key).not.toBe( + captured[1]?.properties.feedback_dedupe_key, + ); + }); + test("routes structured bot-detection feedback to config registry prioritization", async () => { const captured: unknown[] = []; const analytics = { @@ -760,6 +807,80 @@ describe("captureMcpFeedback", () => { ]); }); + test("separates lookup feedback by outcome and applied configuration", async () => { + const captured: { properties: Record }[] = []; + const analytics = { + capture: async (event: { properties: Record }) => { + captured.push(event); + }, + } as McpAnalytics; + const base = { + summary: "The lookup recommendation was tested", + feedback_type: "config_registry" as const, + sentiment: "mixed" as const, + task_completed: false, + bot_detection: { + registrable_domain: "example.com", + observed_outcome: "blocked" as const, + reproducibility: "consistent" as const, + browser_session_id: "session_lookup", + }, + config_registry: { + request_method: "lookup" as const, + recommendation_match_scope: "exact" as const, + recommendation_verification: "verified" as const, + recommendation_evidence: { + sample_size: 5, + success_rate: 1, + last_verified_at: "2026-09-13T12:00:00Z", + }, + applied_browser: { + stealth: true, + headless: false, + gpu: false, + viewport: { width: 1920, height: 1080 }, + }, + applied_proxy: { mode: "direct" as const }, + }, + }; + + await captureMcpFeedback(base, {}, analytics); + await captureMcpFeedback( + { + ...base, + bot_detection: { + ...base.bot_detection, + observed_outcome: "passed", + }, + }, + {}, + analytics, + ); + await captureMcpFeedback( + { + ...base, + bot_detection: { + ...base.bot_detection, + observed_outcome: "passed", + }, + config_registry: { + ...base.config_registry, + applied_browser: { + ...base.config_registry.applied_browser, + headless: true, + }, + }, + }, + {}, + analytics, + ); + + const dedupeKeys = captured.map( + ({ properties }) => properties.feedback_dedupe_key, + ); + expect(new Set(dedupeKeys).size).toBe(3); + }); + test("attributes config-registry feedback to the applied configuration", async () => { const captured: unknown[] = []; const analytics = { @@ -882,6 +1003,7 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { await enabled.client.listTools() ).tools.find(({ name }) => name === "get_more_tools"); expect(missingCapabilityTool?.annotations?.readOnlyHint).toBe(false); + expect(missingCapabilityTool?.annotations?.idempotentHint).toBe(false); expect(missingCapabilityTool?.description).toContain( "after checking the tool list", ); diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index 7a277dd..2859acf 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { isIP } from "node:net"; import { instrument, PostHogMCPAnalyticsEvent, @@ -232,7 +233,6 @@ const INTENT_REDACTIONS: readonly [RegExp, string][] = [ [/[^\s@]+@[^\s@]+\.[^\s@]+/g, "[email]"], [/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "[url]"], [/\b(?:\d{1,3}\.){3}\d{1,3}\b/g, "[ip]"], - [/\b(?=[A-F0-9:]*[A-F])(?:[A-F0-9]{1,4}:){2,}[A-F0-9:]{1,}\b/gi, "[ip]"], [/(?) { properties[MCP_USED_PROJECT_PROPERTY] = hasNonEmptyParam(args, "project"); } +const IPV6_CANDIDATE_PATTERN = + /(? + isIP(candidate) === 6 ? "[ip]" : candidate, + ); + redacted ||= withoutIpv6 !== value; + value = withoutIpv6; for (const [pattern, replacement] of INTENT_REDACTIONS) { const next = value.replace(pattern, replacement); redacted ||= next !== value; @@ -644,17 +652,23 @@ export function captureMcpFeedback( : feedback.affected_tool ? "mcp_quality" : "mcp_unclassified" - : feedback.feedback_type === "product" && - feedback.sentiment === "positive" - ? "product_praise" - : feedback.feedback_type === "product" && !productArea + : feedback.feedback_type === "product" + ? !productArea ? "product_unclassified" - : `${feedback.feedback_type}_feedback`; + : feedback.sentiment === "positive" + ? "product_praise" + : "product_feedback" + : `${feedback.feedback_type}_feedback`; + const appliedConfigKey = configRegistry + ? configRegistryAppliedConfigKey(configRegistry) + : undefined; const dedupeKey = configRegistry ? analyticsDedupeKey([ "config_registry", analysisId ?? configRegistry.request_method, botDetection?.registrable_domain, + appliedConfigKey, + botDetection?.observed_outcome, ]) : botDetection ? analyticsDedupeKey([ @@ -668,7 +682,7 @@ export function captureMcpFeedback( feedback.affected_tool, productArea, feedback.category, - summary.toLowerCase().replace(/[^a-z0-9]+/g, " "), + summary.normalize("NFKC").toLowerCase().replace(/\s+/gu, " "), ]); return captureMcpCustomEvent(analytics, extra, MCP_FEEDBACK_SUBMITTED_EVENT, { @@ -704,9 +718,7 @@ export function captureMcpFeedback( configRegistry?.recommendation_evidence.success_rate, feedback_config_registry_evidence_last_verified_at: configRegistry?.recommendation_evidence.last_verified_at, - feedback_config_registry_applied_config_key: configRegistry - ? configRegistryAppliedConfigKey(configRegistry) - : undefined, + feedback_config_registry_applied_config_key: appliedConfigKey, feedback_config_registry_browser_stealth: configRegistry?.applied_browser.stealth, feedback_config_registry_browser_headless: diff --git a/src/lib/mcp/register.test.ts b/src/lib/mcp/register.test.ts index 03de603..7cd97bc 100644 --- a/src/lib/mcp/register.test.ts +++ b/src/lib/mcp/register.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { instrumentMcpAnalytics } from "@/lib/mcp/analytics"; import { registerMcpCapabilities } from "@/lib/mcp/register"; import { KERNEL_MCP_TOOL_NAMES } from "@/lib/mcp/tool-names"; @@ -24,7 +25,11 @@ const NON_AUTH_TOOLSETS = [ "vaults", ].join(","); -function captureRegistration(mcpApps: boolean, vaults = false) { +function captureRegistration( + mcpApps: boolean, + vaults = false, + analytics = false, +) { const legacyTools: string[] = []; const appTools: string[] = []; const resources: string[] = []; @@ -50,20 +55,25 @@ function captureRegistration(mcpApps: boolean, vaults = false) { }, } as unknown as McpServer; registerMcpCapabilities(server, { mcpApps, vaults }); + if (analytics) instrumentMcpAnalytics(server, null); return { legacyTools, appTools, resources, schemas }; } describe("MCP tool ownership", () => { - test("recognizes every registered KERNEL tool", () => { - const registration = captureRegistration(true, true); - const knownTools = new Set(KERNEL_MCP_TOOL_NAMES); - - for (const toolName of [ + test("matches every registered KERNEL tool in both directions", () => { + const registration = captureRegistration(true, true, true); + const registeredTools = new Set([ ...registration.legacyTools, ...registration.appTools, - ]) { - expect(knownTools.has(toolName)).toBe(true); - } + ]); + const knownTools = new Set(KERNEL_MCP_TOOL_NAMES); + + expect( + [...registeredTools].filter((tool) => !knownTools.has(tool)), + ).toEqual([]); + expect( + [...knownTools].filter((tool) => !registeredTools.has(tool)), + ).toEqual([]); }); }); diff --git a/src/lib/mcp/tool-names.ts b/src/lib/mcp/tool-names.ts index dd2abeb..72bb5c1 100644 --- a/src/lib/mcp/tool-names.ts +++ b/src/lib/mcp/tool-names.ts @@ -20,6 +20,7 @@ export const KERNEL_MCP_TOOL_NAMES = [ "manage_replays", "manage_vault_cards", "manage_vault_items", + "manage_vault_provider_configs", "manage_vault_wallets", "manage_vaults", "open_auth_login", @@ -38,8 +39,17 @@ export function normalizeKernelMcpToolName( value: string, ): KernelMcpToolName | undefined { let candidate = value.trim().toLowerCase().replace(/-/g, "_"); - if (candidate.includes("__")) candidate = candidate.split("__").at(-1) ?? ""; - candidate = candidate.replace(/^(?:mcp_)?kernel_/, ""); + for (const prefix of [ + "mcp__kernel__", + "kernel__", + "mcp_kernel_", + "kernel_", + ]) { + if (candidate.startsWith(prefix)) { + candidate = candidate.slice(prefix.length); + break; + } + } return kernelMcpToolNameSet.has(candidate) ? (candidate as KernelMcpToolName) : undefined; diff --git a/src/lib/mcp/tools/feedback.test.ts b/src/lib/mcp/tools/feedback.test.ts index ed6a0ac..5b3e6d9 100644 --- a/src/lib/mcp/tools/feedback.test.ts +++ b/src/lib/mcp/tools/feedback.test.ts @@ -115,6 +115,26 @@ describe("submit_feedback", () => { }, }); expect(external.isError).toBe(true); + + for (const affectedTool of [ + "mcp__slack__manage_browsers", + "external__manage_apps", + ]) { + const spoofedNamespace = await client.callTool({ + name: KERNEL_FEEDBACK_TOOL_NAME, + arguments: { + context: + "Reporting feedback for an external namespaced tool that resembles a KERNEL tool name.", + summary: "An external tool used a KERNEL-like name", + feedback_type: "mcp", + sentiment: "negative", + task_outcome: "blocked", + affected_tool: affectedTool, + category: "tool_correctness", + }, + }); + expect(spoofedNamespace.isError).toBe(true); + } expect(captured).toHaveLength(1); const missingCapability = await client.callTool({ diff --git a/src/lib/mcp/tools/feedback.ts b/src/lib/mcp/tools/feedback.ts index 43e2aae..1b4b800 100644 --- a/src/lib/mcp/tools/feedback.ts +++ b/src/lib/mcp/tools/feedback.ts @@ -256,7 +256,7 @@ const feedbackFields = { affected_tool: affectedToolSchema .optional() .describe( - 'the single KERNEL MCP tool this report is primarily about. required for `feedback_type: "mcp"`. use the canonical tool name without a client namespace; namespaced forms are normalized when recognized. feedback about tools from another MCP server or the client itself belongs with that owner.', + 'the single KERNEL MCP tool this report is primarily about. preferred for new `feedback_type: "mcp"` submissions; omission remains accepted for legacy clients and routes to unclassified feedback. use the canonical tool name without a client namespace; recognized KERNEL namespace forms are normalized. feedback about tools from another MCP server or the client itself belongs with that owner.', ), product_area: z .string() @@ -265,7 +265,7 @@ const feedbackFields = { .max(100) .optional() .describe( - 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). required for product feedback. use `feedback_type: "bot_detection"` instead of putting bot detection here, and use affected_tool for mcp feedback.', + 'the KERNEL product or area this is about, in free text (e.g. "browsers", "apps", "managed auth", "browser pools", "proxies", or "telemetry"). preferred for new product feedback; omission remains accepted for legacy clients and routes to unclassified feedback. use `feedback_type: "bot_detection"` instead of putting bot detection here, and use affected_tool for mcp feedback.', ), bot_detection: botDetectionReportSchema .optional() diff --git a/src/lib/mcp/tools/missing-capability.test.ts b/src/lib/mcp/tools/missing-capability.test.ts new file mode 100644 index 0000000..c162f84 --- /dev/null +++ b/src/lib/mcp/tools/missing-capability.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test"; +import { connectTestMcp, toolResultJSON } from "@/lib/mcp/mcp-test-fixtures"; +import { + KERNEL_MISSING_CAPABILITY_TOOL_NAME, + type MissingCapabilityReport, + registerMissingCapabilityTool, +} from "@/lib/mcp/tools/missing-capability"; + +describe("get_more_tools", () => { + test("routes external demand and rejects owner mismatches and existing-tool failures", async () => { + const captured: MissingCapabilityReport[] = []; + const { client, close } = await connectTestMcp( + (server) => + registerMissingCapabilityTool(server, (report) => { + captured.push(report); + }), + {}, + ); + + try { + const external = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Sending a notification requires an external integration that no available KERNEL tool currently provides.", + gap_reason: "external_integration_unavailable", + capability_area: "external_integration", + capability: "external notification delivery", + requested_action: "execute", + task_outcome: "blocked", + }, + }); + expect(toolResultJSON(external)).toMatchObject({ + recorded: true, + status: "recorded", + destination: "external_integration_demand", + }); + + const ownerMismatch = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Sending a notification was incorrectly classified as KERNEL product demand instead of an external integration.", + gap_reason: "external_integration_unavailable", + capability_area: "browsers", + capability: "external notification delivery", + requested_action: "execute", + task_outcome: "blocked", + }, + }); + expect(toolResultJSON(ownerMismatch)).toMatchObject({ + recorded: false, + status: "invalid_capability_owner", + }); + + const existingToolFailure = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Creating a browser failed through an existing KERNEL tool and should be routed to feedback instead.", + gap_reason: "existing_tool_failed", + capability_area: "browsers", + capability: "browser creation", + requested_action: "create", + task_outcome: "blocked", + tools_checked: ["manage_browsers"], + }, + }); + expect(toolResultJSON(existingToolFailure)).toMatchObject({ + recorded: false, + status: "not_a_capability_gap", + }); + expect(captured).toHaveLength(1); + } finally { + await close(); + } + }); + + test("reports capture failures without interrupting the task", async () => { + const { client, close } = await connectTestMcp( + (server) => + registerMissingCapabilityTool(server, () => { + throw new Error("capture unavailable"); + }), + {}, + ); + + try { + const result = await client.callTool({ + name: KERNEL_MISSING_CAPABILITY_TOOL_NAME, + arguments: { + context: + "Uploading a local file requires a browser transfer capability that no available KERNEL tool provides.", + gap_reason: "kernel_capability_missing", + capability_area: "browser_files", + capability: "browser filesystem upload", + requested_action: "transfer", + task_outcome: "blocked", + tools_checked: ["manage_browsers"], + }, + }); + expect(toolResultJSON(result)).toMatchObject({ + recorded: false, + status: "failed", + }); + expect(result.isError).not.toBe(true); + } finally { + await close(); + } + }); +}); diff --git a/src/lib/mcp/tools/missing-capability.ts b/src/lib/mcp/tools/missing-capability.ts index 40b275c..3082254 100644 --- a/src/lib/mcp/tools/missing-capability.ts +++ b/src/lib/mcp/tools/missing-capability.ts @@ -128,7 +128,7 @@ export function registerMissingCapabilityTool( title: "Get more tools", readOnlyHint: false, destructiveHint: false, - idempotentHint: true, + idempotentHint: false, openWorldHint: true, }, async (report, extra) => {