From ac8705a860fd00aa9f9589cac4c5ee0ba4d1d8d7 Mon Sep 17 00:00:00 2001 From: zhengchuyi Date: Thu, 6 Aug 2026 18:01:14 +0800 Subject: [PATCH 1/5] feat(studio): consolidate deploy and sandbox telemetry --- frontend/src/App.tsx | 9 +++ frontend/src/adk/telemetry.ts | 6 +- frontend/src/adk/telemetryClassifiers.ts | 10 ++++ frontend/src/adk/telemetryEvents.ts | 41 +++++++++++--- .../feishu/FeishuBotIntegration.tsx | 32 +++++++++++ frontend/src/create/CodePackageCreate.tsx | 6 +- frontend/src/create/CustomCreate.tsx | 11 +++- frontend/src/create/IntelligentCreate.tsx | 6 +- frontend/src/ui/ProjectPreview.tsx | 14 +++-- frontend/tests/studioTelemetry.test.mjs | 55 ++++++++++++++----- 10 files changed, 157 insertions(+), 33 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3159a30b..91e54fef 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -219,6 +219,7 @@ async function probeNewChatCapabilities( type CreateMode = QuickCreateKind | "package"; type CreateView = "menu" | CreateMode | null; +type CustomCreateMode = "custom" | "yaml_import"; // Persist the last view so a page refresh restores where the user was. const LS = { app: "veadk.appName", view: "veadk.view", session: "veadk.sessionId" } as const; @@ -1138,6 +1139,8 @@ export default function App() { const [addMenu, setAddMenu] = useState(false); // A draft imported from YAML, used to pre-fill the custom wizard once. const [importedDraft, setImportedDraft] = useState(null); + const [customCreateMode, setCustomCreateMode] = + useState("custom"); const [savedAgentDrafts, setSavedAgentDrafts] = useState([]); const savedAgentDraftsRef = useRef([]); const pendingWorkspaceDraftRef = useRef(null); @@ -4438,6 +4441,7 @@ export default function App() { onEditDraft={(item) => { setMyAgents(false); setImportedDraft(item.draft); + setCustomCreateMode("custom"); setEditingDraftId(item.id); editingDraftBaselineRef.current = item; setRuntimeUpdateTarget(item.deploymentTarget ?? null); @@ -4531,6 +4535,7 @@ export default function App() { }; setManageAgents(false); setImportedDraft(hydratedDraft); + setCustomCreateMode("custom"); const nextDraftId = `runtime-${capability.runtime.runtimeId}`; setEditingDraftId(nextDraftId); editingDraftBaselineRef.current = @@ -4550,6 +4555,7 @@ export default function App() { onEditDraft={(item) => { setManageAgents(false); setImportedDraft(item.draft); + setCustomCreateMode("custom"); setEditingDraftId(item.id); editingDraftBaselineRef.current = item; setRuntimeUpdateTarget(item.deploymentTarget ?? null); @@ -4648,6 +4654,7 @@ export default function App() { setRuntimeUpdateTarget(null); setFocusedDeploymentTaskId(""); setFocusedWorkspaceAgentId(""); + if (k === "custom") setCustomCreateMode("custom"); setEditingDraftId( k === "custom" ? `draft-${Date.now().toString(36)}` : "", ); @@ -4656,6 +4663,7 @@ export default function App() { }} onImport={(d) => { setImportedDraft(d); + setCustomCreateMode("yaml_import"); setRuntimeUpdateTarget(null); setFocusedDeploymentTaskId(""); setFocusedWorkspaceAgentId(""); @@ -4681,6 +4689,7 @@ export default function App() { onAgentAdded={onAgentAdded} features={features} onDeploymentTaskChange={updateDeploymentTask} + createMode={customCreateMode} deploymentTarget={runtimeUpdateTarget ?? undefined} initialDeployRegion={newRuntimeRegion} onDraftChange={(draft, dirty) => { diff --git a/frontend/src/adk/telemetry.ts b/frontend/src/adk/telemetry.ts index 3b08d2e8..30077a89 100644 --- a/frontend/src/adk/telemetry.ts +++ b/frontend/src/adk/telemetry.ts @@ -7,10 +7,8 @@ import type { export type StudioTelemetryEventName = | "studio_instance_loaded" | "studio_user_authenticated" - | "studio_agent_deploy_succeeded" - | "studio_agent_deploy_failed" - | "studio_sandbox_create_succeeded" - | "studio_sandbox_create_failed"; + | "studio_agent_deploy" + | "studio_sandbox_create"; export interface StudioTelemetryEventOptions { dedupeKey?: string; diff --git a/frontend/src/adk/telemetryClassifiers.ts b/frontend/src/adk/telemetryClassifiers.ts index ebb2b7b9..c6dc77a1 100644 --- a/frontend/src/adk/telemetryClassifiers.ts +++ b/frontend/src/adk/telemetryClassifiers.ts @@ -19,3 +19,13 @@ export function agentDeployErrorKind(error: unknown, phase: string): string { } return "unknown"; } + +export function telemetryErrorSummary(error: unknown): string { + const raw = error instanceof Error ? error.message : String(error); + return raw + .replace( + /\b((?:app[_-]?)?secret|token|api[_-]?key|password)\b\s*[:=]\s*["']?[^"',\s}]+/gi, + "$1=", + ) + .slice(0, 300); +} diff --git a/frontend/src/adk/telemetryEvents.ts b/frontend/src/adk/telemetryEvents.ts index 4b0fbe23..8c370e44 100644 --- a/frontend/src/adk/telemetryEvents.ts +++ b/frontend/src/adk/telemetryEvents.ts @@ -1,16 +1,33 @@ import { agentDeployErrorKind, sandboxCreateErrorKind, + telemetryErrorSummary, } from "./telemetryClassifiers"; import { trackStudioEvent } from "./telemetry"; import type { SandboxAgentKind } from "./sandbox"; export type DeploymentTelemetrySource = - | "custom_create" - | "intelligent_create" + | "scratch" | "code_package" + | "feishu_automation" | "unknown"; +export type DeploymentCreateMode = + | "custom" + | "intelligent" + | "template" + | "workflow" + | "yaml_import" + | "code_package" + | "feishu_template" + | "unknown"; + +export interface DeploymentTelemetryOrigin { + source: DeploymentTelemetrySource; + createMode: DeploymentCreateMode; + aiAssisted: boolean; +} + export interface StudioLoadedTelemetry { agentsSource: "local" | "cloud"; } @@ -18,7 +35,7 @@ export interface StudioLoadedTelemetry { export type SandboxTelemetryKind = "codex" | SandboxAgentKind; export interface AgentDeployTelemetryBase { - source: DeploymentTelemetrySource; + telemetry: DeploymentTelemetryOrigin; action: "create" | "update"; region: string; networkType: string; @@ -49,7 +66,9 @@ export interface SandboxCreateFailedTelemetry extends SandboxCreateTelemetryBase function agentDeployCategories(args: AgentDeployTelemetryBase) { return { - deploy_source: args.source, + deploy_source: args.telemetry.source, + create_mode: args.telemetry.createMode, + ai_assisted: args.telemetry.aiAssisted, deploy_action: args.action, deploy_region: args.region, runtime_network_type: args.networkType, @@ -71,24 +90,28 @@ export function trackStudioLoaded(args: StudioLoadedTelemetry): void { export function trackAgentDeploySucceeded( args: AgentDeploySucceededTelemetry, ): void { - trackStudioEvent("studio_agent_deploy_succeeded", { + trackStudioEvent("studio_agent_deploy", { ...agentDeployCategories(args), + deploy_status: "succeeded", runtime_id: args.runtimeId, }); } export function trackAgentDeployFailed(args: AgentDeployFailedTelemetry): void { - trackStudioEvent("studio_agent_deploy_failed", { + trackStudioEvent("studio_agent_deploy", { ...agentDeployCategories(args), + deploy_status: "failed", failed_phase: args.phase, error_kind: agentDeployErrorKind(args.error, args.phase), + error_summary: telemetryErrorSummary(args.error), }); } export function trackSandboxCreateSucceeded( args: SandboxCreateSucceededTelemetry, ): void { - trackStudioEvent("studio_sandbox_create_succeeded", { + trackStudioEvent("studio_sandbox_create", { + sandbox_status: "succeeded", sandbox_kind: args.kind, sandbox_source: args.source, sandbox_session_id: args.sessionId, @@ -96,9 +119,11 @@ export function trackSandboxCreateSucceeded( } export function trackSandboxCreateFailed(args: SandboxCreateFailedTelemetry): void { - trackStudioEvent("studio_sandbox_create_failed", { + trackStudioEvent("studio_sandbox_create", { + sandbox_status: "failed", sandbox_kind: args.kind, sandbox_source: args.source, error_kind: sandboxCreateErrorKind(args.error), + error_summary: telemetryErrorSummary(args.error), }); } diff --git a/frontend/src/automations/feishu/FeishuBotIntegration.tsx b/frontend/src/automations/feishu/FeishuBotIntegration.tsx index 3bf5debf..e49b2eb3 100644 --- a/frontend/src/automations/feishu/FeishuBotIntegration.tsx +++ b/frontend/src/automations/feishu/FeishuBotIntegration.tsx @@ -12,6 +12,10 @@ import { type DeployAgentkitResult, type DeployStage, } from "../../adk/client"; +import { + trackAgentDeployFailed, + trackAgentDeploySucceeded, +} from "../../adk/telemetryEvents"; import feishuLogo from "../../assets/feishu-logo.svg"; import { agentNameProblem } from "../../create/agentNameValidation"; import { TextShimmer } from "../../ui/text-shimmer/TextShimmer"; @@ -101,6 +105,7 @@ export function FeishuBotIntegration({ onBack }: FeishuBotIntegrationProps) { const regionOptionRefs = useRef>([]); const regionFocusIndexRef = useRef(0); const taskIdRef = useRef(null); + const latestPhaseRef = useRef("prepare"); const cancelledRef = useRef(false); const mountedRef = useRef(true); @@ -166,6 +171,7 @@ export function FeishuBotIntegration({ onBack }: FeishuBotIntegrationProps) { const taskId = crypto.randomUUID(); taskIdRef.current = taskId; + latestPhaseRef.current = "prepare"; cancelledRef.current = false; setDeploymentStatus("preparing"); setActiveStage(null); @@ -179,18 +185,44 @@ export function FeishuBotIntegration({ onBack }: FeishuBotIntegrationProps) { region, taskId, onStage: (stage) => { + latestPhaseRef.current = stage.phase || "deploy"; if (!mountedRef.current || cancelledRef.current) return; setDeploymentStatus("running"); setActiveStage(stage); }, }); if (!mountedRef.current || cancelledRef.current) return; + trackAgentDeploySucceeded({ + telemetry: { + source: "feishu_automation", + createMode: "feishu_template", + aiAssisted: false, + }, + action: "create", + region, + networkType: "public", + feishuEnabled: true, + runtimeId: deployed.runtimeId || "", + }); setResult(deployed); setAppSecret(""); setShowSecret(false); setDeploymentStatus("succeeded"); } catch (error) { if (!mountedRef.current || cancelledRef.current) return; + trackAgentDeployFailed({ + telemetry: { + source: "feishu_automation", + createMode: "feishu_template", + aiAssisted: false, + }, + action: "create", + region, + networkType: "public", + feishuEnabled: true, + phase: latestPhaseRef.current, + error, + }); setDeploymentStatus("failed"); setDeployError(error instanceof Error ? error.message : String(error)); } finally { diff --git a/frontend/src/create/CodePackageCreate.tsx b/frontend/src/create/CodePackageCreate.tsx index 06653240..9b76b141 100644 --- a/frontend/src/create/CodePackageCreate.tsx +++ b/frontend/src/create/CodePackageCreate.tsx @@ -190,7 +190,11 @@ export function CodePackageCreate({ onNetworkChange={setNetwork} deployRegion={deployRegion} onDeployRegionChange={setDeployRegion} - deploymentTelemetrySource="code_package" + deploymentTelemetry={{ + source: "code_package", + createMode: "code_package", + aiAssisted: false, + }} onBack={onBack} backLabel="返回创建方式" deployDisabled={!project || reading} diff --git a/frontend/src/create/CustomCreate.tsx b/frontend/src/create/CustomCreate.tsx index e837a2f6..8d42eec6 100644 --- a/frontend/src/create/CustomCreate.tsx +++ b/frontend/src/create/CustomCreate.tsx @@ -2473,6 +2473,8 @@ interface CustomCreateProps extends CreateModeProps { features?: UiFeatures; /** Publish deploy progress into the persistent app header. */ onDeploymentTaskChange?: (task: DeploymentTaskUpdate) => void; + /** Specific creation path inside the scratch flow. */ + createMode?: "custom" | "yaml_import"; /** Existing Runtime target when editing an Agent from the library. */ deploymentTarget?: { runtimeId: string; @@ -2500,6 +2502,7 @@ export function CustomCreate({ initialDraft, features, onDeploymentTaskChange, + createMode = "custom", deploymentTarget, initialDeployRegion = "cn-beijing", onDeploymentComplete, @@ -2516,6 +2519,7 @@ export function CustomCreate({ const [aiRequirement, setAiRequirement] = useState(""); const [aiGenerating, setAiGenerating] = useState(false); const [aiGenerated, setAiGenerated] = useState(false); + const [usedAiGeneration, setUsedAiGeneration] = useState(false); const [aiErrorDialog, setAiErrorDialog] = useState(null); const trimmedAiRequirement = aiRequirement.trim(); const aiRequirementError = @@ -2757,6 +2761,7 @@ export function CustomCreate({ setShowErrors(false); setBuildErr(""); setAiGenerated(true); + setUsedAiGeneration(true); } catch (error) { setAiErrorDialog( error instanceof Error ? error.message : String(error), @@ -4104,7 +4109,11 @@ export function CustomCreate({ } deployRegion={deployRegion} onDeployRegionChange={setDeployRegion} - deploymentTelemetrySource="custom_create" + deploymentTelemetry={{ + source: "scratch", + createMode, + aiAssisted: usedAiGeneration, + }} onExportYaml={() => downloadText( `${draft.name || "agent"}.yaml`, diff --git a/frontend/src/create/IntelligentCreate.tsx b/frontend/src/create/IntelligentCreate.tsx index c7ef191e..2c63cc13 100644 --- a/frontend/src/create/IntelligentCreate.tsx +++ b/frontend/src/create/IntelligentCreate.tsx @@ -471,7 +471,11 @@ export function IntelligentCreate({ onDeploy={handleDeploy} onAgentAdded={onAgentAdded} onDeploymentTaskChange={onDeploymentTaskChange} - deploymentTelemetrySource="intelligent_create" + deploymentTelemetry={{ + source: "scratch", + createMode: "intelligent", + aiAssisted: true, + }} /> ) : (
diff --git a/frontend/src/ui/ProjectPreview.tsx b/frontend/src/ui/ProjectPreview.tsx index a645654a..ab4cb69e 100644 --- a/frontend/src/ui/ProjectPreview.tsx +++ b/frontend/src/ui/ProjectPreview.tsx @@ -78,7 +78,7 @@ import { import { trackAgentDeployFailed, trackAgentDeploySucceeded, - type DeploymentTelemetrySource, + type DeploymentTelemetryOrigin, } from "../adk/telemetryEvents"; import feishuLogo from "../assets/feishu-logo.svg"; import { buildZip } from "./zip"; @@ -703,8 +703,8 @@ export interface ProjectPreviewProps { deployRegion?: string; /** Called when the user changes the deploy region. */ onDeployRegionChange?: (region: string) => void; - /** Creation entry used to group Studio deployment telemetry. */ - deploymentTelemetrySource?: DeploymentTelemetrySource; + /** Creation entry and method used to group Studio deployment telemetry. */ + deploymentTelemetry?: DeploymentTelemetryOrigin; /** Deploy-page toolbar actions. */ onBack?: () => void; backLabel?: string; @@ -830,7 +830,11 @@ export function ProjectPreview({ onNetworkChange, deployRegion = "cn-beijing", onDeployRegionChange, - deploymentTelemetrySource = "unknown", + deploymentTelemetry = { + source: "unknown", + createMode: "unknown", + aiAssisted: false, + }, onBack, backLabel = "返回配置", onExportYaml, @@ -1000,7 +1004,7 @@ export function ProjectPreview({ project.files.find((f) => f.path === selected) ?? null; const networkMode = network?.mode ?? "public"; const deploymentTelemetryBase = () => ({ - source: deploymentTelemetrySource, + telemetry: deploymentTelemetry, action: deploymentRuntimeId ? "update" as const : "create" as const, region: deployRegion, networkType: networkMode, diff --git a/frontend/tests/studioTelemetry.test.mjs b/frontend/tests/studioTelemetry.test.mjs index 4f6ef623..b72cc46e 100644 --- a/frontend/tests/studioTelemetry.test.mjs +++ b/frontend/tests/studioTelemetry.test.mjs @@ -40,6 +40,10 @@ const codePackageCreateSource = readFileSync( new URL("../src/create/CodePackageCreate.tsx", import.meta.url), "utf8", ); +const feishuIntegrationSource = readFileSync( + new URL("../src/automations/feishu/FeishuBotIntegration.tsx", import.meta.url), + "utf8", +); function sourceFiles(dirUrl) { return readdirSync(dirUrl, { withFileTypes: true }).flatMap((entry) => { @@ -80,25 +84,33 @@ test("tracks Studio load, authenticated users, Agent deploy results, and Sandbox assert.match(appSource, /userId: access\.telemetry\.userId/); assert.doesNotMatch(telemetrySource, /function identityName/); assert.match(telemetrySource, /name !== "studio_instance_loaded"/); - assert.match(telemetrySource, /"studio_agent_deploy_succeeded"/); - assert.match(telemetrySource, /"studio_agent_deploy_failed"/); - assert.match(telemetrySource, /"studio_sandbox_create_succeeded"/); - assert.match(telemetrySource, /"studio_sandbox_create_failed"/); + assert.match(telemetrySource, /"studio_agent_deploy"/); + assert.match(telemetrySource, /"studio_sandbox_create"/); + assert.doesNotMatch(telemetrySource, /"studio_agent_deploy_succeeded"/); + assert.doesNotMatch(telemetrySource, /"studio_agent_deploy_failed"/); + assert.doesNotMatch(telemetrySource, /"studio_sandbox_create_succeeded"/); + assert.doesNotMatch(telemetrySource, /"studio_sandbox_create_failed"/); assert.match(telemetrySource, /user_id: userId/); assert.doesNotMatch(clientSource, /deployerId/); assert.doesNotMatch(telemetrySource, /deployer_id/); - assert.match(telemetryEventsSource, /trackStudioEvent\("studio_agent_deploy_succeeded"/); - assert.match(telemetryEventsSource, /trackStudioEvent\("studio_agent_deploy_failed"/); + assert.match(telemetryEventsSource, /trackStudioEvent\("studio_agent_deploy"/); + assert.match(telemetryEventsSource, /deploy_status: "succeeded"/); + assert.match(telemetryEventsSource, /deploy_status: "failed"/); + assert.match(telemetryEventsSource, /deploy_source: args\.telemetry\.source/); + assert.match(telemetryEventsSource, /create_mode: args\.telemetry\.createMode/); + assert.match(telemetryEventsSource, /ai_assisted: args\.telemetry\.aiAssisted/); assert.match(telemetryEventsSource, /runtime_id: args\.runtimeId/); assert.match(telemetryEventsSource, /failed_phase: args\.phase/); assert.match(telemetryEventsSource, /error_kind: agentDeployErrorKind\(args\.error, args\.phase\)/); + assert.match(telemetryEventsSource, /error_summary: telemetryErrorSummary\(args\.error\)/); assert.match(projectPreviewSource, /trackAgentDeploySucceeded/); assert.match(projectPreviewSource, /trackAgentDeployFailed/); assert.doesNotMatch(projectPreviewSource, /trackStudioEvent/); assert.doesNotMatch(projectPreviewSource, /function deploymentErrorKind/); assert.doesNotMatch(projectPreviewSource, /studio_agent_deploy_started/); - assert.match(telemetryEventsSource, /trackStudioEvent\("studio_sandbox_create_succeeded"/); - assert.match(telemetryEventsSource, /trackStudioEvent\("studio_sandbox_create_failed"/); + assert.match(telemetryEventsSource, /trackStudioEvent\("studio_sandbox_create"/); + assert.match(telemetryEventsSource, /sandbox_status: "succeeded"/); + assert.match(telemetryEventsSource, /sandbox_status: "failed"/); assert.match(telemetryEventsSource, /sandbox_kind: args\.kind/); assert.match(telemetryEventsSource, /sandbox_source: args\.source/); assert.match(telemetryEventsSource, /sandbox_session_id: args\.sessionId/); @@ -113,14 +125,17 @@ test("tracks Studio load, authenticated users, Agent deploy results, and Sandbox test("keeps telemetry event schema and error classification outside UI components", () => { assert.match(telemetryEventsSource, /export type DeploymentTelemetrySource/); + assert.match(telemetryEventsSource, /export type DeploymentCreateMode/); + assert.match(telemetryEventsSource, /export interface DeploymentTelemetryOrigin/); assert.match(telemetryEventsSource, /agentsSource: "local" \| "cloud"/); assert.match(telemetryEventsSource, /export type SandboxTelemetryKind = "codex" \| SandboxAgentKind/); assert.match(telemetryEventsSource, /function agentDeployCategories/); assert.match(telemetryClassifiersSource, /export function agentDeployErrorKind/); assert.match(telemetryClassifiersSource, /export function sandboxCreateErrorKind/); + assert.match(telemetryClassifiersSource, /export function telemetryErrorSummary/); assert.match(telemetryClassifiersSource, /name === "RuntimeProbeError"/); assert.doesNotMatch(telemetryClassifiersSource, /from "\.\/client"/); - assert.doesNotMatch(projectPreviewSource, /export type DeploymentTelemetrySource/); + assert.doesNotMatch(projectPreviewSource, /export type DeploymentTelemetryOrigin/); assert.doesNotMatch(projectPreviewSource, /deploy_source:/); assert.doesNotMatch(projectPreviewSource, /runtime_network_type:/); assert.doesNotMatch(appSource, /sandbox_kind:/); @@ -141,8 +156,22 @@ test("keeps raw Studio event reporting behind telemetry event wrappers", () => { assert.deepEqual(offenders, []); }); -test("tags deploy telemetry with the creation workflow source", () => { - assert.match(customCreateSource, /deploymentTelemetrySource="custom_create"/); - assert.match(intelligentCreateSource, /deploymentTelemetrySource="intelligent_create"/); - assert.match(codePackageCreateSource, /deploymentTelemetrySource="code_package"/); +test("tags deploy telemetry with source, create mode, and AI assistance", () => { + assert.match(appSource, /setCustomCreateMode\("custom"\)/); + assert.match(appSource, /setCustomCreateMode\("yaml_import"\)/); + assert.match(appSource, /createMode=\{customCreateMode\}/); + assert.match(customCreateSource, /source: "scratch"/); + assert.match(customCreateSource, /createMode,/); + assert.match(customCreateSource, /aiAssisted: usedAiGeneration/); + assert.match(customCreateSource, /setUsedAiGeneration\(true\)/); + assert.match(intelligentCreateSource, /source: "scratch"/); + assert.match(intelligentCreateSource, /createMode: "intelligent"/); + assert.match(intelligentCreateSource, /aiAssisted: true/); + assert.match(codePackageCreateSource, /source: "code_package"/); + assert.match(codePackageCreateSource, /createMode: "code_package"/); + assert.match(codePackageCreateSource, /aiAssisted: false/); + assert.match(feishuIntegrationSource, /source: "feishu_automation"/); + assert.match(feishuIntegrationSource, /createMode: "feishu_template"/); + assert.match(feishuIntegrationSource, /trackAgentDeploySucceeded/); + assert.match(feishuIntegrationSource, /trackAgentDeployFailed/); }); From 184b7f04e63949d386e064c1fccaf70a4012ff53 Mon Sep 17 00:00:00 2001 From: zhengchuyi Date: Thu, 6 Aug 2026 18:42:23 +0800 Subject: [PATCH 2/5] feat(studio): track agent debug starts --- frontend/src/adk/telemetry.ts | 3 +- frontend/src/adk/telemetryClassifiers.ts | 8 +++++ frontend/src/adk/telemetryEvents.ts | 43 ++++++++++++++++++++++++ frontend/src/create/CustomCreate.tsx | 20 +++++++++++ frontend/tests/studioTelemetry.test.mjs | 13 ++++++- 5 files changed, 85 insertions(+), 2 deletions(-) diff --git a/frontend/src/adk/telemetry.ts b/frontend/src/adk/telemetry.ts index 30077a89..f85f932a 100644 --- a/frontend/src/adk/telemetry.ts +++ b/frontend/src/adk/telemetry.ts @@ -8,7 +8,8 @@ export type StudioTelemetryEventName = | "studio_instance_loaded" | "studio_user_authenticated" | "studio_agent_deploy" - | "studio_sandbox_create"; + | "studio_sandbox_create" + | "studio_agent_debug"; export interface StudioTelemetryEventOptions { dedupeKey?: string; diff --git a/frontend/src/adk/telemetryClassifiers.ts b/frontend/src/adk/telemetryClassifiers.ts index c6dc77a1..eec774d2 100644 --- a/frontend/src/adk/telemetryClassifiers.ts +++ b/frontend/src/adk/telemetryClassifiers.ts @@ -6,6 +6,14 @@ export function sandboxCreateErrorKind(error: unknown): string { return "unknown"; } +export function agentDebugErrorKind(error: unknown): string { + if ((error as Error | undefined)?.name === "AbortError") return "abort"; + if (error instanceof Error && error.name && error.name !== "Error") { + return error.name; + } + return "unknown"; +} + export function agentDeployErrorKind(error: unknown, phase: string): string { if (phase === "build") return "build_failed"; if ((error as Error | undefined)?.name === "RuntimeProbeError") { diff --git a/frontend/src/adk/telemetryEvents.ts b/frontend/src/adk/telemetryEvents.ts index 8c370e44..98a6d007 100644 --- a/frontend/src/adk/telemetryEvents.ts +++ b/frontend/src/adk/telemetryEvents.ts @@ -1,4 +1,5 @@ import { + agentDebugErrorKind, agentDeployErrorKind, sandboxCreateErrorKind, telemetryErrorSummary, @@ -64,6 +65,19 @@ export interface SandboxCreateFailedTelemetry extends SandboxCreateTelemetryBase error: unknown; } +export type AgentDebugVariantType = "baseline" | "comparison"; +export type AgentDebugFailedPhase = "create_test_run" | "create_test_session"; + +export interface AgentDebugTelemetryBase { + durationMs: number; + variantType?: AgentDebugVariantType; +} + +export interface AgentDebugFailedTelemetry extends AgentDebugTelemetryBase { + phase?: AgentDebugFailedPhase; + error: unknown; +} + function agentDeployCategories(args: AgentDeployTelemetryBase) { return { deploy_source: args.telemetry.source, @@ -127,3 +141,32 @@ export function trackSandboxCreateFailed(args: SandboxCreateFailedTelemetry): vo error_summary: telemetryErrorSummary(args.error), }); } + +export function trackAgentDebugSucceeded(args: AgentDebugTelemetryBase): void { + trackStudioEvent( + "studio_agent_debug", + { + debug_status: "succeeded", + variant_type: args.variantType, + }, + { + duration_ms: args.durationMs, + }, + ); +} + +export function trackAgentDebugFailed(args: AgentDebugFailedTelemetry): void { + trackStudioEvent( + "studio_agent_debug", + { + debug_status: "failed", + variant_type: args.variantType, + failed_phase: args.phase, + error_kind: agentDebugErrorKind(args.error), + error_summary: telemetryErrorSummary(args.error), + }, + { + duration_ms: args.durationMs, + }, + ); +} diff --git a/frontend/src/create/CustomCreate.tsx b/frontend/src/create/CustomCreate.tsx index 8d42eec6..19527ab3 100644 --- a/frontend/src/create/CustomCreate.tsx +++ b/frontend/src/create/CustomCreate.tsx @@ -114,6 +114,11 @@ import { generateAgentProject, runGeneratedAgentTestSSE, } from "../adk/client"; +import { + trackAgentDebugFailed, + trackAgentDebugSucceeded, + type AgentDebugFailedPhase, +} from "../adk/telemetryEvents"; import type { DeployStage, GeneratedAgentTestRun, @@ -3045,6 +3050,9 @@ export function CustomCreate({ setDebugInput(""); let createdRun: GeneratedAgentTestRun | null = null; + let failedPhase: AgentDebugFailedPhase | undefined; + const debugStartedAt = Date.now(); + const variantType = id === "baseline" ? "baseline" : "comparison"; try { await cleanupDebugVariantRun(id); await cleanupStoredDebugRuns(); @@ -3054,6 +3062,7 @@ export function CustomCreate({ description: variant.description, instruction: variant.instruction, }; + failedPhase = "create_test_run"; createdRun = await createGeneratedAgentTestRun( debugRuntimeDraft(variantDraft), deploymentTarget @@ -3064,6 +3073,7 @@ export function CustomCreate({ : undefined, ); rememberDebugTestRun(createdRun.runId); + failedPhase = "create_test_session"; const sessionId = await createGeneratedAgentTestSession( createdRun.runId, "test_user", @@ -3077,6 +3087,10 @@ export function CustomCreate({ : item, ), ); + trackAgentDebugSucceeded({ + durationMs: Date.now() - debugStartedAt, + variantType, + }); } catch (err) { if (createdRun) { try { @@ -3098,6 +3112,12 @@ export function CustomCreate({ : item, ), ); + trackAgentDebugFailed({ + durationMs: Date.now() - debugStartedAt, + variantType, + phase: failedPhase, + error: err, + }); } }; diff --git a/frontend/tests/studioTelemetry.test.mjs b/frontend/tests/studioTelemetry.test.mjs index b72cc46e..dfb8fd54 100644 --- a/frontend/tests/studioTelemetry.test.mjs +++ b/frontend/tests/studioTelemetry.test.mjs @@ -76,7 +76,7 @@ test("initializes APMPlus lazily and sends Studio custom events", () => { assert.match(telemetrySource, /ev_type: "custom"/); }); -test("tracks Studio load, authenticated users, Agent deploy results, and Sandbox creation results", () => { +test("tracks Studio load, authenticated users, Agent deploy, Sandbox creation, and Agent debug results", () => { assert.match(appSource, /initStudioTelemetry\(cfg\.telemetry\)/); assert.match(appSource, /trackStudioLoaded/); assert.match(telemetryEventsSource, /"studio_instance_loaded"/); @@ -86,6 +86,7 @@ test("tracks Studio load, authenticated users, Agent deploy results, and Sandbox assert.match(telemetrySource, /name !== "studio_instance_loaded"/); assert.match(telemetrySource, /"studio_agent_deploy"/); assert.match(telemetrySource, /"studio_sandbox_create"/); + assert.match(telemetrySource, /"studio_agent_debug"/); assert.doesNotMatch(telemetrySource, /"studio_agent_deploy_succeeded"/); assert.doesNotMatch(telemetrySource, /"studio_agent_deploy_failed"/); assert.doesNotMatch(telemetrySource, /"studio_sandbox_create_succeeded"/); @@ -121,6 +122,15 @@ test("tracks Studio load, authenticated users, Agent deploy results, and Sandbox assert.doesNotMatch(appSource, /function sandboxTelemetryErrorKind/); assert.doesNotMatch(appSource, /studio_sandbox_create_started/); assert.doesNotMatch(telemetrySource, /studio_sandbox_create_started/); + assert.match(telemetryEventsSource, /trackStudioEvent\(\s*"studio_agent_debug"/); + assert.match(telemetryEventsSource, /debug_status: "succeeded"/); + assert.match(telemetryEventsSource, /debug_status: "failed"/); + assert.match(telemetryEventsSource, /variant_type: args\.variantType/); + assert.match(telemetryEventsSource, /failed_phase: args\.phase/); + assert.match(telemetryEventsSource, /error_kind: agentDebugErrorKind\(args\.error\)/); + assert.match(telemetryEventsSource, /duration_ms: args\.durationMs/); + assert.match(customCreateSource, /trackAgentDebugSucceeded/); + assert.match(customCreateSource, /trackAgentDebugFailed/); }); test("keeps telemetry event schema and error classification outside UI components", () => { @@ -131,6 +141,7 @@ test("keeps telemetry event schema and error classification outside UI component assert.match(telemetryEventsSource, /export type SandboxTelemetryKind = "codex" \| SandboxAgentKind/); assert.match(telemetryEventsSource, /function agentDeployCategories/); assert.match(telemetryClassifiersSource, /export function agentDeployErrorKind/); + assert.match(telemetryClassifiersSource, /export function agentDebugErrorKind/); assert.match(telemetryClassifiersSource, /export function sandboxCreateErrorKind/); assert.match(telemetryClassifiersSource, /export function telemetryErrorSummary/); assert.match(telemetryClassifiersSource, /name === "RuntimeProbeError"/); From d1a65544cfcc733e813284e04500e7b740b434b2 Mon Sep 17 00:00:00 2001 From: zhengchuyi Date: Thu, 6 Aug 2026 19:27:26 +0800 Subject: [PATCH 3/5] feat(studio): track agent connections --- frontend/src/App.tsx | 173 ++++++++++++++++----- frontend/src/adk/telemetry.ts | 3 +- frontend/src/adk/telemetryClassifiers.ts | 8 + frontend/src/adk/telemetryEvents.ts | 64 ++++++++ frontend/src/ui/AgentSelector.tsx | 44 +++++- frontend/tests/newChatAgentPicker.test.mjs | 4 +- frontend/tests/studioTelemetry.test.mjs | 24 ++- 7 files changed, 269 insertions(+), 51 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 91e54fef..de1b8f30 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -308,9 +308,12 @@ import { initStudioTelemetry, } from "./adk/telemetry"; import { + trackAgentConnectFailed, + trackAgentConnectSucceeded, trackSandboxCreateFailed, trackSandboxCreateSucceeded, trackStudioLoaded, + type AgentConnectSource, } from "./adk/telemetryEvents"; import type { A2uiAction, A2uiComponent } from "./a2ui/types"; import { buildSurfaces } from "./a2ui/Surface"; @@ -2352,32 +2355,59 @@ export default function App() { } } - async function openSandboxAgent(session: SandboxSessionInfo) { + async function openSandboxAgent( + session: SandboxSessionInfo, + source: AgentConnectSource = "my_agents", + ) { setError(""); - if (session.toolName === "codex") { - const connected = await sandboxClient.connectSession(session.id); - viewSidRef.current = ""; - setSessionId(""); - setPendingTurns([]); - setInput(""); - setInvocation(emptyInvocation()); - releaseAllSandboxPreviews(); - setSandboxTurns([]); - setSandboxSession(connected); + const startedAt = Date.now(); + try { + if (session.toolName === "codex") { + const connected = await sandboxClient.connectSession(session.id); + trackAgentConnectSucceeded({ + kind: session.toolName, + source, + durationMs: Date.now() - startedAt, + sandboxStatus: connected.status, + }); + viewSidRef.current = ""; + setSessionId(""); + setPendingTurns([]); + setInput(""); + setInvocation(emptyInvocation()); + releaseAllSandboxPreviews(); + setSandboxTurns([]); + setSandboxSession(connected); + setSandboxAgentDetailTarget(null); + setSandboxAgentWorkspace(null); + setMyAgents(false); + setManageAgents(false); + return; + } + const workspace = await sandboxClient.openAgentSession( + session.toolName, + session.id, + ); + trackAgentConnectSucceeded({ + kind: session.toolName, + source, + durationMs: Date.now() - startedAt, + sandboxStatus: workspace.session.status, + }); + setSandboxAgentWorkspace(workspace); setSandboxAgentDetailTarget(null); - setSandboxAgentWorkspace(null); setMyAgents(false); setManageAgents(false); - return; + } catch (cause) { + trackAgentConnectFailed({ + kind: session.toolName, + source, + durationMs: Date.now() - startedAt, + error: cause, + }); + setError(cause instanceof Error ? cause.message : String(cause)); + throw cause; } - const workspace = await sandboxClient.openAgentSession( - session.toolName, - session.id, - ); - setSandboxAgentWorkspace(workspace); - setSandboxAgentDetailTarget(null); - setMyAgents(false); - setManageAgents(false); } function openSandboxAgentDetails(session: SandboxSessionInfo) { @@ -3883,8 +3913,12 @@ export default function App() { setError(""); }; - const connectMyAgent = async (agent: MyAgentCardData, rethrow = false) => { - if (!agent.runtime) return; + const connectRuntimeForUser = async ( + agent: MyAgentCardData, + source: AgentConnectSource, + ): Promise => { + if (!agent.runtime) throw new Error("缺少 Runtime 信息，无法连接智能体。"); + const startedAt = Date.now(); try { const agentId = await connectRuntime( agent.runtime.runtimeId, @@ -3892,11 +3926,40 @@ export default function App() { agent.runtime.region, agent.runtime.currentVersion, ); + trackAgentConnectSucceeded({ + kind: "runtime", + source, + durationMs: Date.now() - startedAt, + runtimeRegion: agent.runtime.region, + runtimeIsMine: agent.isMine, + }); + return agentId; + } catch (error) { + trackAgentConnectFailed({ + kind: "runtime", + source, + durationMs: Date.now() - startedAt, + error, + }); + throw error; + } + }; + + const connectMyAgent = async ( + agent: MyAgentCardData, + options: { rethrow?: boolean; source?: AgentConnectSource } = {}, + ) => { + if (!agent.runtime) return; + try { + const agentId = await connectRuntimeForUser( + agent, + options.source ?? "my_agents", + ); await refreshCurrentAgentAndStartNewChat(agentId); } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); setError(message); - if (rethrow) throw new Error(message); + if (options.rethrow) throw new Error(message); } }; @@ -3964,6 +4027,7 @@ export default function App() { setFeedbackCaseReturnAgentId(""); setFeedbackTargetEventId(""); if (agent.runtimeId && agent.id.startsWith("detail:")) { + const startedAt = Date.now(); try { const agentId = await connectRuntime( agent.runtimeId, @@ -3971,8 +4035,20 @@ export default function App() { agent.region ?? "cn-beijing", agent.currentVersion, ); + trackAgentConnectSucceeded({ + kind: "runtime", + source: "agent_workspace", + durationMs: Date.now() - startedAt, + runtimeRegion: agent.region, + }); await refreshCurrentAgentAndStartNewChat(agentId); } catch (cause) { + trackAgentConnectFailed({ + kind: "runtime", + source: "agent_workspace", + durationMs: Date.now() - startedAt, + error: cause, + }); setError(cause instanceof Error ? cause.message : String(cause)); } return; @@ -4301,24 +4377,29 @@ export default function App() { selectedRuntimeId={currentRuntime?.runtimeId} runtimeScope={access.capabilities.runtimeScope} onSelectRuntime={async (runtime) => { - await connectMyAgent({ - id: runtime.runtimeId, - name: runtime.name, - description: runtime.description?.trim() || "暂无描述", - createdAt: runtime.createdAt ?? "", - specificationLabel: "地域", - specification: - runtime.region === "cn-shanghai" ? "上海" : "北京", - isMine: runtime.isMine, - runtime: { - runtimeId: runtime.runtimeId, - region: runtime.region, - currentVersion: runtime.currentVersion, - canDelete: runtime.canDelete, + await connectMyAgent( + { + id: runtime.runtimeId, + name: runtime.name, + description: runtime.description?.trim() || "暂无描述", + createdAt: runtime.createdAt ?? "", + specificationLabel: "地域", + specification: + runtime.region === "cn-shanghai" ? "上海" : "北京", + isMine: runtime.isMine, + runtime: { + runtimeId: runtime.runtimeId, + region: runtime.region, + currentVersion: runtime.currentVersion, + canDelete: runtime.canDelete, + }, }, - }, true); + { rethrow: true, source: "new_chat_picker" }, + ); }} - onSelectSandboxSession={openSandboxAgent} + onSelectSandboxSession={(session) => + openSandboxAgent(session, "new_chat_picker") + } showModeSelector={false} temporaryEnabled={newChatCapabilitiesReady && newChatCapabilities.temporaryEnabled} skillCreateEnabled={newChatCapabilitiesReady && newChatCapabilities.skillCreateEnabled} @@ -4418,7 +4499,9 @@ export default function App() { openSandboxAgent(sandboxAgentDetailTarget)} + onOpen={() => + openSandboxAgent(sandboxAgentDetailTarget, "sandbox_detail") + } onDelete={() => deleteSandboxAgent(sandboxAgentDetailTarget)} /> ) : myAgents ? ( @@ -4426,10 +4509,14 @@ export default function App() { canCreate={canCreateAgents} runtimeScope={access.capabilities.runtimeScope} onCreateAgent={openAgentCreateFromMyAgents} - onUseAgent={connectMyAgent} + onUseAgent={(agent) => + connectMyAgent(agent, { source: "my_agents" }) + } onViewAgentDetails={openMyAgentDetails} onCreateSandboxAgent={openSandboxAgentCreate} - onUseSandboxAgent={openSandboxAgent} + onUseSandboxAgent={(session) => + openSandboxAgent(session, "my_agents") + } onViewSandboxAgentDetails={openSandboxAgentDetails} sandboxRefreshKey={sandboxAgentRefreshKey} connectedRuntimeId={connectedRuntimeId} diff --git a/frontend/src/adk/telemetry.ts b/frontend/src/adk/telemetry.ts index f85f932a..ee2d14c9 100644 --- a/frontend/src/adk/telemetry.ts +++ b/frontend/src/adk/telemetry.ts @@ -9,7 +9,8 @@ export type StudioTelemetryEventName = | "studio_user_authenticated" | "studio_agent_deploy" | "studio_sandbox_create" - | "studio_agent_debug"; + | "studio_agent_debug" + | "studio_agent_connect"; export interface StudioTelemetryEventOptions { dedupeKey?: string; diff --git a/frontend/src/adk/telemetryClassifiers.ts b/frontend/src/adk/telemetryClassifiers.ts index eec774d2..12ca8c84 100644 --- a/frontend/src/adk/telemetryClassifiers.ts +++ b/frontend/src/adk/telemetryClassifiers.ts @@ -14,6 +14,14 @@ export function agentDebugErrorKind(error: unknown): string { return "unknown"; } +export function agentConnectErrorKind(error: unknown): string { + if ((error as Error | undefined)?.name === "AbortError") return "abort"; + if (error instanceof Error && error.name && error.name !== "Error") { + return error.name; + } + return "unknown"; +} + export function agentDeployErrorKind(error: unknown, phase: string): string { if (phase === "build") return "build_failed"; if ((error as Error | undefined)?.name === "RuntimeProbeError") { diff --git a/frontend/src/adk/telemetryEvents.ts b/frontend/src/adk/telemetryEvents.ts index 98a6d007..7717ad4e 100644 --- a/frontend/src/adk/telemetryEvents.ts +++ b/frontend/src/adk/telemetryEvents.ts @@ -1,4 +1,5 @@ import { + agentConnectErrorKind, agentDebugErrorKind, agentDeployErrorKind, sandboxCreateErrorKind, @@ -78,6 +79,34 @@ export interface AgentDebugFailedTelemetry extends AgentDebugTelemetryBase { error: unknown; } +export type AgentConnectKind = + | "runtime" + | "local" + | SandboxTelemetryKind; + +export type AgentConnectSource = + | "new_chat_picker" + | "my_agents" + | "agent_workspace" + | "navbar_picker" + | "sandbox_detail"; + +export interface AgentConnectTelemetryBase { + kind: AgentConnectKind; + source: AgentConnectSource; + durationMs: number; +} + +export interface AgentConnectSucceededTelemetry extends AgentConnectTelemetryBase { + runtimeRegion?: string; + runtimeIsMine?: boolean; + sandboxStatus?: string; +} + +export interface AgentConnectFailedTelemetry extends AgentConnectTelemetryBase { + error: unknown; +} + function agentDeployCategories(args: AgentDeployTelemetryBase) { return { deploy_source: args.telemetry.source, @@ -170,3 +199,38 @@ export function trackAgentDebugFailed(args: AgentDebugFailedTelemetry): void { }, ); } + +export function trackAgentConnectSucceeded( + args: AgentConnectSucceededTelemetry, +): void { + trackStudioEvent( + "studio_agent_connect", + { + connect_status: "succeeded", + agent_kind: args.kind, + connect_source: args.source, + runtime_region: args.runtimeRegion, + runtime_is_mine: args.runtimeIsMine, + sandbox_status: args.sandboxStatus, + }, + { + duration_ms: args.durationMs, + }, + ); +} + +export function trackAgentConnectFailed(args: AgentConnectFailedTelemetry): void { + trackStudioEvent( + "studio_agent_connect", + { + connect_status: "failed", + agent_kind: args.kind, + connect_source: args.source, + error_kind: agentConnectErrorKind(args.error), + error_summary: telemetryErrorSummary(args.error), + }, + { + duration_ms: args.durationMs, + }, + ); +} diff --git a/frontend/src/ui/AgentSelector.tsx b/frontend/src/ui/AgentSelector.tsx index 6786e639..a082e7bc 100644 --- a/frontend/src/ui/AgentSelector.tsx +++ b/frontend/src/ui/AgentSelector.tsx @@ -30,6 +30,10 @@ import { type RuntimeDetail, } from "../adk/client"; import { connectRuntime } from "../adk/connections"; +import { + trackAgentConnectFailed, + trackAgentConnectSucceeded, +} from "../adk/telemetryEvents"; import { AgentIdentityIcon } from "./AgentIdentityIcon"; import { SkillCapabilityIcon, ToolCapabilityIcon } from "./CapabilityIcons"; import { RuntimeIdentityIcon } from "./RuntimeIdentityIcon"; @@ -279,13 +283,27 @@ export function AgentSelector({ (pageCache[page + 1] !== undefined || tokens[page + 1] !== undefined); function connect(rt: CloudRuntime) { + const startedAt = Date.now(); setConnecting(rt.runtimeId); connectRuntime(rt.runtimeId, rt.name, rt.region) .then(async (agentId) => { await onSelect(agentId); + trackAgentConnectSucceeded({ + kind: "runtime", + source: "navbar_picker", + durationMs: Date.now() - startedAt, + runtimeRegion: rt.region, + runtimeIsMine: rt.isMine, + }); onClose(); }) .catch((error) => { + trackAgentConnectFailed({ + kind: "runtime", + source: "navbar_picker", + durationMs: Date.now() - startedAt, + error, + }); if (error instanceof RuntimeAccessDeniedError) { setError(error.message); return; @@ -302,6 +320,27 @@ export function AgentSelector({ .finally(() => setConnecting(null)); } + async function selectLocalApp(app: string) { + const startedAt = Date.now(); + try { + await onSelect(app); + trackAgentConnectSucceeded({ + kind: "local", + source: "navbar_picker", + durationMs: Date.now() - startedAt, + }); + onClose(); + } catch (error) { + trackAgentConnectFailed({ + kind: "local", + source: "navbar_picker", + durationMs: Date.now() - startedAt, + error, + }); + setError(error instanceof Error ? error.message : String(error)); + } + } + if (!open) return null; // The visible set: the owner's full list (mineOnly) or the current lazy page, @@ -359,10 +398,7 @@ export function AgentSelector({