Add AI agent detection to CLI analytics

packages/cli-kit/package.json

@@ -115,6 +115,7 @@
     "@opentelemetry/exporter-metrics-otlp-http": "0.57.0",
     "@opentelemetry/resources": "1.30.1",
     "@opentelemetry/sdk-metrics": "1.30.1",
+    "@vercel/detect-agent": "1.2.5",
     "ajv": "8.20.0",
     "ansi-escapes": "6.2.1",
     "archiver": "5.3.2",

packages/cli-kit/src/private/node/analytics.ts

@@ -1,12 +1,20 @@
 import {getLastSeenAuthMethod} from './session.js'
 import {getAutoUpgradeEnabled} from './conf-store.js'
+import {detectedAgentEnvironmentVariables} from './context/agent.js'
 import {hashString} from '../../public/node/crypto.js'
 import {getPackageManager, packageManagerFromUserAgent} from '../../public/node/node-package-manager.js'
 import BaseCommand from '../../public/node/base-command.js'
 import {CommandContent} from '../../public/node/hooks/prerun.js'
 import * as metadata from '../../public/node/metadata.js'
 import {platformAndArch} from '../../public/node/os.js'
-import {ciPlatform, cloudEnvironment, macAddress} from '../../public/node/context/local.js'
+import {
+  alwaysLogAnalytics,
+  alwaysLogMetrics,
+  analyticsDisabled,
+  ciPlatform,
+  cloudEnvironment,
+  macAddress,
+} from '../../public/node/context/local.js'
 import {cwd} from '../../public/node/path.js'
 import {currentProcessIsGlobal, inferPackageManagerForGlobalCLI} from '../../public/node/is-global.js'
 import {isWsl} from '../../public/node/system.js'
@@ -115,14 +123,27 @@ export async function getEnvironmentData(config: Interfaces.Config): Promise<Env
 export async function getSensitiveEnvironmentData(config: Interfaces.Config) {
   return {
     env_plugin_installed_all: JSON.stringify(getPluginNames(config)),
-    env_shopify_variables: JSON.stringify(getShopifyEnvironmentVariables()),
+    env_shopify_variables: JSON.stringify(await getShopifyEnvironmentVariables()),
   }
 }
 
-function getShopifyEnvironmentVariables() {
-  return Object.fromEntries(
-    Object.entries(process.env).filter(([key]) => allowedShopifyEnvironmentVariableNames.has(key)),
+async function getShopifyEnvironmentVariables(env: NodeJS.ProcessEnv = process.env) {
+  const declaredVariables = Object.fromEntries(
+    Object.entries(env).filter(([key]) => allowedShopifyEnvironmentVariableNames.has(key)),
   )
+
+  // An `alwaysLog*` override still sends the event, so this can't be `analyticsDisabled()` alone.
+  if (monorailAnalyticsSkipped() && metricAnalyticsSkipped()) return declaredVariables
+
+  return {...declaredVariables, ...(await detectedAgentEnvironmentVariables(env))}
+}
+
+export function monorailAnalyticsSkipped(): boolean {
+  return !alwaysLogAnalytics() && analyticsDisabled()
+}
+
+export function metricAnalyticsSkipped(): boolean {
+  return !alwaysLogMetrics() && analyticsDisabled()
 }
 
 function getPluginNames(config: Interfaces.Config) {

packages/cli-kit/src/private/node/context/agent.test.ts

@@ -0,0 +1,236 @@
+import {detectedAgentEnvironmentVariables} from './agent.js'
+import {determineAgent, type KnownAgentNames} from '@vercel/detect-agent'
+import {afterEach, describe, expect, test, vi} from 'vitest'
+
+vi.mock('@vercel/detect-agent')
+vi.mock('../../../public/node/output.js')
+
+const {determineAgent: realDetermineAgent} =
+  await vi.importActual<typeof import('@vercel/detect-agent')>('@vercel/detect-agent')
+
+function agentDetected(name: string) {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: true, agent: {name: name as KnownAgentNames}})
+}
+
+function noAgentDetected() {
+  vi.mocked(determineAgent).mockResolvedValue({isAgent: false, agent: undefined})
+}
+
+const declaredAgentVariableNames = ['SHOPIFY_CLI_AGENT_INFO', 'SHOPIFY_CLI_AGENT_IDS']
+
+const legacyAgentVariableNames = [
+  'SHOPIFY_CLI_AGENT',
+  'SHOPIFY_CLI_AGENT_VERSION',
+  'SHOPIFY_CLI_AGENT_RUN_ID',
+  'SHOPIFY_CLI_AGENT_SESSION_ID',
+  'SHOPIFY_CLI_AGENT_PROVIDER',
+]
+
+const agentVariableNamesReadByTheDependency = [
+  'AI_AGENT',
+  'CURSOR_TRACE_ID',
+  'CURSOR_AGENT',
+  'CURSOR_EXTENSION_HOST_ROLE',
+  'GEMINI_CLI',
+  'CODEX_SANDBOX',
+  'CODEX_CI',
+  'CODEX_THREAD_ID',
+  'ANTIGRAVITY_AGENT',
+  'AUGMENT_AGENT',
+  'OPENCODE_CLIENT',
+  'CLAUDECODE',
+  'CLAUDE_CODE',
+  'CLAUDE_CODE_IS_COWORK',
+  'REPL_ID',
+  'COPILOT_MODEL',
+  'COPILOT_ALLOW_ALL',
+  'COPILOT_GITHUB_TOKEN',
+]
+
+function stubDeclaredAgentVariablesEmpty() {
+  declaredAgentVariableNames.forEach((variableName) => vi.stubEnv(variableName, undefined))
+}
+
+// This host is often itself an agent session, and `AI_AGENT` outranks the rest of what the dependency reads.
+function useRealDetectionWithCleanEnvironment() {
+  vi.mocked(determineAgent).mockImplementation(realDetermineAgent)
+  stubDeclaredAgentVariablesEmpty()
+  agentVariableNamesReadByTheDependency.forEach((variableName) => vi.stubEnv(variableName, undefined))
+}
+
+describe('detectedAgentEnvironmentVariables', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs()
+  })
+
+  test('reports nothing when SHOPIFY_CLI_AGENT_INFO was declared, so a guess cannot clobber it', async () => {
+    agentDetected('claude')
+
+    const got = await detectedAgentEnvironmentVariables({
+      SHOPIFY_CLI_AGENT_INFO: 'n:shopify-ai-toolkit|v:1.0.0|p:anthropic|m:claude-opus-5',
+    })
+
+    expect(got).toEqual({})
+  })
+
+  test('reports nothing when SHOPIFY_CLI_AGENT_IDS was declared', async () => {
+    agentDetected('claude')
+
+    const got = await detectedAgentEnvironmentVariables({SHOPIFY_CLI_AGENT_IDS: 's:session-id|r:run-id'})
+
+    expect(got).toEqual({})
+  })
+
+  test('leaves a declared SHOPIFY_CLI_AGENT_INFO value untouched', async () => {
+    agentDetected('claude')
+    const env = {SHOPIFY_CLI_AGENT_INFO: 'n:shopify-ai-toolkit|v:1.0.0'}
+
+    await detectedAgentEnvironmentVariables(env)
+
+    expect(env).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:shopify-ai-toolkit|v:1.0.0'})
+  })
+
+  test.each(legacyAgentVariableNames)(
+    'reports the detected agent when only the legacy %s was declared',
+    async (declaredVariableName) => {
+      agentDetected('devin')
+
+      const got = await detectedAgentEnvironmentVariables({[declaredVariableName]: 'declared-by-the-producer'})
+
+      expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+    },
+  )
+
+  test('reports the detected agent when only SHOPIFY_INVOKED_BY was declared', async () => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({SHOPIFY_INVOKED_BY: 'shopify-ai-toolkit'})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test.each([
+    ['undefined', undefined],
+    ['empty', ''],
+    ['whitespace-only', '   '],
+  ])('treats a %s declaration as absent and reports the detected agent', async (_description, declaredValue) => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({
+      SHOPIFY_CLI_AGENT_INFO: declaredValue,
+      SHOPIFY_CLI_AGENT_IDS: declaredValue,
+    })
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test('reports the detected agent and a marker when nothing was declared', async () => {
+    agentDetected('devin')
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devin', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test('reports nothing when no agent is detected', async () => {
+    noAgentDetected()
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({})
+  })
+
+  test.each([
+    ['claude', 'claude-code'],
+    ['gemini', 'gemini-cli'],
+  ])('reports the detected name %s using the toolkit name %s', async (detectedName, expectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: `n:${expectedName}`, SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test.each([
+    ['a known name needing no translation', 'cursor'],
+    ['an arbitrary AI_AGENT pass-through value', 'claude-code_2-1-267_agent'],
+  ])('reports %s verbatim', async (_description, detectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: `n:${detectedName}`, SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test('strips the tag separator from the detected name so it cannot inject tags', async () => {
+    agentDetected('devin|v:9.9.9')
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:devinv:9.9.9', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+  })
+
+  test.each([
+    ['whitespace-only', '   '],
+    ['nothing but the tag separator', '|'],
+  ])('reports nothing when the detected name is %s', async (_description, detectedName) => {
+    agentDetected(detectedName)
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({})
+  })
+
+  test('reports nothing when detection fails', async () => {
+    vi.mocked(determineAgent).mockRejectedValue(new Error('EACCES: permission denied, stat /opt/.devin'))
+
+    const got = await detectedAgentEnvironmentVariables({})
+
+    expect(got).toEqual({})
+  })
+
+  test('does not mutate process.env when reporting a detected agent', async () => {
+    agentDetected('claude')
+    stubDeclaredAgentVariablesEmpty()
+    const environmentBefore = {...process.env}
+
+    const got = await detectedAgentEnvironmentVariables()
+
+    expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:claude-code', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+    expect({...process.env}).toEqual(environmentBefore)
+  })
+
+  describe('against the real dependency', () => {
+    test('reports an arbitrary AI_AGENT value verbatim', async () => {
+      useRealDetectionWithCleanEnvironment()
+      vi.stubEnv('AI_AGENT', 'claude-code_2-1-267_agent')
+
+      const got = await detectedAgentEnvironmentVariables()
+
+      expect(got).toEqual({
+        SHOPIFY_CLI_AGENT_INFO: 'n:claude-code_2-1-267_agent',
+        SHOPIFY_CLI_AGENT_DETECTED: 'true',
+      })
+    })
+
+    // The dependency reports bare `claude`; only driving it for real proves the map matches its vocabulary.
+    test('reports a CLAUDECODE environment using the toolkit name', async () => {
+      useRealDetectionWithCleanEnvironment()
+      vi.stubEnv('CLAUDECODE', '1')
+
+      const got = await detectedAgentEnvironmentVariables()
+
+      expect(got).toEqual({SHOPIFY_CLI_AGENT_INFO: 'n:claude-code', SHOPIFY_CLI_AGENT_DETECTED: 'true'})
+    })
+
+    test('reports nothing when a declaration exists, whatever the dependency would detect', async () => {
+      useRealDetectionWithCleanEnvironment()
+      vi.stubEnv('CLAUDECODE', '1')
+      vi.stubEnv('SHOPIFY_CLI_AGENT_IDS', 's:session-id')
+
+      const got = await detectedAgentEnvironmentVariables()
+
+      expect(got).toEqual({})
+    })
+  })
+})

packages/cli-kit/src/private/node/context/agent.ts

@@ -0,0 +1,43 @@
+import {outputDebug} from '../../../public/node/output.js'
+import {determineAgent} from '@vercel/detect-agent'
+
+const toolkitAgentNamesByDetectedName: {[detectedName: string]: string} = {
+  claude: 'claude-code',
+  gemini: 'gemini-cli',
+}
+
+const explicitAgentVariableNames = ['SHOPIFY_CLI_AGENT_INFO', 'SHOPIFY_CLI_AGENT_IDS']
+
+function hasExplicitAgentAttribution(env: NodeJS.ProcessEnv): boolean {
+  return explicitAgentVariableNames.some((variableName) => (env[variableName] ?? '').trim() !== '')
+}
+
+export async function detectedAgentEnvironmentVariables(
+  env: NodeJS.ProcessEnv = process.env,
+): Promise<NodeJS.ProcessEnv> {
+  // Emitting while a declaration exists would clobber the producer's whole packed value, not just the name.
+  if (hasExplicitAgentAttribution(env)) return {}
+
+  try {
+    const detection = await determineAgent()
+    if (!detection.isAgent) return {}
+
+    // `|` separates tags, so a name containing one could otherwise inject tags nothing detected.
+    const detectedName = detection.agent.name.replaceAll('|', '').trim()
+    if (detectedName === '') return {}
+
+    return {
+      SHOPIFY_CLI_AGENT_INFO: `n:${toolkitAgentNamesByDetectedName[detectedName] ?? detectedName}`,
+      SHOPIFY_CLI_AGENT_DETECTED: 'true',
+    }
+
+    // eslint-disable-next-line no-catch-all/no-catch-all
+  } catch (error) {
+    let message = 'Unable to detect which AI agent is running the CLI'
+    if (error instanceof Error) {
+      message = message.concat(`: ${error.message}`)
+    }
+    outputDebug(message)
+    return {}
+  }
+}