-
-
Notifications
You must be signed in to change notification settings - Fork 1.8k
feat(core)!: Remove streamGenAiSpans flag
#22495
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 30 additions & 36 deletions
66
dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/ai-streaming.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,60 +1,54 @@ | ||
| import { expect, test } from '@playwright/test'; | ||
| import { waitForTransaction } from '@sentry-internal/test-utils'; | ||
| import { getSpanOp, waitForStreamedSpan, waitForTransaction } from '@sentry-internal/test-utils'; | ||
| import type { SerializedStreamedSpan } from '@sentry/core'; | ||
|
|
||
| // Drives Workers AI through the real Cloudflare Agents SDK + Vercel AI SDK + `workers-ai-provider` | ||
| // stack (the OpenAI-compatible SSE shape, `choices[].delta.content`) from an Agent's `onRequest`. | ||
| // Asserts the streaming response text is captured on the gen_ai span — the regression seen in | ||
| // production where only input + usage survived. The model output is read from | ||
| // `gen_ai.output.messages`, so the streaming instrumentation must emit it alongside the | ||
| // deprecated `gen_ai.response.text`. | ||
| function assertGenAiStreamingSpan(spans: Array<Record<string, any>> | undefined): void { | ||
| const genAiSpan = (spans ?? []).find(span => span.op === 'gen_ai.chat'); | ||
|
|
||
| expect(genAiSpan).toBeDefined(); | ||
| expect(genAiSpan.origin).toBe('auto.ai.cloudflare.workers_ai'); | ||
| expect(genAiSpan.data).toEqual( | ||
| expect.objectContaining({ | ||
| 'sentry.origin': 'auto.ai.cloudflare.workers_ai', | ||
| 'gen_ai.operation.name': 'chat', | ||
| 'gen_ai.request.model': '@cf/meta/llama-3.1-8b-instruct', | ||
| 'gen_ai.response.streaming': true, | ||
| 'gen_ai.response.text': 'The capital of France is Paris.', | ||
| 'gen_ai.output.messages': JSON.stringify([ | ||
| { role: 'assistant', parts: [{ type: 'text', content: 'The capital of France is Paris.' }] }, | ||
| ]), | ||
| 'gen_ai.usage.input_tokens': 15, | ||
| 'gen_ai.usage.output_tokens': 8, | ||
| 'gen_ai.usage.total_tokens': 23, | ||
| }), | ||
| function assertGenAiStreamingSpan(span: SerializedStreamedSpan): void { | ||
| expect(getSpanOp(span)).toBe('gen_ai.chat'); | ||
| expect(span.attributes['sentry.origin']?.value).toBe('auto.ai.cloudflare.workers_ai'); | ||
| expect(span.attributes['gen_ai.operation.name']?.value).toBe('chat'); | ||
| expect(span.attributes['gen_ai.request.model']?.value).toBe('@cf/meta/llama-3.1-8b-instruct'); | ||
| expect(span.attributes['gen_ai.response.streaming']?.value).toBe(true); | ||
| expect(span.attributes['gen_ai.response.text']?.value).toBe('The capital of France is Paris.'); | ||
| expect(span.attributes['gen_ai.output.messages']?.value).toBe( | ||
| JSON.stringify([{ role: 'assistant', parts: [{ type: 'text', content: 'The capital of France is Paris.' }] }]), | ||
| ); | ||
| expect(span.attributes['gen_ai.usage.input_tokens']?.value).toBe(15); | ||
| expect(span.attributes['gen_ai.usage.output_tokens']?.value).toBe(8); | ||
| expect(span.attributes['gen_ai.usage.total_tokens']?.value).toBe(23); | ||
| } | ||
|
|
||
| test('captures Workers AI streaming output when driven via an Agent', async ({ request, baseURL }) => { | ||
| const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { | ||
| return ( | ||
| transactionEvent.transaction === 'GET /agents/my-agent/test' && | ||
| (transactionEvent.spans ?? []).some(span => span.op === 'gen_ai.chat') | ||
| ); | ||
| }); | ||
| const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); | ||
| const transactionPromise = waitForTransaction( | ||
| 'cloudflare-agent', | ||
| transactionEvent => transactionEvent.transaction === 'GET /agents/my-agent/test', | ||
| ); | ||
|
|
||
| const response = await request.get(`${baseURL}/agents/my-agent/test`); | ||
| expect(response.ok()).toBe(true); | ||
|
|
||
| const transaction = await transactionPromise; | ||
| assertGenAiStreamingSpan(transaction.spans); | ||
| const [span, transaction] = await Promise.all([spanPromise, transactionPromise]); | ||
| expect(span.trace_id).toBe(transaction.contexts?.trace?.trace_id); | ||
| assertGenAiStreamingSpan(span); | ||
| }); | ||
|
|
||
| test('captures Workers AI streaming output when driven via an AIChatAgent', async ({ request, baseURL }) => { | ||
| const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { | ||
| return ( | ||
| transactionEvent.transaction === 'GET /agents/my-chat-agent/test' && | ||
| (transactionEvent.spans ?? []).some(span => span.op === 'gen_ai.chat') | ||
| ); | ||
| }); | ||
| const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); | ||
| const transactionPromise = waitForTransaction( | ||
| 'cloudflare-agent', | ||
| transactionEvent => transactionEvent.transaction === 'GET /agents/my-chat-agent/test', | ||
| ); | ||
|
|
||
| const response = await request.get(`${baseURL}/agents/my-chat-agent/test`); | ||
| expect(response.ok()).toBe(true); | ||
|
|
||
| const transaction = await transactionPromise; | ||
| assertGenAiStreamingSpan(transaction.spans); | ||
| const [span, transaction] = await Promise.all([spanPromise, transactionPromise]); | ||
| expect(span.trace_id).toBe(transaction.contexts?.trace?.trace_id); | ||
| assertGenAiStreamingSpan(span); | ||
| }); |
25 changes: 9 additions & 16 deletions
25
dev-packages/e2e-tests/test-applications/cloudflare-agent/tests/chat-conversation.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,32 +1,25 @@ | ||
| import { expect, test } from '@playwright/test'; | ||
| import { waitForTransaction } from '@sentry-internal/test-utils'; | ||
| import { getSpanOp, waitForStreamedSpan, waitForTransaction } from '@sentry-internal/test-utils'; | ||
| import { sendChatMessage } from './agent-socket'; | ||
|
|
||
| // In the Agents model one agent instance is one conversation, so the instance name is the | ||
| // conversation id the SDK correlates the turn's gen_ai spans with. | ||
| const CONVERSATION_ID = 'chat-conv-instance'; | ||
|
|
||
| test('stamps the conversation id on gen_ai spans created inside a chat turn', async ({ baseURL }) => { | ||
| const transactionPromise = waitForTransaction('cloudflare-agent', transactionEvent => { | ||
| return ( | ||
| transactionEvent.transaction === 'webSocketMessage' && | ||
| (transactionEvent.spans ?? []).some(span => span.op === 'gen_ai.chat') | ||
| ); | ||
| }); | ||
| const spanPromise = waitForStreamedSpan('cloudflare-agent', span => getSpanOp(span) === 'gen_ai.chat'); | ||
| const transactionPromise = waitForTransaction( | ||
| 'cloudflare-agent', | ||
| transactionEvent => transactionEvent.transaction === 'webSocketMessage', | ||
| ); | ||
|
|
||
| await sendChatMessage(baseURL!, { | ||
| binding: 'my-chat-agent', | ||
| instance: CONVERSATION_ID, | ||
| prompt: 'What is the capital of France?', | ||
| }); | ||
|
|
||
| const transaction = await transactionPromise; | ||
|
|
||
| const genAiSpan = (transaction.spans ?? []).find(span => span.op === 'gen_ai.chat'); | ||
| expect(genAiSpan).toBeDefined(); | ||
| expect(genAiSpan?.data).toEqual( | ||
| expect.objectContaining({ | ||
| 'gen_ai.conversation.id': CONVERSATION_ID, | ||
| }), | ||
| ); | ||
| const [genAiSpan, transaction] = await Promise.all([spanPromise, transactionPromise]); | ||
| expect(genAiSpan.trace_id).toBe(transaction.contexts?.trace?.trace_id); | ||
| expect(genAiSpan.attributes['gen_ai.conversation.id']?.value).toBe(CONVERSATION_ID); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 18 additions & 8 deletions
26
dev-packages/e2e-tests/test-applications/cloudflare-vercelai-v7-als/tests/index.test.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,33 @@ | ||
| import { expect, test } from '@playwright/test'; | ||
| import { waitForTransaction } from '@sentry-internal/test-utils'; | ||
| import { getSpanOp, waitForRequest } from '@sentry-internal/test-utils'; | ||
|
|
||
| test('does not capture Vercel AI v7 spans without nodejs_compat', async ({ baseURL }) => { | ||
| const transactionPromise = waitForTransaction('cloudflare-vercelai-v7-als', txn => { | ||
| return txn.transaction === 'GET /generate'; | ||
| // The transaction envelope also carries any extracted gen_ai spans as a span v2 container item, | ||
| // so we wait for the whole envelope and assert neither place contains AI spans. | ||
| const envelopePromise = waitForRequest('cloudflare-vercelai-v7-als', ({ envelope }) => { | ||
| const transactionItem = envelope[1].find(([header]) => header.type === 'transaction'); | ||
| return (transactionItem?.[1] as any)?.transaction === 'GET /generate'; | ||
| }); | ||
|
|
||
| const response = await fetch(`${baseURL}/generate`); | ||
| expect(response.status).toBe(200); | ||
|
|
||
| const transaction = await transactionPromise; | ||
| const { envelope } = await envelopePromise; | ||
|
|
||
| const transaction = envelope[1].find(([header]) => header.type === 'transaction')?.[1] as any; | ||
| expect(transaction.transaction).toBe('GET /generate'); | ||
| expect(transaction.contexts?.trace?.op).toBe('http.server'); | ||
|
|
||
| // v7 uses diagnostics_channel which is not available with nodejs_als, | ||
| // so no AI spans should be present. | ||
| const aiSpans = (transaction.spans || []).filter( | ||
| // v7 uses diagnostics_channel which is not available with nodejs_als, so no AI spans should be | ||
| // present — neither embedded in the transaction nor streamed as a span v2 container item. | ||
| const embeddedAiSpans = (transaction.spans || []).filter( | ||
| (span: any) => span.op?.startsWith('gen_ai.') || span.description?.includes('generateText'), | ||
| ); | ||
| expect(aiSpans).toHaveLength(0); | ||
| expect(embeddedAiSpans).toHaveLength(0); | ||
|
|
||
| const streamedGenAiSpans = envelope[1] | ||
| .filter(([header]) => header.type === 'span') | ||
| .flatMap(([, payload]) => (payload as any).items ?? []) | ||
| .filter((span: any) => getSpanOp(span)?.startsWith('gen_ai.')); | ||
| expect(streamedGenAiSpans).toHaveLength(0); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.