From f20c610c6eb296f80a83ede395bb88afe61f047e Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 15:41:20 +0200 Subject: [PATCH 1/7] test(e2e): Add node-openai-decisions-send-to-sentry test app Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 4 + dev-packages/e2e-tests/.env.example | 4 + .../.gitignore | 5 + .../package.json | 43 +++++ .../playwright.config.ts | 28 ++++ .../src/app.ts | 85 ++++++++++ .../src/instrument.ts | 10 ++ .../tests/send-to-sentry.test.ts | 152 ++++++++++++++++++ .../tsconfig.json | 13 ++ 9 files changed, 344 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55dcd184d04c..99ad92d4515d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1197,6 +1197,10 @@ jobs: E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} + # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. + E2E_OPENAI_DECISIONS_API_KEY: + ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || + '' }} # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index 180fb07078a1..3815cae556f0 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -16,6 +16,10 @@ E2E_TEST_SENTRY_PROJECT= # Only needed to run that test app locally. E2E_OPENROUTER_API_KEY= +# An OpenAI API key for the Decisions API (OpenRouter has no Decisions route). +# Only needed to run node-openai-decisions-send-to-sentry locally. +E2E_OPENAI_DECISIONS_API_KEY= + # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry). # The API token needs "Workers Scripts: Edit" on the account; "Workers KV Storage: Read" additionally silences a # warning when a worker is deleted. Leave it empty to use a `wrangler login` session instead. diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore new file mode 100644 index 000000000000..795f6b6d40d1 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore @@ -0,0 +1,5 @@ +node_modules +pnpm-lock.yaml +dist +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json new file mode 100644 index 000000000000..f0924e7f3193 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json @@ -0,0 +1,43 @@ +{ + "name": "node-openai-decisions-send-to-sentry", + "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "start": "node --import ./dist/instrument.js dist/app.js", + "test": "playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", + "test:assert": "pnpm test" + }, + "dependencies": { + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "express": "^4.21.2", + "openai": "7.32.0" + }, + "devDependencies": { + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.27.0", + "@types/express": "^4.17.21", + "@types/node": "^24.0.0", + "sentry": "~0.44.1", + "typescript": "~5.9.0" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-openai-decisions-send-to-sentry (latest)" + } + ] + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts new file mode 100644 index 000000000000..ef4764ce8d73 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts @@ -0,0 +1,28 @@ +import { defineConfig } from '@playwright/test'; + +const port = 3030; + +export default defineConfig({ + testDir: './tests', + /* + * Spans take ~2min to become queryable via the trace endpoint, and each poll has its own 180s + * budget. The first test polls twice in a row (the model span, then its parent), so the ceiling + * has to hold two polls back to back. + */ + timeout: 400_000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + // Every test spends most of its time polling Sentry, so run them all at once. + workers: '100%', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + use: { + baseURL: `http://localhost:${port}`, + }, + webServer: { + command: 'pnpm start', + port, + stdout: 'pipe', + stderr: 'pipe', + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts new file mode 100644 index 000000000000..f37d3198784c --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts @@ -0,0 +1,85 @@ +// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +// OpenRouter has no `/decisions` route, so this app calls OpenAI directly. CI only passes this key to +// this app. +const apiKey = process.env.E2E_OPENAI_DECISIONS_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENAI_DECISIONS_API_KEY is not set'); +} + +const client = new OpenAI({ apiKey }); + +const MODEL = 'gpt-6-luna'; + +// A 1x1 red PNG, so the image request is cheap. +const RED_PIXEL = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; + +const app = express(); + +// Each route makes one `decisions.create` call and answers with the trace id, so the test can find +// the request's spans in Sentry. + +app.get('/decide', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: 'I was charged twice this month and want a refund.', + questions: [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [ + { value: 'billing', description: 'Charges and refunds' }, + { value: 'other', description: 'Anything else' }, + ], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, + ], + }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.get('/decide-image', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', image_url: RED_PIXEL }, + ], + }, + ], + questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], + }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + res.status(500).send({ message: error.message }); +}); + +const port = Number(process.env.PORT ?? 3030); +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts new file mode 100644 index 000000000000..c6c15df960cb --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; + +// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to +// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The OpenAI integration +// is on by default; its runtime channel injection is what picks up the `openai` client in app.ts. +Sentry.init({ + dsn: process.env.E2E_TEST_DSN, + environment: 'qa', // dynamic sampling bias to keep transactions + tracesSampleRate: 1, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts new file mode 100644 index 000000000000..6f1f1155f54d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -0,0 +1,152 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; +import { + EVENT_POLLING_OPTIONS, + fetchSpanAttributes, + fetchTrace, + findSpanInTrace, + flattenTrace, + traceTarget, +} from '@sentry-internal/test-utils/cli'; + +// Mirrors src/app.ts. +const MODEL = 'gpt-6-luna'; + +/** + * What every decide span carries once Sentry has stored it, whatever the model answers. The names are the + * stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` attributes the SDK sends. + * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. + */ +const DECIDE_SPAN_ATTRIBUTES = { + 'span.name': `decide ${MODEL}`, + 'span.op': GEN_AI_DECIDE, + 'span.status': 'ok', + origin: 'auto.ai.openai', + [GEN_AI_OPERATION_NAME]: 'decide', + [GEN_AI_PROVIDER_NAME]: 'openai', + [GEN_AI_REQUEST_MODEL]: MODEL, + [GEN_AI_RESPONSE_MODEL]: MODEL, +}; + +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/decide`); + expect(response.status).toBe(200); + const { traceId }: { traceId: string } = await response.json(); + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); + + // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. + expect(Number(attributes![GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); + expect(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(attributes![GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])); + + // The request span is the segment; it ends last, so it can land after the decide span. + await expect + .poll(() => { + const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); + return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); + }, EVENT_POLLING_OPTIONS) + .toBe(true); + + expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'high'], + }, + }, + }, + ]); + + // The probabilities depend on the model, the shape does not. + expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: { + wants_refund: { type: 'predicate', probability: expect.any(Number) }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, + confidence: expect.any(Number), + }, + urgency: { + type: 'score', + score: expect.any(Number), + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + legend: { 0: 'low', 1: 'high' }, + confidence: expect.any(Number), + }, + }, + }, + ]); +}); + +test('records an image decision by media type only', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/decide-image`); + expect(response.status).toBe(200); + const { traceId }: { traceId: string } = await response.json(); + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); + + expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', mime_type: 'image/png' }, + ], + }, + ], + questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, + }, + ]); + + expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, + ]); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json new file mode 100644 index 000000000000..795574fd9df2 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} From de42015a1e7c6e698350f39dc8d8cf3983d0f1b6 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 17:15:44 +0200 Subject: [PATCH 2/7] Accept either team as the decisions e2e choice answer --- .../tests/send-to-sentry.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 6f1f1155f54d..0641e32f3f2c 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -89,7 +89,7 @@ test('records a text decision with its state, questions and answers', async ({ b }, ]); - // The probabilities depend on the model, the shape does not. + // The answers depend on the model, the shape does not. expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', @@ -97,7 +97,7 @@ test('records a text decision with its state, questions and answers', async ({ b wants_refund: { type: 'predicate', probability: expect.any(Number) }, department: { type: 'choice', - choice: 'billing', + choice: expect.stringMatching(/^(billing|other)$/), probabilities: { billing: expect.any(Number), other: expect.any(Number) }, confidence: expect.any(Number), }, From 401985e22602e46790872f36098d354fd4678e0c Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 17:21:14 +0200 Subject: [PATCH 3/7] Use the conventions search aliases in the decisions e2e test --- .../tests/send-to-sentry.test.ts | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 0641e32f3f2c..6a36093cf18e 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -1,15 +1,18 @@ import { expect, test } from '@playwright/test'; import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_PROVIDER_NAME, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, -} from '@sentry/conventions/attributes'; + SEARCH_GEN_AI__INPUT__MESSAGES, + SEARCH_GEN_AI__OPERATION__NAME, + SEARCH_GEN_AI__OUTPUT__MESSAGES, + SEARCH_GEN_AI__PROVIDER__NAME, + SEARCH_GEN_AI__REQUEST__MODEL, + SEARCH_GEN_AI__RESPONSE__MODEL, + SEARCH_GEN_AI__USAGE__INPUT_TOKENS, + SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, + SEARCH_GEN_AI__USAGE__TOTAL_TOKENS, + SEARCH_ORIGIN, + SEARCH_SPAN__OP, + SEARCH_SPAN__STATUS, +} from '@sentry/conventions/attributes/search'; import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; import { EVENT_POLLING_OPTIONS, @@ -24,19 +27,19 @@ import { const MODEL = 'gpt-6-luna'; /** - * What every decide span carries once Sentry has stored it, whatever the model answers. The names are the - * stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` attributes the SDK sends. + * What every decide span carries once Sentry has stored it, whatever the model answers. The keys are the + * search names, e.g. `span.op` and `origin` rather than the `sentry.*` attributes the SDK sends. * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. */ const DECIDE_SPAN_ATTRIBUTES = { 'span.name': `decide ${MODEL}`, - 'span.op': GEN_AI_DECIDE, - 'span.status': 'ok', - origin: 'auto.ai.openai', - [GEN_AI_OPERATION_NAME]: 'decide', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: MODEL, - [GEN_AI_RESPONSE_MODEL]: MODEL, + [SEARCH_SPAN__OP]: GEN_AI_DECIDE, + [SEARCH_SPAN__STATUS]: 'ok', + [SEARCH_ORIGIN]: 'auto.ai.openai', + [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai', + [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, + [SEARCH_GEN_AI__RESPONSE__MODEL]: MODEL, }; test('records a text decision with its state, questions and answers', async ({ baseURL }) => { @@ -57,9 +60,11 @@ test('records a text decision with its state, questions and answers', async ({ b .toMatchObject(DECIDE_SPAN_ATTRIBUTES); // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. - expect(Number(attributes![GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); - expect(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); - expect(Number(attributes![GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])); + expect(Number(attributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBe(0); + expect(Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(attributes![SEARCH_GEN_AI__USAGE__TOTAL_TOKENS])).toBe( + Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS]), + ); // The request span is the segment; it ends last, so it can land after the decide span. await expect @@ -69,7 +74,7 @@ test('records a text decision with its state, questions and answers', async ({ b }, EVENT_POLLING_OPTIONS) .toBe(true); - expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ { type: 'evaluation', state: 'I was charged twice this month and want a refund.', @@ -90,7 +95,7 @@ test('records a text decision with its state, questions and answers', async ({ b ]); // The answers depend on the model, the shape does not. - expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { @@ -130,7 +135,7 @@ test('records an image decision by media type only', async ({ baseURL }) => { .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ { type: 'evaluation', state: [ @@ -146,7 +151,7 @@ test('records an image decision by media type only', async ({ baseURL }) => { }, ]); - expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, ]); }); From b9f5dfbfc8fa820a8d77512e6b63cf442a742fcc Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 17:22:59 +0200 Subject: [PATCH 4/7] Clarify the decisions e2e timeout comment --- .../playwright.config.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts index ef4764ce8d73..af5ea8f0b951 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts @@ -5,9 +5,8 @@ const port = 3030; export default defineConfig({ testDir: './tests', /* - * Spans take ~2min to become queryable via the trace endpoint, and each poll has its own 180s - * budget. The first test polls twice in a row (the model span, then its parent), so the ceiling - * has to hold two polls back to back. + * Each Sentry poll can take up to 180s. The first poll also waits the ~2min until the span can be + * queried, so the later polls usually pass at once. This leaves room for two full polls. */ timeout: 400_000, fullyParallel: true, From 441ec42a5bce8d620361ac4b07cf07c38c91b56b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 17:24:45 +0200 Subject: [PATCH 5/7] Drop the setup comment from the decisions e2e instrument file --- .../node-openai-decisions-send-to-sentry/src/instrument.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts index c6c15df960cb..e0a0c1807f75 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts @@ -1,8 +1,5 @@ import * as Sentry from '@sentry/node'; -// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to -// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The OpenAI integration -// is on by default; its runtime channel injection is what picks up the `openai` client in app.ts. Sentry.init({ dsn: process.env.E2E_TEST_DSN, environment: 'qa', // dynamic sampling bias to keep transactions From 655821079cdc8f33433548e3e19fa692e048eff5 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 17:42:17 +0200 Subject: [PATCH 6/7] Show the response body when a decisions e2e request fails --- .../tests/send-to-sentry.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 6a36093cf18e..8bae1eec67e1 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -44,8 +44,9 @@ const DECIDE_SPAN_ATTRIBUTES = { test('records a text decision with its state, questions and answers', async ({ baseURL }) => { const response = await fetch(`${baseURL}/decide`); - expect(response.status).toBe(200); - const { traceId }: { traceId: string } = await response.json(); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + const { traceId }: { traceId: string } = body; console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); @@ -120,8 +121,9 @@ test('records a text decision with its state, questions and answers', async ({ b test('records an image decision by media type only', async ({ baseURL }) => { const response = await fetch(`${baseURL}/decide-image`); - expect(response.status).toBe(200); - const { traceId }: { traceId: string } = await response.json(); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + const { traceId }: { traceId: string } = body; console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); From 3cc237cae52a3695654295e2e346cd0730b3b971 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 16:07:30 +0200 Subject: [PATCH 7/7] Expect the image data in the decisions e2e image test --- .../tests/send-to-sentry.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 8bae1eec67e1..5116ba2ebe5b 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -119,7 +119,7 @@ test('records a text decision with its state, questions and answers', async ({ b ]); }); -test('records an image decision by media type only', async ({ baseURL }) => { +test('records an image decision with its image data', async ({ baseURL }) => { const response = await fetch(`${baseURL}/decide-image`); const body = await response.json(); expect(response.status, JSON.stringify(body)).toBe(200); @@ -145,7 +145,7 @@ test('records an image decision by media type only', async ({ baseURL }) => { role: 'user', content: [ { type: 'input_text', text: 'Is this image mostly red?' }, - { type: 'input_image', mime_type: 'image/png' }, + { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) }, ], }, ],