diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55dcd184d04c..99ad92d4515d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1197,6 +1197,10 @@ jobs: E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} + # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. + E2E_OPENAI_DECISIONS_API_KEY: + ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || + '' }} # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index 180fb07078a1..3815cae556f0 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -16,6 +16,10 @@ E2E_TEST_SENTRY_PROJECT= # Only needed to run that test app locally. E2E_OPENROUTER_API_KEY= +# An OpenAI API key for the Decisions API (OpenRouter has no Decisions route). +# Only needed to run node-openai-decisions-send-to-sentry locally. +E2E_OPENAI_DECISIONS_API_KEY= + # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry). # The API token needs "Workers Scripts: Edit" on the account; "Workers KV Storage: Read" additionally silences a # warning when a worker is deleted. Leave it empty to use a `wrangler login` session instead. diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore new file mode 100644 index 000000000000..795f6b6d40d1 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore @@ -0,0 +1,5 @@ +node_modules +pnpm-lock.yaml +dist +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json new file mode 100644 index 000000000000..f0924e7f3193 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json @@ -0,0 +1,43 @@ +{ + "name": "node-openai-decisions-send-to-sentry", + "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "start": "node --import ./dist/instrument.js dist/app.js", + "test": "playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", + "test:assert": "pnpm test" + }, + "dependencies": { + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "express": "^4.21.2", + "openai": "7.32.0" + }, + "devDependencies": { + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.27.0", + "@types/express": "^4.17.21", + "@types/node": "^24.0.0", + "sentry": "~0.44.1", + "typescript": "~5.9.0" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-openai-decisions-send-to-sentry (latest)" + } + ] + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts new file mode 100644 index 000000000000..af5ea8f0b951 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts @@ -0,0 +1,27 @@ +import { defineConfig } from '@playwright/test'; + +const port = 3030; + +export default defineConfig({ + testDir: './tests', + /* + * Each Sentry poll can take up to 180s. The first poll also waits the ~2min until the span can be + * queried, so the later polls usually pass at once. This leaves room for two full polls. + */ + timeout: 400_000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + // Every test spends most of its time polling Sentry, so run them all at once. + workers: '100%', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + use: { + baseURL: `http://localhost:${port}`, + }, + webServer: { + command: 'pnpm start', + port, + stdout: 'pipe', + stderr: 'pipe', + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts new file mode 100644 index 000000000000..f37d3198784c --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts @@ -0,0 +1,85 @@ +// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +// OpenRouter has no `/decisions` route, so this app calls OpenAI directly. CI only passes this key to +// this app. +const apiKey = process.env.E2E_OPENAI_DECISIONS_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENAI_DECISIONS_API_KEY is not set'); +} + +const client = new OpenAI({ apiKey }); + +const MODEL = 'gpt-6-luna'; + +// A 1x1 red PNG, so the image request is cheap. +const RED_PIXEL = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; + +const app = express(); + +// Each route makes one `decisions.create` call and answers with the trace id, so the test can find +// the request's spans in Sentry. + +app.get('/decide', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: 'I was charged twice this month and want a refund.', + questions: [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [ + { value: 'billing', description: 'Charges and refunds' }, + { value: 'other', description: 'Anything else' }, + ], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, + ], + }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.get('/decide-image', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', image_url: RED_PIXEL }, + ], + }, + ], + questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], + }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + res.status(500).send({ message: error.message }); +}); + +const port = Number(process.env.PORT ?? 3030); +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts new file mode 100644 index 000000000000..e0a0c1807f75 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts @@ -0,0 +1,7 @@ +import * as Sentry from '@sentry/node'; + +Sentry.init({ + dsn: process.env.E2E_TEST_DSN, + environment: 'qa', // dynamic sampling bias to keep transactions + tracesSampleRate: 1, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts new file mode 100644 index 000000000000..5116ba2ebe5b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -0,0 +1,159 @@ +import { expect, test } from '@playwright/test'; +import { + SEARCH_GEN_AI__INPUT__MESSAGES, + SEARCH_GEN_AI__OPERATION__NAME, + SEARCH_GEN_AI__OUTPUT__MESSAGES, + SEARCH_GEN_AI__PROVIDER__NAME, + SEARCH_GEN_AI__REQUEST__MODEL, + SEARCH_GEN_AI__RESPONSE__MODEL, + SEARCH_GEN_AI__USAGE__INPUT_TOKENS, + SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, + SEARCH_GEN_AI__USAGE__TOTAL_TOKENS, + SEARCH_ORIGIN, + SEARCH_SPAN__OP, + SEARCH_SPAN__STATUS, +} from '@sentry/conventions/attributes/search'; +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; +import { + EVENT_POLLING_OPTIONS, + fetchSpanAttributes, + fetchTrace, + findSpanInTrace, + flattenTrace, + traceTarget, +} from '@sentry-internal/test-utils/cli'; + +// Mirrors src/app.ts. +const MODEL = 'gpt-6-luna'; + +/** + * What every decide span carries once Sentry has stored it, whatever the model answers. The keys are the + * search names, e.g. `span.op` and `origin` rather than the `sentry.*` attributes the SDK sends. + * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. + */ +const DECIDE_SPAN_ATTRIBUTES = { + 'span.name': `decide ${MODEL}`, + [SEARCH_SPAN__OP]: GEN_AI_DECIDE, + [SEARCH_SPAN__STATUS]: 'ok', + [SEARCH_ORIGIN]: 'auto.ai.openai', + [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai', + [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, + [SEARCH_GEN_AI__RESPONSE__MODEL]: MODEL, +}; + +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/decide`); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + const { traceId }: { traceId: string } = body; + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); + + // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. + expect(Number(attributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBe(0); + expect(Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(attributes![SEARCH_GEN_AI__USAGE__TOTAL_TOKENS])).toBe( + Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS]), + ); + + // The request span is the segment; it ends last, so it can land after the decide span. + await expect + .poll(() => { + const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); + return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); + }, EVENT_POLLING_OPTIONS) + .toBe(true); + + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, + }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'high'], + }, + }, + }, + ]); + + // The answers depend on the model, the shape does not. + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: { + wants_refund: { type: 'predicate', probability: expect.any(Number) }, + department: { + type: 'choice', + choice: expect.stringMatching(/^(billing|other)$/), + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, + confidence: expect.any(Number), + }, + urgency: { + type: 'score', + score: expect.any(Number), + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + legend: { 0: 'low', 1: 'high' }, + confidence: expect.any(Number), + }, + }, + }, + ]); +}); + +test('records an image decision with its image data', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/decide-image`); + const body = await response.json(); + expect(response.status, JSON.stringify(body)).toBe(200); + const { traceId }: { traceId: string } = body; + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); + + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) }, + ], + }, + ], + questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, + }, + ]); + + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ + { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, + ]); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json new file mode 100644 index 000000000000..795574fd9df2 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +}