diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/orchestrion/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/orchestrion/test.ts index 19bd448f4db0..2070b7c3f98b 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/orchestrion/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/orchestrion/test.ts @@ -37,7 +37,7 @@ describe('OpenAI integration (orchestrion)', () => { const chatSpans = container.items.filter( span => span.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]?.value === 'gen_ai.chat', ); - expect(chatSpans).toHaveLength(2); + expect(chatSpans).toHaveLength(3); const chatSpan = container.items.find( span => span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]?.value === 'chatcmpl-mock123', @@ -92,6 +92,50 @@ describe('OpenAI integration (orchestrion)', () => { type: 'string', value: ORCHESTRION_ORIGIN, }); + + const responsesSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]?.value === 'resp_mock456', + ); + expect(responsesSpan).toBeDefined(); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: ORCHESTRION_ORIGIN, + }); + expect(responsesSpan!.attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 13, + }); }, }) .start() @@ -135,6 +179,23 @@ describe('OpenAI integration (orchestrion)', () => { type: 'string', value: '["Hello from OpenAI mock!"]', }); + + const responsesSpan = container.items.find( + span => span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]?.value === 'resp_mock456', + ); + expect(responsesSpan).toBeDefined(); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1, + }); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); }, }) .start() diff --git a/packages/server-utils/src/integrations/tracing-channel/openai.ts b/packages/server-utils/src/integrations/tracing-channel/openai.ts index e024a380947c..1f5430bf11e7 100644 --- a/packages/server-utils/src/integrations/tracing-channel/openai.ts +++ b/packages/server-utils/src/integrations/tracing-channel/openai.ts @@ -24,7 +24,11 @@ const INTEGRATION_NAME = 'OpenAI' as const; // are attributable separately from the OTel/proxy one. const ORIGIN = 'auto.ai.orchestrion.openai'; -const OPERATION = 'chat'; +// Each instrumented `create` method maps to the gen_ai operation its span reports. +const INSTRUMENTED_CHANNELS = [ + { channel: CHANNELS.OPENAI_CHAT, operation: 'chat' }, + { channel: CHANNELS.OPENAI_RESPONSES, operation: 'chat' }, +] as const; /** * The context object orchestrion shares across the tracing-channel lifecycle hooks: `arguments` is the @@ -48,37 +52,38 @@ const _openaiChannelIntegration = ((options: OpenAiOptions = {}) => { } subscribed = true; - DEBUG_BUILD && debug.log(`[orchestrion:openai] subscribing to channel "${CHANNELS.OPENAI_CHAT}"`); - // `bindTracingChannelToSpan` needs the async-context binding that `initOpenTelemetry()` registers // after `setupOnce` runs, so wait for it before subscribing. waitForTracingChannelBinding(() => { - bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(CHANNELS.OPENAI_CHAT), - data => createChatSpan(data, options), - { - beforeSpanEnd: (span, data) => { - if ('result' in data) { - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); - } + for (const { channel, operation } of INSTRUMENTED_CHANNELS) { + DEBUG_BUILD && debug.log(`[orchestrion:openai] subscribing to channel "${channel}"`); + bindTracingChannelToSpan( + diagnosticsChannel.tracingChannel(channel), + data => createGenAiSpan(data, operation, options), + { + beforeSpanEnd: (span, data) => { + if ('result' in data) { + addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + } + }, + captureError: () => ({ mechanism: { type: ORIGIN, handled: false } }), }, - captureError: () => ({ mechanism: { type: ORIGIN, handled: false } }), - }, - ); + ); + } }); }, }; }) satisfies IntegrationFn; /** - * Build the span for a `chat.completions.create` call. + * Build the span for an instrumented `create` call. * Returning `undefined` opts the payload out so no span is opened. */ -function createChatSpan(data: OpenAiChatChannelContext, options: OpenAiOptions): Span | undefined { +function createGenAiSpan(data: OpenAiChatChannelContext, operation: string, options: OpenAiOptions): Span | undefined { const args = data.arguments ?? []; const params = args[0] as Record | undefined; - // streaming is not supported yet + // streaming is not supported if (params?.stream === true) { return undefined; } @@ -86,27 +91,26 @@ function createChatSpan(data: OpenAiChatChannelContext, options: OpenAiOptions): const { recordInputs } = resolveAIRecordingOptions(options); const enableTruncation = shouldEnableTruncation(options.enableTruncation); - const attributes = extractRequestAttributes(args, OPERATION); + const attributes = extractRequestAttributes(args, operation); attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] = ORIGIN; const model = (params?.model as string) || 'unknown'; const span = startInactiveSpan({ - name: `${OPERATION} ${model}`, - op: `gen_ai.${OPERATION}`, + name: `${operation} ${model}`, + op: `gen_ai.${operation}`, attributes: attributes as Record, }); if (recordInputs && params) { - addRequestAttributes(span, params, OPERATION, enableTruncation); + addRequestAttributes(span, params, operation, enableTruncation); } return span; } /** - * EXPERIMENTAL — orchestrion-driven OpenAI integration. Subscribes to the `orchestrion:openai:chat` - * diagnostics_channel injected into `openai`'s `Completions.prototype.create`, so it requires the - * orchestrion runtime hook or bundler plugin. Covers non-streaming `chat.completions.create`; streaming - * and the other methods are follow-ups. Browser/edge keep using the proxy `instrumentOpenAiClient`. + * EXPERIMENTAL — orchestrion-driven OpenAI integration. Subscribes to the `orchestrion:openai:*` + * diagnostics_channels injected into `openai`'s `create` methods, so it requires the orchestrion runtime + * hook or bundler plugin. Covers non-streaming `chat.completions.create` and `responses.create`. */ export const openaiChannelIntegration = defineIntegration(_openaiChannelIntegration); diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index afba59a4c6bc..785f8aef8c53 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -15,6 +15,7 @@ export const CHANNELS = { MYSQL_QUERY: 'orchestrion:mysql:query', LRU_MEMOIZER_LOAD: 'orchestrion:lru-memoizer:load', OPENAI_CHAT: 'orchestrion:openai:chat', + OPENAI_RESPONSES: 'orchestrion:openai:responses', } as const; export type ChannelName = (typeof CHANNELS)[keyof typeof CHANNELS]; diff --git a/packages/server-utils/src/orchestrion/config.ts b/packages/server-utils/src/orchestrion/config.ts index 96b2d2998df2..f1b41f802c34 100644 --- a/packages/server-utils/src/orchestrion/config.ts +++ b/packages/server-utils/src/orchestrion/config.ts @@ -46,6 +46,12 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, functionQuery: { className: 'Completions', methodName: 'create', kind: 'Auto' as const }, })) satisfies InstrumentationConfig[]), + // OpenAI responses API — same `create(body, options)` shape as chat completions. + ...(['resources/responses/responses.js', 'resources/responses/responses.mjs'].map(filePath => ({ + channelName: 'responses', + module: { name: 'openai', versionRange: '>=4.0.0 <7', filePath }, + functionQuery: { className: 'Responses', methodName: 'create', kind: 'Auto' as const }, + })) satisfies InstrumentationConfig[]), ]; /**