From 3b1d083efc6c3eff7b726b68c18522830b4dce89 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 1 Jul 2026 20:36:26 +0200 Subject: [PATCH] test(anthropic): Port node integration scenarios to the real SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hand-written MockAnthropic classes across the anthropic node-integration-test scenarios (streaming, tools, errors, and truncation) with the real @anthropic-ai/sdk driven by express mock servers, matching the rest of the suite. Removes the redundant manual-client scenario — the auto integration already wraps clients via the same instrumentAnthropicAiClient it would exercise. No SDK/instrumentation behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tracing/anthropic/scenario-errors.mjs | 102 +++++----- .../anthropic/scenario-manual-client.mjs | 115 ----------- .../anthropic/scenario-media-truncation.mjs | 61 +++--- .../anthropic/scenario-message-truncation.mjs | 61 +++--- .../anthropic/scenario-no-truncation.mjs | 44 +++-- .../anthropic/scenario-stream-errors.mjs | 183 +++++++---------- .../anthropic/scenario-stream-tools.mjs | 161 ++++++++------- .../tracing/anthropic/scenario-stream.mjs | 185 ++++++------------ .../tracing/anthropic/scenario-tools.mjs | 56 +++--- .../suites/tracing/anthropic/test.ts | 98 ++-------- 10 files changed, 386 insertions(+), 680 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-manual-client.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-errors.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-errors.mjs index 5501ed1a01ff..f5e86a8ef4b3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-errors.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-errors.mjs @@ -1,74 +1,60 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - this.messages = { - create: this._messagesCreate.bind(this), - }; - this.models = { - retrieve: this._modelsRetrieve.bind(this), - }; - } +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - - // Case 1: Invalid tool format error - if (params.model === 'invalid-format') { - const error = new Error('Invalid format'); - error.status = 400; - error.headers = { 'x-request-id': 'mock-invalid-tool-format-error' }; - throw error; + app.post('/anthropic/v1/messages', (req, res) => { + if (req.body.model === 'invalid-format') { + res + .status(400) + .set('x-request-id', 'mock-invalid-tool-format-error') + .send({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid format' } }); + return; } - // Default case (success) - return tool use for successful tool usage test - return { + res.send({ id: 'msg_ok', type: 'message', - model: params.model, + model: req.body.model, role: 'assistant', - content: [ - { - type: 'tool_use', - id: 'tool_ok_1', - name: 'calculator', - input: { expression: '2+2' }, - }, - ], + content: [{ type: 'tool_use', id: 'tool_ok_1', name: 'calculator', input: { expression: '2+2' } }], stop_reason: 'tool_use', + stop_sequence: null, usage: { input_tokens: 7, output_tokens: 9 }, - }; - } - - async _modelsRetrieve(modelId) { - await new Promise(resolve => setTimeout(resolve, 5)); + }); + }); - // Case for model retrieval error - if (modelId === 'nonexistent-model') { - const error = new Error('Model not found'); - error.status = 404; - error.headers = { 'x-request-id': 'mock-model-retrieval-error' }; - throw error; + app.get('/anthropic/v1/models/:model', (req, res) => { + if (req.params.model === 'nonexistent-model') { + res + .status(404) + .set('x-request-id', 'mock-model-retrieval-error') + .send({ type: 'error', error: { type: 'not_found_error', message: 'Model not found' } }); + return; } + res.send({ id: req.params.model, name: req.params.model, created_at: 1715145600, model: req.params.model }); + }); - return { - id: modelId, - name: modelId, - created_at: 1715145600, - model: modelId, - }; - } + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); - // 1. Test invalid format error - // https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use#handling-tool-use-and-tool-result-content-blocks + // 1. Invalid format error try { await client.messages.create({ model: 'invalid-format', @@ -76,7 +62,7 @@ async function run() { { role: 'user', content: [ - { type: 'text', text: 'Here are the results:' }, // ❌ Text before tool_result + { type: 'text', text: 'Here are the results:' }, { type: 'tool_result', tool_use_id: 'toolu_01' }, ], }, @@ -86,14 +72,14 @@ async function run() { // Error expected } - // 2. Test model retrieval error + // 2. Model retrieval error try { await client.models.retrieve('nonexistent-model'); } catch { // Error expected } - // 3. Test successful tool usage for comparison + // 3. Successful tool usage for comparison await client.messages.create({ model: 'claude-3-haiku-20240307', messages: [{ role: 'user', content: 'Calculate 2+2' }], @@ -110,6 +96,10 @@ async function run() { ], }); }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-manual-client.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-manual-client.mjs deleted file mode 100644 index 590796931315..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-manual-client.mjs +++ /dev/null @@ -1,115 +0,0 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; -import * as Sentry from '@sentry/node'; - -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - - // Create messages object with create and countTokens methods - this.messages = { - create: this._messagesCreate.bind(this), - countTokens: this._messagesCountTokens.bind(this), - }; - - this.models = { - retrieve: this._modelsRetrieve.bind(this), - }; - } - - /** - * Create a mock message - */ - async _messagesCreate(params) { - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 10)); - - if (params.model === 'error-model') { - const error = new Error('Model not found'); - error.status = 404; - error.headers = { 'x-request-id': 'mock-request-123' }; - throw error; - } - - return { - id: 'msg_mock123', - type: 'message', - model: params.model, - role: 'assistant', - content: [ - { - type: 'text', - text: 'Hello from Anthropic mock!', - }, - ], - stop_reason: 'end_turn', - stop_sequence: null, - usage: { - input_tokens: 10, - output_tokens: 15, - }, - }; - } - - async _messagesCountTokens() { - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 10)); - - // For countTokens, just return input_tokens - return { - input_tokens: 15, - }; - } - - async _modelsRetrieve(modelId) { - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 10)); - - // Match what the actual implementation would return - return { - id: modelId, - name: modelId, - created_at: 1715145600, - model: modelId, // Add model field to match the check in addResponseAttributes - }; - } -} - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ - apiKey: 'mock-api-key', - }); - - const client = instrumentAnthropicAiClient(mockClient); - - // First test: basic message completion - await client.messages.create({ - model: 'claude-3-haiku-20240307', - system: 'You are a helpful assistant.', - messages: [{ role: 'user', content: 'What is the capital of France?' }], - temperature: 0.7, - max_tokens: 100, - }); - - // Second test: error handling - try { - await client.messages.create({ - model: 'error-model', - messages: [{ role: 'user', content: 'This will fail' }], - }); - } catch { - // Error is expected and handled - } - - // Third test: count tokens with cached tokens - await client.messages.countTokens({ - model: 'claude-3-haiku-20240307', - messages: [{ role: 'user', content: 'What is the capital of France?' }], - }); - - // Fourth test: models.retrieve - await client.models.retrieve('claude-3-haiku-20240307'); - }); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs index ce5253cc34d7..48f337e2b23c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs @@ -1,53 +1,40 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - this.baseURL = config.baseURL; +function startMockAnthropicServer() { + const app = express(); + app.use(express.json({ limit: '10mb' })); - // Create messages object with create method - this.messages = { - create: this._messagesCreate.bind(this), - }; - } - - /** - * Create a mock message - */ - async _messagesCreate(params) { - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 10)); - - return { + app.post('/anthropic/v1/messages', (req, res) => { + res.send({ id: 'msg-truncation-test', type: 'message', role: 'assistant', - content: [ - { - type: 'text', - text: 'This is the number **3**.', - }, - ], - model: params.model, + content: [{ type: 'text', text: 'This is the number **3**.' }], + model: req.body.model, stop_reason: 'end_turn', stop_sequence: null, - usage: { - input_tokens: 10, - output_tokens: 15, - }, - }; - } + usage: { input_tokens: 10, output_tokens: 15 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ + const client = new Anthropic({ apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, }); - const client = instrumentAnthropicAiClient(mockClient, { enableTruncation: true, recordInputs: true }); - // Send the image showing the number 3 // Put the image in the last message so it doesn't get dropped await client.messages.create({ @@ -75,6 +62,10 @@ async function run() { temperature: 0.7, }); }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs index 15711d019e2a..27aa8494f693 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-message-truncation.mjs @@ -1,53 +1,40 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - this.baseURL = config.baseURL; +function startMockAnthropicServer() { + const app = express(); + app.use(express.json({ limit: '10mb' })); - // Create messages object with create method - this.messages = { - create: this._messagesCreate.bind(this), - }; - } - - /** - * Create a mock message - */ - async _messagesCreate(params) { - // Simulate processing time - await new Promise(resolve => setTimeout(resolve, 10)); - - return { + app.post('/anthropic/v1/messages', (req, res) => { + res.send({ id: 'msg-truncation-test', type: 'message', role: 'assistant', - content: [ - { - type: 'text', - text: 'Response to truncated messages', - }, - ], - model: params.model, + content: [{ type: 'text', text: 'Response to truncated messages' }], + model: req.body.model, stop_reason: 'end_turn', stop_sequence: null, - usage: { - input_tokens: 10, - output_tokens: 15, - }, - }; - } + usage: { input_tokens: 10, output_tokens: 15 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ + const client = new Anthropic({ apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, }); - const client = instrumentAnthropicAiClient(mockClient, { enableTruncation: true, recordInputs: true }); - // Test 1: Given an array of messages only the last message should be kept // The last message should be truncated to fit within the 20KB limit const largeContent1 = 'A'.repeat(15000); // ~15KB @@ -79,6 +66,10 @@ async function run() { temperature: 0.7, }); }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-no-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-no-truncation.mjs index 36f2ffe8c35c..f66cee978733 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-no-truncation.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-no-truncation.mjs @@ -1,33 +1,39 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - this.messages = { - create: this._messagesCreate.bind(this), - }; - } - - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 10)); - return { +function startMockAnthropicServer() { + const app = express(); + app.use(express.json({ limit: '10mb' })); + + app.post('/anthropic/v1/messages', (req, res) => { + res.send({ id: 'msg-no-truncation-test', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'Response' }], - model: params.model, + model: req.body.model, stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 10, output_tokens: 5 }, - }; - } + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient, { enableTruncation: false, recordInputs: true }); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); // Multiple messages with long content (would normally be truncated and popped to last message only) const longContent = 'A'.repeat(50_000); @@ -49,6 +55,10 @@ async function run() { input: longStringInput, }); }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-errors.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-errors.mjs index 9112f96363ce..8ea49997643e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-errors.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-errors.mjs @@ -1,133 +1,100 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; - -// Generator for default fallback -function createMockDefaultFallbackStream() { - async function* generator() { - yield { - type: 'content_block_start', - index: 0, - }; - yield { - type: 'content_block_delta', - index: 0, - delta: { text: 'This stream will work fine.' }, - }; - yield { - type: 'content_block_stop', - index: 0, - }; - } - return generator(); -} - -// Generator that errors midway through streaming -function createMockMidwayErrorStream() { - async function* generator() { - // First yield some initial data to start the stream - yield { - type: 'content_block_start', - message: { - id: 'msg_error_stream_1', - type: 'message', - role: 'assistant', - model: 'claude-3-haiku-20240307', - content: [], - usage: { input_tokens: 5 }, - }, - }; - - // Yield one chunk of content - yield { type: 'content_block_delta', delta: { text: 'This stream will ' } }; - - // Then throw an error - await new Promise(resolve => setTimeout(resolve, 5)); - throw new Error('Stream interrupted'); - } - - return generator(); -} - -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - - this.messages = { - create: this._messagesCreate.bind(this), - stream: this._messagesStream.bind(this), - }; - } - - // client.messages.create with stream: true - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - - // Error on initialization for 'error-stream-init' model - if (params.model === 'error-stream-init') { - if (params?.stream === true) { - throw new Error('Failed to initialize stream'); - } +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/anthropic/v1/messages', (req, res) => { + const model = req.body.model; + + // Fail before any streaming begins. + if (model === 'error-stream-init') { + res + .status(400) + .set('x-request-id', 'mock-stream-init-error') + .send({ type: 'error', error: { type: 'invalid_request_error', message: 'Failed to initialize stream' } }); + return; } - // Error midway for 'error-stream-midway' model - if (params.model === 'error-stream-midway') { - if (params?.stream === true) { - return createMockMidwayErrorStream(); - } - } - - // Default fallback - return { - id: 'msg_mock123', - type: 'message', - model: params.model, - role: 'assistant', - content: [{ type: 'text', text: 'Non-stream response' }], - usage: { input_tokens: 5, output_tokens: 7 }, - }; - } - - // client.messages.stream - async _messagesStream(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - - // Error on initialization for 'error-stream-init' model - if (params.model === 'error-stream-init') { - throw new Error('Failed to initialize stream'); + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + + // Start a valid stream, then drop the connection mid-message (no message_stop). + if (model === 'error-stream-midway') { + const events = [ + { + type: 'message_start', + message: { + id: 'msg_error_stream_1', + type: 'message', + role: 'assistant', + model, + content: [], + usage: { input_tokens: 5 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'This stream will ' } }, + ]; + events.forEach((event, index) => { + setTimeout(() => { + res.write(`event: ${event.type}\n`); + res.write(`data: ${JSON.stringify(event)}\n\n`); + }, index * 10); + }); + // Drop the connection only after the last event has been received and parsed, so the streamed + // text is captured before the stream errors out. + setTimeout(() => res.destroy(), events.length * 10 + 50); + return; } - // Error midway for 'error-stream-midway' model - if (params.model === 'error-stream-midway') { - return createMockMidwayErrorStream(); - } + res.end(); + }); - // Default fallback - return createMockDefaultFallbackStream(); - } + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); // 1) Error on stream initialization with messages.create try { - await client.messages.create({ + const stream = await client.messages.create({ model: 'error-stream-init', messages: [{ role: 'user', content: 'This will fail immediately' }], stream: true, }); + for await (const _ of stream) { + void _; + } } catch { // Error expected } // 2) Error on stream initialization with messages.stream try { - await client.messages.stream({ + const stream = client.messages.stream({ model: 'error-stream-init', messages: [{ role: 'user', content: 'This will also fail immediately' }], }); + for await (const _ of stream) { + void _; + } } catch { // Error expected } @@ -139,7 +106,6 @@ async function run() { messages: [{ role: 'user', content: 'This will fail midway' }], stream: true, }); - for await (const _ of stream) { void _; } @@ -149,11 +115,10 @@ async function run() { // 4) Error midway through streaming with messages.stream try { - const stream = await client.messages.stream({ + const stream = client.messages.stream({ model: 'error-stream-midway', messages: [{ role: 'user', content: 'This will also fail midway' }], }); - for await (const _ of stream) { void _; } @@ -161,6 +126,10 @@ async function run() { // Error expected } }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-tools.mjs index 8d423fd0bbe0..8c6d114eae49 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-tools.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-tools.mjs @@ -1,112 +1,111 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -function createMockStreamEvents(model = 'claude-3-haiku-20240307') { - async function* generator() { - // initial message metadata with id/model and input tokens - yield { - type: 'content_block_start', - message: { - id: 'msg_stream_tool_1', - type: 'message', - role: 'assistant', - model, - content: [], - stop_reason: 'end_turn', - usage: { input_tokens: 11 }, - }, - }; - - // streamed text - yield { type: 'content_block_delta', delta: { text: 'Starting tool...' } }; - - // tool_use streamed via partial json - yield { - type: 'content_block_start', - index: 0, - content_block: { type: 'tool_use', id: 'tool_weather_2', name: 'weather' }, - }; - yield { type: 'content_block_delta', index: 0, delta: { partial_json: '{"city":' } }; - yield { type: 'content_block_delta', index: 0, delta: { partial_json: '"Paris"}' } }; - yield { type: 'content_block_stop', index: 0 }; +const TOOLS = [ + { + name: 'weather', + description: 'Get weather', + input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] }, + }, +]; - // more text - yield { type: 'content_block_delta', delta: { text: 'Done.' } }; +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); - // final usage - yield { type: 'message_delta', usage: { output_tokens: 9 } }; - } - return generator(); -} + app.post('/anthropic/v1/messages', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; - this.messages = { - create: this._messagesCreate.bind(this), - stream: this._messagesStream.bind(this), - }; - } + const model = req.body.model; + const events = [ + { + type: 'message_start', + message: { + id: 'msg_stream_tool_1', + type: 'message', + role: 'assistant', + model, + content: [], + stop_reason: null, + usage: { input_tokens: 11 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Starting tool...' } }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { type: 'tool_use', id: 'tool_weather_2', name: 'weather', input: {} }, + }, + { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '{"city":' } }, + { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '"Paris"}' } }, + { type: 'content_block_stop', index: 1 }, + { + type: 'message_delta', + delta: { stop_reason: 'tool_use', stop_sequence: null }, + usage: { output_tokens: 9 }, + }, + { type: 'message_stop' }, + ]; - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - if (params?.stream) { - return createMockStreamEvents(params.model); - } - return { - id: 'msg_mock_no_stream', - type: 'message', - model: params.model, - role: 'assistant', - content: [{ type: 'text', text: 'No stream' }], - usage: { input_tokens: 2, output_tokens: 3 }, - }; - } + events.forEach((event, index) => { + setTimeout(() => { + res.write(`event: ${event.type}\n`); + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (index === events.length - 1) { + res.end(); + } + }, index * 10); + }); + }); - async _messagesStream(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - return createMockStreamEvents(params?.model); - } + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); - // stream via create(stream:true) + // 1) Streaming tool call via stream: true param on messages.create const stream1 = await client.messages.create({ model: 'claude-3-haiku-20240307', messages: [{ role: 'user', content: 'Need the weather' }], - tools: [ - { - name: 'weather', - description: 'Get weather', - input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] }, - }, - ], + tools: TOOLS, stream: true, }); for await (const _ of stream1) { void _; } - // stream via messages.stream - const stream2 = await client.messages.stream({ + // 2) Streaming tool call via messages.stream API + const stream2 = client.messages.stream({ model: 'claude-3-haiku-20240307', messages: [{ role: 'user', content: 'Need the weather' }], - tools: [ - { - name: 'weather', - description: 'Get weather', - input_schema: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] }, - }, - ], + tools: TOOLS, }); for await (const _ of stream2) { void _; } }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream.mjs index 4e0fa74fdd0d..2fe300dac5b7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream.mjs @@ -1,139 +1,70 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -function createMockStreamEvents(model = 'claude-3-haiku-20240307') { - async function* generator() { - // Provide message metadata early so the span can capture id/model/usage input tokens - yield { - type: 'content_block_start', - message: { - id: 'msg_stream_1', - type: 'message', - role: 'assistant', - model, - content: [], - stop_reason: 'end_turn', - stop_sequence: null, - usage: { - input_tokens: 10, - }, - }, - }; - - // Streamed text chunks - yield { type: 'content_block_delta', delta: { text: 'Hello ' } }; - yield { type: 'content_block_delta', delta: { text: 'from ' } }; - yield { type: 'content_block_delta', delta: { text: 'stream!' } }; - - // Final usage totals for output tokens - yield { type: 'message_delta', usage: { output_tokens: 15 } }; - } - - return generator(); -} - -// Mimics Anthropic SDK's MessageStream class -class MockMessageStream { - constructor(model) { - this._model = model; - this._eventHandlers = {}; - } - - on(event, handler) { - if (!this._eventHandlers[event]) { - this._eventHandlers[event] = []; - } - this._eventHandlers[event].push(handler); - - // Start processing events asynchronously (don't await) - if (event === 'streamEvent' && !this._processing) { - this._processing = true; - this._processEvents(); - } - - return this; - } - - async _processEvents() { - try { - const generator = createMockStreamEvents(this._model); - for await (const event of generator) { - if (this._eventHandlers['streamEvent']) { - for (const handler of this._eventHandlers['streamEvent']) { - handler(event); - } - } - } - - // Emit 'message' event when done - if (this._eventHandlers['message']) { - for (const handler of this._eventHandlers['message']) { - handler(); - } - } - } catch (error) { - if (this._eventHandlers['error']) { - for (const handler of this._eventHandlers['error']) { - handler(error); - } - } - } - } - - async *[Symbol.asyncIterator]() { - const generator = createMockStreamEvents(this._model); - for await (const event of generator) { - yield event; - } - } -} - -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); - this.messages = { - create: this._messagesCreate.bind(this), - stream: this._messagesStream.bind(this), - }; - } + app.post('/anthropic/v1/messages', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - if (params?.stream === true) { - return createMockStreamEvents(params.model); - } - // Fallback non-streaming behavior (not used in this scenario) - return { - id: 'msg_mock123', - type: 'message', - model: params.model, - role: 'assistant', - content: [ - { - type: 'text', - text: 'Hello from Anthropic mock!', + const model = req.body.model; + const events = [ + { + type: 'message_start', + message: { + id: 'msg_stream_1', + type: 'message', + role: 'assistant', + model, + content: [], + usage: { input_tokens: 10 }, }, - ], - stop_reason: 'end_turn', - stop_sequence: null, - usage: { - input_tokens: 10, - output_tokens: 15, }, - }; - } + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Hello ' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'from ' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'stream!' } }, + { type: 'content_block_stop', index: 0 }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 15 }, + }, + { type: 'message_stop' }, + ]; + + events.forEach((event, index) => { + setTimeout(() => { + res.write(`event: ${event.type}\n`); + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (index === events.length - 1) { + res.end(); + } + }, index * 10); + }); + }); - // This should return synchronously (like the real Anthropic SDK) - _messagesStream(params) { - return new MockMessageStream(params?.model); - } + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); // 1) Streaming via stream: true param on messages.create const stream1 = await client.messages.create({ @@ -168,6 +99,10 @@ async function run() { void _; } }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-tools.mjs index 1637a77c9dd8..30c7ceb34b4d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-tools.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-tools.mjs @@ -1,47 +1,43 @@ -import { instrumentAnthropicAiClient } from '@sentry/core'; +import Anthropic from '@anthropic-ai/sdk'; import * as Sentry from '@sentry/node'; +import express from 'express'; -class MockAnthropic { - constructor(config) { - this.apiKey = config.apiKey; +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); - this.messages = { - create: this._messagesCreate.bind(this), - }; - } - - async _messagesCreate(params) { - await new Promise(resolve => setTimeout(resolve, 5)); - - return { + app.post('/anthropic/v1/messages', (req, res) => { + res.send({ id: 'msg_mock_tool_1', type: 'message', - model: params.model, + model: req.body.model, role: 'assistant', content: [ { type: 'text', text: 'Let me check the weather.' }, - { - type: 'tool_use', - id: 'tool_weather_1', - name: 'weather', - input: { city: 'Paris' }, - }, + { type: 'tool_use', id: 'tool_weather_1', name: 'weather', input: { city: 'Paris' } }, { type: 'text', text: 'It is sunny.' }, ], stop_reason: 'end_turn', stop_sequence: null, - usage: { - input_tokens: 5, - output_tokens: 7, - }, - }; - } + usage: { input_tokens: 5, output_tokens: 7 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); } async function run() { + const server = await startMockAnthropicServer(); + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' }); - const client = instrumentAnthropicAiClient(mockClient); + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); await client.messages.create({ model: 'claude-3-haiku-20240307', @@ -59,6 +55,10 @@ async function run() { ], }); }); + + await Sentry.flush(2000); + + server.close(); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 96760bb10e9f..cbb90ce72bca 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -8,7 +8,6 @@ import { GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_REQUEST_STREAM_ATTRIBUTE, GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, GEN_AI_RESPONSE_ID_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_STREAMING_ATTRIBUTE, @@ -54,68 +53,6 @@ describe('Anthropic integration', () => { message: 'stream event from user-added event listener captured', }; - createEsmAndCjsTests(__dirname, 'scenario-manual-client.mjs', 'instrument.mjs', (createRunner, test) => { - test('creates anthropic related spans when manually insturmenting client', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) - .expect({ - span: container => { - expect(container.items).toHaveLength(4); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - expect(completionSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE].value).toBe(0.7); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE].value).toBe(100); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(15); - expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(25); - - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - expect(errorSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('error-model'); - - const tokenCountingSpan = container.items.find( - span => - span.name === 'chat claude-3-haiku-20240307' && - span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE] === undefined, - ); - expect(tokenCountingSpan).toBeDefined(); - expect(tokenCountingSpan!.status).toBe('ok'); - expect(tokenCountingSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(tokenCountingSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - - const modelsSpan = container.items.find(span => span.name === 'models claude-3-haiku-20240307'); - expect(modelsSpan).toBeDefined(); - expect(modelsSpan!.status).toBe('ok'); - expect(modelsSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('models'); - expect(modelsSpan!.attributes['sentry.op'].value).toBe('gen_ai.models'); - expect(modelsSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - expect(modelsSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); - expect(modelsSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - expect(modelsSpan!.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - expect(modelsSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('claude-3-haiku-20240307'); - }, - }) - .start() - .completed(); - }); - }); - createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { await createRunner() @@ -364,9 +301,7 @@ describe('Anthropic integration', () => { expect(span.attributes[GEN_AI_RESPONSE_ID_ATTRIBUTE].value).toBe('msg_stream_1'); } - const detailedStreamSpan = requestStreamSpans.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["end_turn"]', - ); + const detailedStreamSpan = requestStreamSpans[0]; expect(detailedStreamSpan).toBeDefined(); expect(detailedStreamSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('anthropic'); expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE].value).toBe('chat'); @@ -486,23 +421,24 @@ describe('Anthropic integration', () => { .expect({ span: container => { expect(container.items).toHaveLength(2); - const streamingToolSpan = container.items.find(span => span.status === 'ok'); - expect(streamingToolSpan).toBeDefined(); - expect(streamingToolSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingToolSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(streamingToolSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(streamingToolSpan!.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); - expect(streamingToolSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe( - EXPECTED_TOOLS_JSON, + for (const span of container.items) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE].value).toBe(true); + expect(span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toBe(EXPECTED_TOOLS_JSON); + expect(span.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toBe(EXPECTED_TOOL_CALLS_JSON); + } + + // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. + const createStreamSpan = container.items.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(streamingToolSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE].value).toBe( - EXPECTED_TOOL_CALLS_JSON, + expect(createStreamSpan).toBeDefined(); + const messagesStreamSpan = container.items.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - - const errorSpan = container.items.find(span => span.status === 'error'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(messagesStreamSpan).toBeDefined(); }, }) .start()