diff --git a/packages/core/src/evaluation/providers/codex-cli.ts b/packages/core/src/evaluation/providers/codex-cli.ts index 283ebcea9..ace312894 100644 --- a/packages/core/src/evaluation/providers/codex-cli.ts +++ b/packages/core/src/evaluation/providers/codex-cli.ts @@ -34,6 +34,7 @@ interface CodexRunOptions { readonly args: readonly string[]; readonly cwd: string; readonly prompt: string; + readonly appServerRequest?: CodexAppServerRunRequest; readonly timeoutMs?: number; readonly env: NodeJS.ProcessEnv; readonly signal?: AbortSignal; @@ -52,6 +53,16 @@ interface CodexRunResult { type CodexRunner = (options: CodexRunOptions) => Promise; +interface CodexAppServerRunRequest { + readonly prompt: string; + readonly model?: string; + readonly modelProvider?: string; + readonly modelReasoningEffort?: string; + readonly sandboxMode?: string; + readonly approvalPolicy?: string; + readonly systemPrompt?: string; +} + export class CodexCliProvider implements Provider { readonly id: string; readonly kind: 'codex-cli' | 'codex-app-server'; @@ -114,6 +125,7 @@ export class CodexCliProvider implements Provider { env, request.signal, logger, + this.buildAppServerRequest(promptContent), ); } catch (error) { const message = formatError(error); @@ -169,7 +181,7 @@ export class CodexCliProvider implements Provider { const detail = pickDetail(result.stderr, result.stdout); const prefix = `Codex ${this.providerLabel()} exited with code ${result.exitCode}`; return this.buildErrorResponse({ - errorKind: result.signal ? 'signal_crash' : 'nonzero_exit', + errorKind: this.classifyNonzeroExit(result), message: detail ? `${prefix}: ${detail}` : prefix, args, cwd, @@ -305,12 +317,14 @@ export class CodexCliProvider implements Provider { env: NodeJS.ProcessEnv, signal: AbortSignal | undefined, logger: CodexStreamLogger | undefined, + appServerRequest?: CodexAppServerRunRequest, ): Promise { return await this.runCodex({ executable: this.resolvedExecutable ?? this.commandExecutable(), args, cwd, prompt: stdinPayload, + appServerRequest, timeoutMs: this.config.timeoutMs, env, signal, @@ -332,6 +346,21 @@ export class CodexCliProvider implements Provider { })}\n`; } + private buildAppServerRequest(promptContent: string): CodexAppServerRunRequest | undefined { + if (this.kind !== 'codex-app-server') { + return undefined; + } + return { + prompt: promptContent, + model: this.config.model, + modelProvider: inferCodexModelProvider(this.config.command ?? []), + modelReasoningEffort: this.config.modelReasoningEffort, + sandboxMode: this.config.sandboxMode, + approvalPolicy: this.config.approvalPolicy, + systemPrompt: this.config.systemPrompt, + }; + } + private async createWorkspace(): Promise { return await mkdtemp(path.join(tmpdir(), WORKSPACE_PREFIX)); } @@ -348,6 +377,13 @@ export class CodexCliProvider implements Provider { return this.kind === 'codex-app-server' ? 'app-server' : 'CLI'; } + private classifyNonzeroExit(result: CodexRunResult): TargetExecutionErrorKind { + if (this.kind === 'codex-app-server' && isCodexAppServerProtocolFailure(result.stderr)) { + return 'malformed_output'; + } + return result.signal ? 'signal_crash' : 'nonzero_exit'; + } + private async buildProcessEnv(workspaceRoot: string): Promise { if (this.config.runtime.mode === 'host') { return process.env; @@ -979,6 +1015,13 @@ function extractFromEvent(event: unknown): string | undefined { return undefined; } const record = event as Record; + const method = typeof record.method === 'string' ? record.method : undefined; + if (method === 'item/completed') { + return extractFromAppServerItem((record.params as Record | undefined)?.item); + } + if (method === 'turn/completed') { + return extractFromAppServerTurn(record.params); + } const type = typeof record.type === 'string' ? record.type : undefined; if (type === JSONL_TYPE_ITEM_COMPLETED) { const item = record.item; @@ -995,6 +1038,39 @@ function extractFromEvent(event: unknown): string | undefined { return undefined; } +function extractFromAppServerTurn(params: unknown): string | undefined { + if (!params || typeof params !== 'object') { + return undefined; + } + const turn = (params as Record).turn; + if (!turn || typeof turn !== 'object') { + return undefined; + } + const items = (turn as Record).items; + if (!Array.isArray(items)) { + return undefined; + } + for (let index = items.length - 1; index >= 0; index -= 1) { + const text = extractFromAppServerItem(items[index]); + if (text) { + return text; + } + } + return undefined; +} + +function extractFromAppServerItem(item: unknown): string | undefined { + if (!item || typeof item !== 'object') { + return undefined; + } + const record = item as Record; + if (record.type !== 'agentMessage') { + return undefined; + } + const text = record.text; + return typeof text === 'string' && text.length > 0 ? text : undefined; +} + function extractFromItem(item: unknown): string | undefined { if (!item || typeof item !== 'object') { return undefined; @@ -1113,7 +1189,20 @@ function formatError(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function isCodexAppServerProtocolFailure(stderr: string): boolean { + return ( + stderr.includes('Codex app-server JSON-RPC error') || + stderr.includes('Codex app-server emitted invalid JSON-RPC line') || + stderr.includes('Codex app-server thread/start response did not include a thread id') || + stderr.includes('Codex app-server exited before turn/completed') + ); +} + async function defaultCodexRunner(options: CodexRunOptions): Promise { + if (options.appServerRequest) { + return await defaultCodexAppServerRunner(options); + } + return await new Promise((resolve, reject) => { const child = spawn(options.executable, options.args, { cwd: options.cwd, @@ -1195,6 +1284,289 @@ async function defaultCodexRunner(options: CodexRunOptions): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(options.executable, options.args, { + cwd: options.cwd, + env: options.env, + stdio: ['pipe', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + shell: shouldShellExecute(options.executable), + }); + trackChild(child); + + let stdout = ''; + let stderr = ''; + let stdoutBuffer = ''; + let timedOut = false; + let cancelled = false; + let protocolError: string | undefined; + let turnCompleted = false; + let requestId = 1; + const initializeId = requestId; + let threadStartId: number | undefined; + let turnStartId: number | undefined; + + const appServerRequest = options.appServerRequest; + if (!appServerRequest) { + reject(new Error('Codex app-server request metadata was not provided')); + return; + } + + const send = (method: string, params: Record): number => { + const id = requestId; + requestId += 1; + child.stdin.write(`${JSON.stringify({ id, method, params })}\n`); + return id; + }; + + const failProtocol = (message: string): void => { + if (!protocolError) { + protocolError = message; + } + child.stdin.end(); + terminateChild(child, 'SIGTERM'); + }; + + const handleMessage = (message: unknown): void => { + if (!message || typeof message !== 'object') { + return; + } + const record = message as Record; + if ('error' in record) { + failProtocol(`Codex app-server JSON-RPC error: ${formatJsonRpcError(record.error)}`); + return; + } + + if (record.id === initializeId && record.result) { + threadStartId = send( + 'thread/start', + buildAppServerThreadStartParams(options, appServerRequest), + ); + return; + } + + if (record.id === threadStartId && record.result && typeof record.result === 'object') { + const threadId = extractAppServerThreadId(record.result); + if (!threadId) { + failProtocol('Codex app-server thread/start response did not include a thread id'); + return; + } + turnStartId = send( + 'turn/start', + buildAppServerTurnStartParams(options, appServerRequest, threadId), + ); + return; + } + + if (record.id === turnStartId && record.result) { + return; + } + + if (record.method === 'turn/completed') { + turnCompleted = true; + const turn = (record.params as Record | undefined)?.turn; + const status = (turn as Record | undefined)?.status; + if (status === 'failed') { + const error = (turn as Record).error; + protocolError = `Codex app-server turn failed: ${formatTurnError(error)}`; + } + child.stdin.end(); + } + }; + + const onAbort = (): void => { + cancelled = true; + terminateChild(child, 'SIGTERM'); + setTimeout(() => terminateChild(child, 'SIGKILL'), 2_000).unref?.(); + }; + + if (options.signal) { + if (options.signal.aborted) { + onAbort(); + } else { + options.signal.addEventListener('abort', onAbort, { once: true }); + } + } + + let timeoutHandle: NodeJS.Timeout | undefined; + if (options.timeoutMs && options.timeoutMs > 0) { + timeoutHandle = setTimeout(() => { + timedOut = true; + terminateChild(child, 'SIGTERM'); + setTimeout(() => terminateChild(child, 'SIGKILL'), 2_000).unref?.(); + }, options.timeoutMs); + timeoutHandle.unref?.(); + } + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + options.onStdoutChunk?.(chunk); + stdoutBuffer += chunk; + const lines = stdoutBuffer.split(/\r?\n/); + stdoutBuffer = lines.pop() ?? ''; + for (const rawLine of lines) { + const line = rawLine.trim(); + if (line.length === 0) { + continue; + } + try { + handleMessage(JSON.parse(line)); + } catch (error) { + failProtocol(`Codex app-server emitted invalid JSON-RPC line: ${formatError(error)}`); + } + } + }); + + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + stderr += chunk; + options.onStderrChunk?.(chunk); + }); + + const cleanup = (): void => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + if (options.signal) { + options.signal.removeEventListener('abort', onAbort); + } + }; + + child.on('error', (error) => { + cleanup(); + reject(error); + }); + + child.on('close', (code, signal) => { + cleanup(); + const resultExitCode = + protocolError && typeof code === 'number' && code === 0 + ? 1 + : typeof code === 'number' + ? code + : null; + const appServerStderr = + protocolError && stderr.trim().length > 0 + ? `${stderr}\n${protocolError}` + : (protocolError ?? stderr); + resolve({ + stdout, + stderr: + !protocolError && !turnCompleted && !timedOut && !cancelled + ? appendLine(appServerStderr, 'Codex app-server exited before turn/completed') + : appServerStderr, + exitCode: + !protocolError && !turnCompleted && !timedOut && !cancelled && resultExitCode === 0 + ? 1 + : resultExitCode, + signal, + timedOut, + cancelled, + }); + }); + + send('initialize', { + clientInfo: { name: 'agentv', version: '0.0.0' }, + capabilities: null, + }); + }); +} + +function buildAppServerThreadStartParams( + options: CodexRunOptions, + request: CodexAppServerRunRequest, +): Record { + return removeUndefined({ + model: request.model, + modelProvider: request.modelProvider, + cwd: options.cwd, + approvalPolicy: request.approvalPolicy, + sandbox: request.sandboxMode, + baseInstructions: request.systemPrompt, + ephemeral: true, + }); +} + +function buildAppServerTurnStartParams( + options: CodexRunOptions, + request: CodexAppServerRunRequest, + threadId: string, +): Record { + return removeUndefined({ + threadId, + input: [{ type: 'text', text: request.prompt, text_elements: [] }], + cwd: options.cwd, + approvalPolicy: request.approvalPolicy, + model: request.model, + effort: request.modelReasoningEffort, + }); +} + +function extractAppServerThreadId(result: unknown): string | undefined { + if (!result || typeof result !== 'object') { + return undefined; + } + const thread = (result as Record).thread; + if (!thread || typeof thread !== 'object') { + return undefined; + } + const id = (thread as Record).id; + return typeof id === 'string' ? id : undefined; +} + +function inferCodexModelProvider(command: readonly string[]): string | undefined { + for (let index = 0; index < command.length - 1; index += 1) { + const flag = command[index]; + if (flag !== '-c' && flag !== '--config') { + continue; + } + const value = command[index + 1]; + const match = /^model_provider=(?:"([^"]+)"|'([^']+)'|(.+))$/.exec(value); + const provider = match?.[1] ?? match?.[2] ?? match?.[3]; + if (provider && provider.trim().length > 0) { + return provider.trim(); + } + } + return undefined; +} + +function removeUndefined(record: Record): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(record)) { + if (value !== undefined) { + result[key] = value; + } + } + return result; +} + +function formatJsonRpcError(error: unknown): string { + if (!error || typeof error !== 'object') { + return String(error); + } + const record = error as Record; + const code = typeof record.code === 'number' ? `${record.code}: ` : ''; + const message = typeof record.message === 'string' ? record.message : JSON.stringify(error); + return `${code}${message}`; +} + +function formatTurnError(error: unknown): string { + if (!error || typeof error !== 'object') { + return String(error); + } + const record = error as Record; + const message = typeof record.message === 'string' ? record.message : JSON.stringify(error); + const details = + typeof record.additionalDetails === 'string' ? ` (${record.additionalDetails})` : ''; + return `${message}${details}`; +} + +function appendLine(base: string, line: string): string { + return base.trim().length > 0 ? `${base}\n${line}` : line; +} + function terminateChild(child: ChildProcessWithoutNullStreams, signal: NodeJS.Signals): void { if (child.pid === undefined) { return; diff --git a/packages/core/test/evaluation/providers/codex.test.ts b/packages/core/test/evaluation/providers/codex.test.ts index 9a7ba904c..e9cf2c44d 100644 --- a/packages/core/test/evaluation/providers/codex.test.ts +++ b/packages/core/test/evaluation/providers/codex.test.ts @@ -350,34 +350,140 @@ describe('CodexCliProvider', () => { expect(response.targetExecution?.errorKind).toBe('sandbox_infra_failure'); }); - it('runs codex-app-server through the configured argv and JSON request payload', async () => { - const runner = mock( - async (opts: { prompt: string; executable: string; args: readonly string[] }) => ({ - stdout: JSON.stringify({ output: [{ text: 'app server answer' }] }), - stderr: '', - exitCode: 0, - }), - ); - const provider = new CodexAppServerProvider( - 'codex-app', - { - command: [process.execPath, 'app-server', '--profile', 'eng'], - runtime: { mode: 'host' }, - }, - runner, + it('runs codex-app-server through initialize, thread/start, and turn/start JSON-RPC messages', async () => { + const serverScript = path.join(fixturesRoot, 'fake-codex-app-server.js'); + const captureFile = path.join(fixturesRoot, 'app-server-requests.json'); + await writeFile( + serverScript, + [ + "const { writeFileSync } = require('node:fs');", + 'const captureFile = process.argv[2];', + 'let buffer = "";', + 'const requests = [];', + 'function send(message) { process.stdout.write(`${JSON.stringify(message)}\\n`); }', + 'function capture(message) { requests.push(message); writeFileSync(captureFile, JSON.stringify(requests)); }', + 'function handle(message) {', + ' capture(message);', + ' if (message.method === "initialize") {', + ' send({ id: message.id, result: { userAgent: "fake", codexHome: process.cwd(), platformFamily: "unix", platformOs: "linux" } });', + ' return;', + ' }', + ' if (message.method === "thread/start") {', + ' send({ id: message.id, result: { thread: { id: "thread-1" } } });', + ' return;', + ' }', + ' if (message.method === "turn/start") {', + ' send({ id: message.id, result: { turn: { id: "turn-1", items: [], itemsView: "notLoaded", status: "inProgress", error: null, startedAt: null, completedAt: null, durationMs: null } } });', + ' send({ method: "item/completed", params: { threadId: "thread-1", turnId: "turn-1", completedAtMs: Date.now(), item: { type: "agentMessage", id: "msg-1", text: "app server answer", phase: "final_answer", memoryCitation: null } } });', + ' send({ method: "turn/completed", params: { threadId: "thread-1", turn: { id: "turn-1", items: [], itemsView: "notLoaded", status: "completed", error: null, startedAt: 1, completedAt: 2, durationMs: 1000 } } });', + ' setTimeout(() => process.exit(0), 0);', + ' }', + '}', + 'process.stdin.setEncoding("utf8");', + 'process.stdin.on("data", (chunk) => {', + ' buffer += chunk;', + ' const lines = buffer.split(/\\r?\\n/);', + ' buffer = lines.pop() ?? "";', + ' for (const line of lines) {', + ' if (line.trim()) handle(JSON.parse(line));', + ' }', + '});', + ].join('\n'), + 'utf8', ); + const provider = new CodexAppServerProvider('codex-app', { + command: [ + process.execPath, + serverScript, + captureFile, + '-c', + 'model_provider="agentv-openai"', + 'app-server', + '--stdio', + ], + model: 'gpt-5.4-mini', + modelReasoningEffort: 'low', + sandboxMode: 'workspace-write', + approvalPolicy: 'never', + runtime: { mode: 'host' }, + systemPrompt: 'system instructions', + }); const response = await provider.invoke({ question: 'hello', evalCaseId: 'case-app' }); - const invocation = runner.mock.calls[0][0]; - expect(invocation.executable).toBe(process.execPath); - expect(invocation.args).toEqual(['app-server', '--profile', 'eng']); - expect(JSON.parse(invocation.prompt)).toMatchObject({ - type: 'agentv.invoke', - question: 'hello', - eval_case_id: 'case-app', + const captured = JSON.parse(await readFile(captureFile, 'utf8')) as Array<{ + method: string; + params: Record; + }>; + expect(captured.map((message) => message.method)).toEqual([ + 'initialize', + 'thread/start', + 'turn/start', + ]); + expect(captured[1].params).toMatchObject({ + model: 'gpt-5.4-mini', + modelProvider: 'agentv-openai', + approvalPolicy: 'never', + sandbox: 'workspace-write', + baseInstructions: 'system instructions', + ephemeral: true, }); + expect(captured[2].params).toMatchObject({ + threadId: 'thread-1', + approvalPolicy: 'never', + model: 'gpt-5.4-mini', + effort: 'low', + }); + expect(captured[2].params.input).toEqual([ + expect.objectContaining({ + type: 'text', + text: expect.stringContaining('[[ ## user_query ## ]]'), + text_elements: [], + }), + ]); expect(response.targetExecution?.providerKind).toBe('codex-app-server'); expect(extractLastAssistantContent(response.output)).toBe('app server answer'); }); + + it('maps codex-app-server JSON-RPC errors to a target execution envelope', async () => { + const serverScript = path.join(fixturesRoot, 'fake-codex-app-server-error.js'); + await writeFile( + serverScript, + [ + 'let buffer = "";', + 'function send(message) { process.stdout.write(`${JSON.stringify(message)}\\n`); }', + 'function handle(message) {', + ' if (message.method === "initialize") {', + ' send({ id: message.id, result: { userAgent: "fake", codexHome: process.cwd(), platformFamily: "unix", platformOs: "linux" } });', + ' return;', + ' }', + ' send({ id: message.id, error: { code: -32602, message: "bad thread config" } });', + '}', + 'process.stdin.setEncoding("utf8");', + 'process.stdin.on("data", (chunk) => {', + ' buffer += chunk;', + ' const lines = buffer.split(/\\r?\\n/);', + ' buffer = lines.pop() ?? "";', + ' for (const line of lines) {', + ' if (line.trim()) handle(JSON.parse(line));', + ' }', + '});', + ].join('\n'), + 'utf8', + ); + const provider = new CodexAppServerProvider('codex-app-error', { + command: [process.execPath, serverScript], + runtime: { mode: 'host' }, + timeoutMs: 1000, + }); + + const response = await provider.invoke({ question: 'hello' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(response.targetExecution?.message).toContain('bad thread config'); + expect(response.targetExecution?.logs?.stderr?.text).toContain( + 'Codex app-server JSON-RPC error', + ); + }); });