diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index abccda5fe..4ee595af1 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -975,6 +975,16 @@ async function readArtifactCatalogEntryText( return { content }; } +function readableLocalCatalogEntry( + meta: SourcedResultFileMeta, + entry: ArtifactCatalogEntry | undefined, +): ArtifactCatalogEntry | undefined { + if (!entry || entry.storage !== 'local') return entry; + const baseDir = runWorkspaceDirFromManifestPath(meta.path); + const resolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath); + return resolved.absolutePath ? entry : undefined; +} + function artifactFileContentResponse(c: C, filePath: string, fileContent: string) { if (c.req.query('raw') === '1' || c.req.query('download') === '1') { c.header('Content-Type', inferRawContentType(filePath)); @@ -2143,6 +2153,12 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) runPath, ); const answerEntry = findPointerArtifactCatalogEntry(catalog, answer, 'answer', runPath); + const transcriptRawEntry = readableLocalCatalogEntry( + meta, + record.transcript_raw_path + ? findArtifactCatalogEntry(catalog, record.transcript_raw_path) + : undefined, + ); if (!transcriptEntry && !transcript.path) { return c.json({ @@ -2194,6 +2210,7 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext) return c.json({ status: 'ok', transcript_path: transcriptEntry.displayPath, + ...(transcriptRawEntry && { transcript_raw_path: transcriptRawEntry.displayPath }), content, language: inferLanguage(transcriptEntry.displayPath), ...(answerEntry && { answer_path: answerEntry.displayPath }), diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index b4e1139f2..66d7e32c8 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -3115,8 +3115,10 @@ describe('serve app', () => { const timestampDir = path.join(runsDir, runId); const resultDir = 'demo-test-greeting--111111111111'; const transcriptArtifactPath = `${resultDir}/sample-1/transcript.json`; + const transcriptRawArtifactPath = `${resultDir}/sample-1/transcript-raw.jsonl`; const answerArtifactPath = `${resultDir}/sample-1/outputs/answer.md`; const transcriptPath = path.join(timestampDir, transcriptArtifactPath); + const transcriptRawPath = path.join(timestampDir, transcriptRawArtifactPath); const answerPath = path.join(timestampDir, answerArtifactPath); const transcriptJson = `${JSON.stringify( { @@ -3142,9 +3144,14 @@ describe('serve app', () => { null, 2, )}\n`; + const transcriptRawJsonl = `${JSON.stringify({ + type: 'raw_event', + message: 'provider event', + })}\n`; mkdirSync(path.dirname(transcriptPath), { recursive: true }); writeFileSync(transcriptPath, transcriptJson); + writeFileSync(transcriptRawPath, transcriptRawJsonl); mkdirSync(path.dirname(answerPath), { recursive: true }); writeFileSync(answerPath, 'done'); mkdirSync(path.join(timestampDir, '.internal'), { recursive: true }); @@ -3155,6 +3162,7 @@ describe('serve app', () => { experiment: 'canonical-transcript', result_dir: resultDir, transcript_path: transcriptArtifactPath, + transcript_raw_path: transcriptRawArtifactPath, answer_path: answerArtifactPath, }), ); @@ -3168,6 +3176,7 @@ describe('serve app', () => { const transcriptData = (await transcriptRes.json()) as { status: string; transcript_path: string; + transcript_raw_path: string; content: string; answer_path: string; answer_content: string; @@ -3175,6 +3184,7 @@ describe('serve app', () => { expect(transcriptData).toMatchObject({ status: 'ok', transcript_path: transcriptArtifactPath, + transcript_raw_path: transcriptRawArtifactPath, content: transcriptJson, answer_path: answerArtifactPath, answer_content: 'done', @@ -3186,6 +3196,68 @@ describe('serve app', () => { expect(rawRes.status).toBe(200); expect(await rawRes.text()).toBe(transcriptJson); + + const rawSidecarRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${transcriptRawArtifactPath}?result_dir=${encodeURIComponent(resultDir)}&raw=1`, + ); + + expect(rawSidecarRes.status).toBe(200); + expect(await rawSidecarRes.text()).toBe(transcriptRawJsonl); + }); + + it('omits missing raw transcript sidecars from the transcript response', async () => { + const runsDir = localResultsExperimentDir(tempDir, 'canonical-transcript-normalized-only'); + const runId = '2026-03-25T10-45-00-000Z'; + const timestampDir = path.join(runsDir, runId); + const resultDir = 'demo-test-greeting--222222222222'; + const transcriptArtifactPath = `${resultDir}/sample-1/transcript.json`; + const transcriptRawArtifactPath = `${resultDir}/sample-1/transcript-raw.jsonl`; + const transcriptPath = path.join(timestampDir, transcriptArtifactPath); + const transcriptJson = `${JSON.stringify({ + schema_version: 'agentv.normalized_transcript.v1', + target: 'codex', + turns: [ + { + v: 1, + agent: 'codex', + type: 'assistant', + content: [{ type: 'text', text: 'done' }], + }, + ], + })}\n`; + + mkdirSync(path.dirname(transcriptPath), { recursive: true }); + writeFileSync(transcriptPath, transcriptJson); + mkdirSync(path.join(timestampDir, '.internal'), { recursive: true }); + writeFileSync( + path.join(timestampDir, '.internal', 'index.jsonl'), + toJsonl({ + ...RESULT_A, + experiment: 'canonical-transcript-normalized-only', + result_dir: resultDir, + transcript_path: transcriptArtifactPath, + transcript_raw_path: transcriptRawArtifactPath, + }), + ); + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const transcriptRes = await app.request( + `/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/transcript?result_dir=${encodeURIComponent(resultDir)}`, + ); + + expect(transcriptRes.status).toBe(200); + const transcriptData = (await transcriptRes.json()) as { + status: string; + transcript_path: string; + transcript_raw_path?: string; + content: string; + }; + expect(transcriptData).toMatchObject({ + status: 'ok', + transcript_path: transcriptArtifactPath, + content: transcriptJson, + }); + expect(transcriptData.transcript_raw_path).toBeUndefined(); }); it('loads pointer-shaped transcript metadata when it resolves to a local artifact path', async () => { diff --git a/apps/dashboard/src/components/EvalDetail.tsx b/apps/dashboard/src/components/EvalDetail.tsx index be3346edd..de8845ab0 100644 --- a/apps/dashboard/src/components/EvalDetail.tsx +++ b/apps/dashboard/src/components/EvalDetail.tsx @@ -133,6 +133,7 @@ function selectedTrialResult(result: EvalResult, trial: EvalCaseTrial): EvalResu timing_path: trial.timing_path, metrics_path: trial.metrics_path, transcript_path: trial.transcript_path, + transcript_raw_path: trial.transcript_raw_path, output_path: trial.answer_path, answer_path: trial.answer_path, }; @@ -902,6 +903,7 @@ function TrialTranscriptTab({ const evalId = result.testId; const resultDir = result.result_dir; const transcriptPath = trial.transcript_path; + const transcriptRawPath = trial.transcript_raw_path; const answerPath = trial.answer_path; const { data: transcriptContent, isLoading: isLoadingTranscript } = projectId && transcriptPath @@ -987,6 +989,16 @@ function TrialTranscriptTab({ resultDir, download: true, }); + const transcriptRawHref = transcriptRawPath + ? artifactFileContentUrl({ + projectId, + runId, + evalId, + filePath: transcriptRawPath, + resultDir, + raw: true, + }) + : undefined; return ( ); @@ -1023,6 +1037,7 @@ function TranscriptTab({ ? useQuery(projectEvalTranscriptOptions(projectId, runId, evalId, resultDir)) : useEvalTranscript(runId, evalId, resultDir); const transcriptPath = transcriptData?.transcript_path; + const transcriptRawPath = transcriptData?.transcript_raw_path; const answerPath = transcriptData?.answer_path; const transcriptContent = transcriptData?.status === 'ok' ? (transcriptData.content ?? '') : ''; @@ -1159,6 +1174,16 @@ function TranscriptTab({ download: true, }) : undefined; + const transcriptRawHref = transcriptRawPath + ? artifactFileContentUrl({ + projectId, + runId, + evalId, + filePath: transcriptRawPath, + resultDir, + raw: true, + }) + : undefined; return ( ); diff --git a/apps/dashboard/src/components/TranscriptTimeline.tsx b/apps/dashboard/src/components/TranscriptTimeline.tsx index c54576f4c..7b8d49b92 100644 --- a/apps/dashboard/src/components/TranscriptTimeline.tsx +++ b/apps/dashboard/src/components/TranscriptTimeline.tsx @@ -59,9 +59,11 @@ interface TranscriptTimelineProps { finalAnswer?: string; answerPath?: string; transcriptPath?: string; + transcriptRawPath?: string; answerHref?: string; transcriptHref?: string; transcriptDownloadHref?: string; + transcriptRawHref?: string; onOpenFile?: (path: string) => void; } @@ -833,9 +835,11 @@ export function TranscriptTimeline({ finalAnswer, answerPath, transcriptPath, + transcriptRawPath, answerHref, transcriptHref, transcriptDownloadHref, + transcriptRawHref, onOpenFile, }: TranscriptTimelineProps) { const messages = useMemo(() => buildTranscriptViewModel(entries), [entries]); @@ -943,6 +947,15 @@ export function TranscriptTimeline({ Download normalized JSON + + Open transcript-raw.jsonl in Files + + Open raw JSONL + {!transcriptRawPath && ( + + Raw JSONL unavailable + + )} diff --git a/apps/dashboard/src/components/transcript-timeline.test.tsx b/apps/dashboard/src/components/transcript-timeline.test.tsx index 7eaf5914f..1239f7048 100644 --- a/apps/dashboard/src/components/transcript-timeline.test.tsx +++ b/apps/dashboard/src/components/transcript-timeline.test.tsx @@ -22,9 +22,12 @@ describe('TranscriptTimeline', () => { finalAnswer={'{"answer":42,"source":"src/app.ts"}'} answerPath="final-json-answer__codex/outputs/answer.md" transcriptPath="final-json-answer__codex/transcript.json" + transcriptRawPath="final-json-answer__codex/transcript-raw.jsonl" answerHref="/api/raw-answer" transcriptHref="/api/raw-transcript" transcriptDownloadHref="/api/download-transcript" + transcriptRawHref="/api/raw-transcript-jsonl" + onOpenFile={() => undefined} />, ); } @@ -187,6 +190,9 @@ describe('TranscriptTimeline', () => { expect(html).toContain('success'); expect(html).toContain('Open normalized JSON'); expect(html).toContain('Download normalized JSON'); + expect(html).toContain('Open transcript-raw.jsonl in Files'); + expect(html).toContain('Open raw JSONL'); + expect(html).toContain('/api/raw-transcript-jsonl'); expect(html).toContain('{"answer":42,"source":"src/app.ts"}'); }); @@ -207,4 +213,21 @@ describe('TranscriptTimeline', () => { 'very-long-project-name/very-long-test-case-name/sample-1/transcript.json', ); }); + + it('shows a clear unavailable state when only normalized transcript JSON exists', () => { + const parsed = parseTranscriptJsonl(structuredTranscriptJsonl); + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain('Open normalized JSON'); + expect(html).toContain('Raw JSONL unavailable'); + expect(html).not.toContain('Open transcript-raw.jsonl in Files'); + expect(html).not.toContain('Open raw JSONL'); + }); });