Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions apps/cli/src/commands/results/serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -975,6 +975,16 @@ async function readArtifactCatalogEntryText(
return { content };
}

function readableLocalCatalogEntry(
meta: SourcedResultFileMeta,
entry: ArtifactCatalogEntry | undefined,
): ArtifactCatalogEntry | undefined {
if (!entry || entry.storage !== 'local') return entry;
const baseDir = runWorkspaceDirFromManifestPath(meta.path);
const resolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath);
return resolved.absolutePath ? entry : undefined;
}

function artifactFileContentResponse(c: C, filePath: string, fileContent: string) {
if (c.req.query('raw') === '1' || c.req.query('download') === '1') {
c.header('Content-Type', inferRawContentType(filePath));
Expand Down Expand Up @@ -2143,6 +2153,12 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext)
runPath,
);
const answerEntry = findPointerArtifactCatalogEntry(catalog, answer, 'answer', runPath);
const transcriptRawEntry = readableLocalCatalogEntry(
meta,
record.transcript_raw_path
? findArtifactCatalogEntry(catalog, record.transcript_raw_path)
: undefined,
);

if (!transcriptEntry && !transcript.path) {
return c.json({
Expand Down Expand Up @@ -2194,6 +2210,7 @@ async function handleEvalTranscript(c: C, { searchDir, projectId }: DataContext)
return c.json({
status: 'ok',
transcript_path: transcriptEntry.displayPath,
...(transcriptRawEntry && { transcript_raw_path: transcriptRawEntry.displayPath }),
content,
language: inferLanguage(transcriptEntry.displayPath),
...(answerEntry && { answer_path: answerEntry.displayPath }),
Expand Down
72 changes: 72 additions & 0 deletions apps/cli/test/commands/results/serve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3115,8 +3115,10 @@ describe('serve app', () => {
const timestampDir = path.join(runsDir, runId);
const resultDir = 'demo-test-greeting--111111111111';
const transcriptArtifactPath = `${resultDir}/sample-1/transcript.json`;
const transcriptRawArtifactPath = `${resultDir}/sample-1/transcript-raw.jsonl`;
const answerArtifactPath = `${resultDir}/sample-1/outputs/answer.md`;
const transcriptPath = path.join(timestampDir, transcriptArtifactPath);
const transcriptRawPath = path.join(timestampDir, transcriptRawArtifactPath);
const answerPath = path.join(timestampDir, answerArtifactPath);
const transcriptJson = `${JSON.stringify(
{
Expand All @@ -3142,9 +3144,14 @@ describe('serve app', () => {
null,
2,
)}\n`;
const transcriptRawJsonl = `${JSON.stringify({
type: 'raw_event',
message: 'provider event',
})}\n`;

mkdirSync(path.dirname(transcriptPath), { recursive: true });
writeFileSync(transcriptPath, transcriptJson);
writeFileSync(transcriptRawPath, transcriptRawJsonl);
mkdirSync(path.dirname(answerPath), { recursive: true });
writeFileSync(answerPath, 'done');
mkdirSync(path.join(timestampDir, '.internal'), { recursive: true });
Expand All @@ -3155,6 +3162,7 @@ describe('serve app', () => {
experiment: 'canonical-transcript',
result_dir: resultDir,
transcript_path: transcriptArtifactPath,
transcript_raw_path: transcriptRawArtifactPath,
answer_path: answerArtifactPath,
}),
);
Expand All @@ -3168,13 +3176,15 @@ describe('serve app', () => {
const transcriptData = (await transcriptRes.json()) as {
status: string;
transcript_path: string;
transcript_raw_path: string;
content: string;
answer_path: string;
answer_content: string;
};
expect(transcriptData).toMatchObject({
status: 'ok',
transcript_path: transcriptArtifactPath,
transcript_raw_path: transcriptRawArtifactPath,
content: transcriptJson,
answer_path: answerArtifactPath,
answer_content: 'done',
Expand All @@ -3186,6 +3196,68 @@ describe('serve app', () => {

expect(rawRes.status).toBe(200);
expect(await rawRes.text()).toBe(transcriptJson);

const rawSidecarRes = await app.request(
`/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/files/${transcriptRawArtifactPath}?result_dir=${encodeURIComponent(resultDir)}&raw=1`,
);

expect(rawSidecarRes.status).toBe(200);
expect(await rawSidecarRes.text()).toBe(transcriptRawJsonl);
});

it('omits missing raw transcript sidecars from the transcript response', async () => {
const runsDir = localResultsExperimentDir(tempDir, 'canonical-transcript-normalized-only');
const runId = '2026-03-25T10-45-00-000Z';
const timestampDir = path.join(runsDir, runId);
const resultDir = 'demo-test-greeting--222222222222';
const transcriptArtifactPath = `${resultDir}/sample-1/transcript.json`;
const transcriptRawArtifactPath = `${resultDir}/sample-1/transcript-raw.jsonl`;
const transcriptPath = path.join(timestampDir, transcriptArtifactPath);
const transcriptJson = `${JSON.stringify({
schema_version: 'agentv.normalized_transcript.v1',
target: 'codex',
turns: [
{
v: 1,
agent: 'codex',
type: 'assistant',
content: [{ type: 'text', text: 'done' }],
},
],
})}\n`;

mkdirSync(path.dirname(transcriptPath), { recursive: true });
writeFileSync(transcriptPath, transcriptJson);
mkdirSync(path.join(timestampDir, '.internal'), { recursive: true });
writeFileSync(
path.join(timestampDir, '.internal', 'index.jsonl'),
toJsonl({
...RESULT_A,
experiment: 'canonical-transcript-normalized-only',
result_dir: resultDir,
transcript_path: transcriptArtifactPath,
transcript_raw_path: transcriptRawArtifactPath,
}),
);

const app = createApp([], tempDir, tempDir, undefined, { studioDir });
const transcriptRes = await app.request(
`/api/runs/${encodeURIComponent(runId)}/evals/test-greeting/transcript?result_dir=${encodeURIComponent(resultDir)}`,
);

expect(transcriptRes.status).toBe(200);
const transcriptData = (await transcriptRes.json()) as {
status: string;
transcript_path: string;
transcript_raw_path?: string;
content: string;
};
expect(transcriptData).toMatchObject({
status: 'ok',
transcript_path: transcriptArtifactPath,
content: transcriptJson,
});
expect(transcriptData.transcript_raw_path).toBeUndefined();
});

it('loads pointer-shaped transcript metadata when it resolves to a local artifact path', async () => {
Expand Down
27 changes: 27 additions & 0 deletions apps/dashboard/src/components/EvalDetail.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ function selectedTrialResult(result: EvalResult, trial: EvalCaseTrial): EvalResu
timing_path: trial.timing_path,
metrics_path: trial.metrics_path,
transcript_path: trial.transcript_path,
transcript_raw_path: trial.transcript_raw_path,
output_path: trial.answer_path,
answer_path: trial.answer_path,
};
Expand Down Expand Up @@ -902,6 +903,7 @@ function TrialTranscriptTab({
const evalId = result.testId;
const resultDir = result.result_dir;
const transcriptPath = trial.transcript_path;
const transcriptRawPath = trial.transcript_raw_path;
const answerPath = trial.answer_path;
const { data: transcriptContent, isLoading: isLoadingTranscript } =
projectId && transcriptPath
Expand Down Expand Up @@ -987,16 +989,28 @@ function TrialTranscriptTab({
resultDir,
download: true,
});
const transcriptRawHref = transcriptRawPath
? artifactFileContentUrl({
projectId,
runId,
evalId,
filePath: transcriptRawPath,
resultDir,
raw: true,
})
: undefined;

return (
<TranscriptTimeline
entries={parsedTranscript.entries}
finalAnswer={answerPath ? (answerContent?.content ?? result.output) : undefined}
answerPath={answerPath}
transcriptPath={transcriptPath}
transcriptRawPath={transcriptRawPath}
answerHref={answerHref}
transcriptHref={transcriptHref}
transcriptDownloadHref={transcriptDownloadHref}
transcriptRawHref={transcriptRawHref}
onOpenFile={onOpenFile}
/>
);
Expand All @@ -1023,6 +1037,7 @@ function TranscriptTab({
? useQuery(projectEvalTranscriptOptions(projectId, runId, evalId, resultDir))
: useEvalTranscript(runId, evalId, resultDir);
const transcriptPath = transcriptData?.transcript_path;
const transcriptRawPath = transcriptData?.transcript_raw_path;
const answerPath = transcriptData?.answer_path;
const transcriptContent = transcriptData?.status === 'ok' ? (transcriptData.content ?? '') : '';

Expand Down Expand Up @@ -1159,16 +1174,28 @@ function TranscriptTab({
download: true,
})
: undefined;
const transcriptRawHref = transcriptRawPath
? artifactFileContentUrl({
projectId,
runId,
evalId,
filePath: transcriptRawPath,
resultDir,
raw: true,
})
: undefined;

return (
<TranscriptTimeline
entries={parsedTranscript.entries}
finalAnswer={answerPath ? (transcriptData.answer_content ?? result.output) : undefined}
answerPath={answerPath}
transcriptPath={transcriptPath}
transcriptRawPath={transcriptRawPath}
answerHref={answerHref}
transcriptHref={transcriptHref}
transcriptDownloadHref={transcriptDownloadHref}
transcriptRawHref={transcriptRawHref}
onOpenFile={onOpenFile}
/>
);
Expand Down
13 changes: 13 additions & 0 deletions apps/dashboard/src/components/TranscriptTimeline.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,11 @@ interface TranscriptTimelineProps {
finalAnswer?: string;
answerPath?: string;
transcriptPath?: string;
transcriptRawPath?: string;
answerHref?: string;
transcriptHref?: string;
transcriptDownloadHref?: string;
transcriptRawHref?: string;
onOpenFile?: (path: string) => void;
}

Expand Down Expand Up @@ -833,9 +835,11 @@ export function TranscriptTimeline({
finalAnswer,
answerPath,
transcriptPath,
transcriptRawPath,
answerHref,
transcriptHref,
transcriptDownloadHref,
transcriptRawHref,
onOpenFile,
}: TranscriptTimelineProps) {
const messages = useMemo(() => buildTranscriptViewModel(entries), [entries]);
Expand Down Expand Up @@ -943,6 +947,15 @@ export function TranscriptTimeline({
<ActionLink href={transcriptDownloadHref} download>
Download normalized JSON
</ActionLink>
<OpenFileButton path={transcriptRawPath} onOpenFile={onOpenFile}>
Open transcript-raw.jsonl in Files
</OpenFileButton>
<ActionLink href={transcriptRawHref}>Open raw JSONL</ActionLink>
{!transcriptRawPath && (
<span className="rounded-md border border-gray-800 bg-gray-950 px-3 py-1.5 text-sm text-gray-500">
Raw JSONL unavailable
</span>
)}
</div>
</div>
</section>
Expand Down
23 changes: 23 additions & 0 deletions apps/dashboard/src/components/transcript-timeline.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ describe('TranscriptTimeline', () => {
finalAnswer={'{"answer":42,"source":"src/app.ts"}'}
answerPath="final-json-answer__codex/outputs/answer.md"
transcriptPath="final-json-answer__codex/transcript.json"
transcriptRawPath="final-json-answer__codex/transcript-raw.jsonl"
answerHref="/api/raw-answer"
transcriptHref="/api/raw-transcript"
transcriptDownloadHref="/api/download-transcript"
transcriptRawHref="/api/raw-transcript-jsonl"
onOpenFile={() => undefined}
/>,
);
}
Expand Down Expand Up @@ -187,6 +190,9 @@ describe('TranscriptTimeline', () => {
expect(html).toContain('success');
expect(html).toContain('Open normalized JSON');
expect(html).toContain('Download normalized JSON');
expect(html).toContain('Open transcript-raw.jsonl in Files');
expect(html).toContain('Open raw JSONL');
expect(html).toContain('/api/raw-transcript-jsonl');
expect(html).toContain('{&quot;answer&quot;:42,&quot;source&quot;:&quot;src/app.ts&quot;}');
});

Expand All @@ -207,4 +213,21 @@ describe('TranscriptTimeline', () => {
'very-long-project-name/very-long-test-case-name/sample-1/transcript.json',
);
});

it('shows a clear unavailable state when only normalized transcript JSON exists', () => {
const parsed = parseTranscriptJsonl(structuredTranscriptJsonl);
const html = renderToStaticMarkup(
<TranscriptTimeline
entries={parsed.entries}
transcriptPath="final-json-answer__codex/sample-1/transcript.json"
transcriptHref="/api/raw-transcript"
transcriptDownloadHref="/api/download-transcript"
/>,
);

expect(html).toContain('Open normalized JSON');
expect(html).toContain('Raw JSONL unavailable');
expect(html).not.toContain('Open transcript-raw.jsonl in Files');
expect(html).not.toContain('Open raw JSONL');
});
});
Loading