From 7de2b2e58cfd01920581531f4b6b28db206946e7 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 11:49:13 +0200 Subject: [PATCH 1/4] feat(runtime): execute host environment recipes --- .../core/src/evaluation/environment/host.ts | 129 +++++++++ .../src/evaluation/graders/script-grader.ts | 8 +- .../evaluation/loaders/environment-recipe.ts | 13 +- .../core/src/evaluation/workspace/setup.ts | 247 +++++++++++++++++- .../environment-host-runtime.test.ts | 189 ++++++++++++++ .../loaders/environment-recipe.test.ts | 2 + .../evaluation/providers/pi-runtime.test.ts | 1 + 7 files changed, 569 insertions(+), 20 deletions(-) create mode 100644 packages/core/src/evaluation/environment/host.ts create mode 100644 packages/core/test/evaluation/environment-host-runtime.test.ts diff --git a/packages/core/src/evaluation/environment/host.ts b/packages/core/src/evaluation/environment/host.ts new file mode 100644 index 000000000..ce9743556 --- /dev/null +++ b/packages/core/src/evaluation/environment/host.ts @@ -0,0 +1,129 @@ +import { mkdir } from 'node:fs/promises'; +import path from 'node:path'; + +import { execFileWithStdin, execShellWithStdin } from '../../runtime/exec.js'; +import type { + EnvironmentSetupConfig, + HostEnvironmentRecipe, +} from '../loaders/environment-recipe.js'; +import type { JsonObject } from '../types.js'; + +export type HostEnvironmentSetupStatus = 'skipped' | 'success' | 'failed'; + +export interface HostEnvironmentSetupResult { + readonly type: 'host'; + readonly workdir: string; + readonly status: HostEnvironmentSetupStatus; + readonly command?: readonly string[] | string; + readonly cwd?: string; + readonly args?: JsonObject; + readonly stdout?: string; + readonly stderr?: string; + readonly exitCode?: number; +} + +export class HostEnvironmentSetupError extends Error { + readonly result: HostEnvironmentSetupResult; + + constructor(message: string, result: HostEnvironmentSetupResult) { + super(message); + this.name = 'HostEnvironmentSetupError'; + this.result = result; + } +} + +function timeoutMs(setup: EnvironmentSetupConfig): number | undefined { + return setup.timeout_seconds !== undefined ? setup.timeout_seconds * 1000 : undefined; +} + +function setupPayload(recipe: HostEnvironmentRecipe): JsonObject { + return { + args: recipe.setup?.args ?? {}, + environment: { + type: 'host', + workdir: recipe.workdir, + }, + }; +} + +function formatSetupFailure(result: HostEnvironmentSetupResult): string { + const command = + typeof result.command === 'string' ? result.command : (result.command ?? []).join(' '); + const stderr = result.stderr?.trim(); + const stdout = result.stdout?.trim(); + const details = stderr || stdout; + return details + ? `environment.setup failed with exit code ${result.exitCode ?? 1} (${command}): ${details}` + : `environment.setup failed with exit code ${result.exitCode ?? 1} (${command})`; +} + +export async function prepareHostEnvironment( + recipe: HostEnvironmentRecipe, +): Promise { + const workdir = path.resolve(recipe.workdir); + await mkdir(workdir, { recursive: true }); + + const setup = recipe.setup; + if (!setup) { + return { + type: 'host', + workdir, + status: 'skipped', + }; + } + + const cwd = path.resolve(recipe.sourceDir); + const env = { + ...(recipe.env ?? {}), + ...(setup.env ?? {}), + AGENTV_ENVIRONMENT_WORKDIR: workdir, + }; + const payload = JSON.stringify(setupPayload({ ...recipe, workdir }), null, 2); + + let result: { readonly stdout: string; readonly stderr: string; readonly exitCode: number }; + try { + result = + typeof setup.command === 'string' + ? await execShellWithStdin(setup.command, payload, { + cwd, + env, + timeoutMs: timeoutMs(setup), + }) + : await execFileWithStdin(setup.command, payload, { + cwd, + env, + timeoutMs: timeoutMs(setup), + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const failure: HostEnvironmentSetupResult = { + type: 'host', + workdir, + status: 'failed', + command: setup.command, + cwd, + ...(setup.args !== undefined ? { args: setup.args } : {}), + stderr: message, + exitCode: 1, + }; + throw new HostEnvironmentSetupError(formatSetupFailure(failure), failure); + } + + const setupResult: HostEnvironmentSetupResult = { + type: 'host', + workdir, + status: result.exitCode === 0 ? 'success' : 'failed', + command: setup.command, + cwd, + ...(setup.args !== undefined ? { args: setup.args } : {}), + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }; + + if (result.exitCode !== 0) { + throw new HostEnvironmentSetupError(formatSetupFailure(setupResult), setupResult); + } + + return setupResult; +} diff --git a/packages/core/src/evaluation/graders/script-grader.ts b/packages/core/src/evaluation/graders/script-grader.ts index cb6e2d849..cf8446584 100644 --- a/packages/core/src/evaluation/graders/script-grader.ts +++ b/packages/core/src/evaluation/graders/script-grader.ts @@ -310,6 +310,8 @@ export class ScriptGrader implements Grader { getProxyUsage = proxy.getUsageMetadata; } + const effectiveCwd = this.cwd ?? context.workspacePath; + // Build workspace env if workspace path is available const workspaceEnv = context.workspacePath ? { AGENTV_WORKSPACE_PATH: context.workspacePath } @@ -339,7 +341,7 @@ export class ScriptGrader implements Grader { this.command, inputPayload, this.agentTimeoutMs, - this.cwd, + effectiveCwd, env, ); exitCode = result.exitCode; @@ -385,7 +387,7 @@ export class ScriptGrader implements Grader { const proxyUsage = getProxyUsage?.(); const graderRawRequest: JsonObject = { command: this.command, - ...(this.cwd ? { cwd: this.cwd } : {}), + ...(effectiveCwd ? { cwd: effectiveCwd } : {}), ...(proxyUsage ? { target_proxy: { @@ -426,7 +428,7 @@ export class ScriptGrader implements Grader { expectedAspectCount: 1, graderRawRequest: { command: this.command, - ...(this.cwd ? { cwd: this.cwd } : {}), + ...(effectiveCwd ? { cwd: effectiveCwd } : {}), ...(proxyUsage ? { target_proxy: { diff --git a/packages/core/src/evaluation/loaders/environment-recipe.ts b/packages/core/src/evaluation/loaders/environment-recipe.ts index 1d163b44c..751e035bc 100644 --- a/packages/core/src/evaluation/loaders/environment-recipe.ts +++ b/packages/core/src/evaluation/loaders/environment-recipe.ts @@ -15,13 +15,17 @@ export type EnvironmentSetupConfig = { readonly timeout_seconds?: number; }; +type EnvironmentRecipeSource = { + readonly recipeFilePath?: string; + readonly sourceDir: string; +}; + export type HostEnvironmentRecipe = { readonly type: 'host'; readonly workdir: string; readonly setup?: EnvironmentSetupConfig; readonly env?: Readonly>; - readonly recipeFilePath?: string; -}; +} & EnvironmentRecipeSource; export type DockerEnvironmentMount = { readonly source: string; @@ -48,8 +52,7 @@ export type DockerEnvironmentRecipe = { readonly mounts?: readonly DockerEnvironmentMount[]; readonly secrets?: Readonly>; readonly setup?: EnvironmentSetupConfig; - readonly recipeFilePath?: string; -}; +} & EnvironmentRecipeSource; export type EnvironmentRecipe = HostEnvironmentRecipe | DockerEnvironmentRecipe; @@ -116,6 +119,7 @@ function parseEnvironmentRecipe( return { type, workdir: resolveHostPath(workdir, baseDir), + sourceDir: baseDir, ...(setup !== undefined && { setup }), ...(env !== undefined && { env }), ...(recipeFilePath !== undefined && { recipeFilePath }), @@ -144,6 +148,7 @@ function parseEnvironmentRecipe( return { type, workdir, + sourceDir: baseDir, ...(context !== undefined && { context: resolveHostPath(context, baseDir) }), ...(dockerfile !== undefined && { dockerfile: resolveHostPath(dockerfile, baseDir) }), ...(image !== undefined && { image }), diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 725e62a68..8f5550c4a 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -17,7 +17,13 @@ import { copyFile, mkdir, stat } from 'node:fs/promises'; import path from 'node:path'; import { promisify } from 'node:util'; +import { + HostEnvironmentSetupError, + type HostEnvironmentSetupResult, + prepareHostEnvironment, +} from '../environment/host.js'; import { type ExtensionRuntimeState, runExtensionsForHook } from '../extensions/runner.js'; +import type { HostEnvironmentRecipe } from '../loaders/environment-recipe.js'; import type { AgentVExtensionConfig, EvalTest, @@ -64,10 +70,24 @@ export interface WorkspaceSetupHookExecution { readonly error?: string; } +export interface EnvironmentSetupExecution { + readonly scope: 'environment'; + readonly name: 'setup'; + readonly status: HostEnvironmentSetupResult['status']; + readonly testId: string; + readonly workdir: string; + readonly command?: readonly string[] | string; + readonly cwd?: string; + readonly output?: string; + readonly error?: string; + readonly exitCode?: number; +} + export class WorkspaceSetupError extends Error { readonly failureStage: FailureStage; readonly failureReasonCode: string; readonly hookExecutions: readonly WorkspaceSetupHookExecution[]; + readonly environmentSetupExecutions: readonly EnvironmentSetupExecution[]; constructor( message: string, @@ -75,6 +95,7 @@ export class WorkspaceSetupError extends Error { readonly failureStage: FailureStage; readonly failureReasonCode: string; readonly hookExecutions?: readonly WorkspaceSetupHookExecution[]; + readonly environmentSetupExecutions?: readonly EnvironmentSetupExecution[]; readonly cause?: unknown; }, ) { @@ -83,6 +104,7 @@ export class WorkspaceSetupError extends Error { this.failureStage = options.failureStage; this.failureReasonCode = options.failureReasonCode; this.hookExecutions = options.hookExecutions ?? []; + this.environmentSetupExecutions = options.environmentSetupExecutions ?? []; if (options.cause !== undefined) { this.cause = options.cause; } @@ -103,6 +125,7 @@ export interface SharedWorkspaceSetupOptions { export interface SharedWorkspaceSetup { readonly suiteWorkspace?: WorkspaceConfig; readonly sharedWorkspaceOwnerKey?: string; + readonly sharedEnvironmentOwnerKey?: string; readonly sharedWorkspaceAppliesToAllCases: boolean; readonly sharedWorkspacePath?: string; readonly sharedBaselineCommit?: string; @@ -112,6 +135,7 @@ export interface SharedWorkspaceSetup { readonly useStaticWorkspace: boolean; readonly configuredMode: WorkspaceSetupMode; readonly hookExecutions: readonly WorkspaceSetupHookExecution[]; + readonly environmentSetupExecutions: readonly EnvironmentSetupExecution[]; readonly extensionState?: ExtensionRuntimeState; } @@ -138,6 +162,7 @@ export interface EvalCaseWorkspaceSetup { readonly baselineCommit?: string; readonly isSharedWorkspace: boolean; readonly hookExecutions: readonly WorkspaceSetupHookExecution[]; + readonly environmentSetupExecutions: readonly EnvironmentSetupExecution[]; readonly extensionState?: ExtensionRuntimeState; } @@ -180,7 +205,10 @@ export function isAttemptScopedWorkspace( export function caseUsesSharedWorkspaceSetup( evalCase: EvalTest, - setup: Pick, + setup: Pick< + SharedWorkspaceSetup, + 'sharedWorkspaceAppliesToAllCases' | 'sharedWorkspaceOwnerKey' | 'sharedEnvironmentOwnerKey' + >, ): boolean { const workspace = effectiveRuntimeWorkspace(evalCase); if (isAttemptScopedWorkspace(workspace)) { @@ -189,10 +217,17 @@ export function caseUsesSharedWorkspaceSetup( if (setup.sharedWorkspaceAppliesToAllCases) { return true; } - return ( + if ( setup.sharedWorkspaceOwnerKey !== undefined && workspaceNeedsSharedSetup(workspace) && sharedWorkspaceOwnerKey(evalCase) === setup.sharedWorkspaceOwnerKey + ) { + return true; + } + return !!( + setup.sharedEnvironmentOwnerKey !== undefined && + hostEnvironment(evalCase) && + hostEnvironmentOwnerKey(evalCase) === setup.sharedEnvironmentOwnerKey ); } @@ -212,14 +247,16 @@ function workspaceNeedsSharedSetup( } function effectiveRuntimeWorkspace(evalCase: EvalTest): WorkspaceConfig | undefined { - const workspace = evalCase.workspace; - if (evalCase.environment?.type !== 'host') { - return workspace; - } - return { - ...(workspace ?? {}), - template: workspace?.template ?? evalCase.environment.workdir, - }; + return evalCase.workspace; +} + +function hostEnvironment(evalCase: EvalTest): HostEnvironmentRecipe | undefined { + const environment = evalCase.environment; + return environment?.type === 'host' ? environment : undefined; +} + +function unsupportedDockerEnvironment(evalCase: EvalTest): boolean { + return evalCase.environment?.type === 'docker'; } function stableWorkspaceValue(value: unknown): string { @@ -261,6 +298,16 @@ function sharedWorkspaceOwnerKey(evalCase: EvalTest): string { return `${sourceKey}:${stableWorkspaceValue(effectiveRuntimeWorkspace(evalCase))}`; } +function hostEnvironmentOwnerKey(evalCase: EvalTest): string { + const source = evalCase.source; + const sourceKey = source?.importedSuiteName + ? `imported:${source.evalFileAbsolutePath}:${source.importedSuiteName}` + : source?.evalFileAbsolutePath + ? `parent:${source.evalFileAbsolutePath}` + : 'programmatic'; + return `${sourceKey}:${stableWorkspaceValue(hostEnvironment(evalCase))}`; +} + interface SelectedSharedWorkspace { readonly key: string; readonly workspace: WorkspaceConfig; @@ -307,6 +354,67 @@ function selectSuiteWorkspace(evalCases: readonly EvalTest[]): SelectedSharedWor ); } +interface SelectedSharedEnvironment { + readonly key: string; + readonly environment: HostEnvironmentRecipe; +} + +function selectSuiteHostEnvironment( + evalCases: readonly EvalTest[], +): SelectedSharedEnvironment | undefined { + const candidates = new Map< + string, + { + readonly environment: HostEnvironmentRecipe; + readonly owner: string; + readonly testIds: string[]; + } + >(); + + for (const evalCase of evalCases) { + if (unsupportedDockerEnvironment(evalCase)) { + throw new WorkspaceSetupError( + `Docker environment recipes are not implemented for runtime execution yet: test '${evalCase.id}'.`, + { + failureStage: 'setup', + failureReasonCode: 'unsupported_environment', + }, + ); + } + const environment = hostEnvironment(evalCase); + if (!environment || isAttemptScopedWorkspace(effectiveRuntimeWorkspace(evalCase))) { + continue; + } + const key = hostEnvironmentOwnerKey(evalCase); + const existing = candidates.get(key); + if (existing) { + existing.testIds.push(evalCase.id); + continue; + } + candidates.set(key, { + environment, + owner: describeWorkspaceOwner(evalCase), + testIds: [evalCase.id], + }); + } + + if (candidates.size <= 1) { + const [key, candidate] = [...candidates.entries()][0] ?? []; + return key && candidate ? { key, environment: candidate.environment } : undefined; + } + + const owners = [...candidates.values()] + .map((candidate) => `${candidate.owner} for tests ${candidate.testIds.join(', ')}`) + .join('; '); + throw new WorkspaceSetupError( + `Wrapper eval contains multiple suite host environments: ${owners}. AgentV does not merge multiple environment.workdir values in one wrapper execution. Split them into separate runs or use workspace.scope: attempt for per-case setup.`, + { + failureStage: 'setup', + failureReasonCode: 'ambiguous_shared_environment', + }, + ); +} + function selectSuiteExtensions(evalCases: readonly EvalTest[]): readonly AgentVExtensionConfig[] { const candidates = new Map(); for (const evalCase of evalCases) { @@ -409,6 +517,58 @@ export async function releaseSharedWorkspaceSetup(setup: SharedWorkspaceSetup): void setup; } +function environmentSetupExecution(options: { + readonly result: HostEnvironmentSetupResult; + readonly testId: string; + readonly error?: string; +}): EnvironmentSetupExecution { + return { + scope: 'environment', + name: 'setup', + status: options.error ? 'failed' : options.result.status, + testId: options.testId, + workdir: options.result.workdir, + ...(options.result.command !== undefined && { command: options.result.command }), + ...(options.result.cwd !== undefined && { cwd: options.result.cwd }), + ...(options.result.stdout !== undefined && { output: options.result.stdout }), + ...((options.error ?? options.result.stderr) + ? { error: options.error ?? options.result.stderr } + : {}), + ...(options.result.exitCode !== undefined && { exitCode: options.result.exitCode }), + }; +} + +async function prepareHostEnvironmentForSetup( + environment: HostEnvironmentRecipe, + testId: string, +): Promise<{ + readonly workdir: string; + readonly execution: EnvironmentSetupExecution; +}> { + try { + const result = await prepareHostEnvironment(environment); + return { + workdir: result.workdir, + execution: environmentSetupExecution({ result, testId }), + }; + } catch (error) { + if (error instanceof HostEnvironmentSetupError) { + const execution = environmentSetupExecution({ + result: error.result, + testId, + error: error.message, + }); + throw new WorkspaceSetupError(error.message, { + failureStage: 'setup', + failureReasonCode: 'environment_setup_error', + environmentSetupExecutions: [execution], + cause: error, + }); + } + throw error; + } +} + export async function prepareSharedWorkspaceSetup( options: SharedWorkspaceSetupOptions, ): Promise { @@ -423,6 +583,7 @@ export async function prepareSharedWorkspaceSetup( legacyWorkspacePath, } = options; const selectedSuiteWorkspace = selectSuiteWorkspace(evalCases); + const selectedSuiteEnvironment = selectSuiteHostEnvironment(evalCases); const suiteWorkspace = selectedSuiteWorkspace?.workspace; const suiteExtensions = selectSuiteExtensions(evalCases); const rawTemplate = suiteWorkspace?.template; @@ -442,7 +603,7 @@ export async function prepareSharedWorkspaceSetup( const configuredMode: WorkspaceSetupMode = cliWorkspacePath ? 'static' : 'temp'; const configuredStaticPath = cliWorkspacePath; - const useStaticWorkspace = configuredMode === 'static'; + const useStaticWorkspace = configuredMode === 'static' || !!selectedSuiteEnvironment; if ( useStaticWorkspace && @@ -454,6 +615,7 @@ export async function prepareSharedWorkspaceSetup( } const hasSharedWorkspace = !!( useStaticWorkspace || + selectedSuiteEnvironment || (!isAttemptWorkspace && (workspaceTemplate || suiteWorkspace?.hooks || suiteWorkspace?.repos?.length)) || suiteExtensions.length > 0 @@ -477,6 +639,7 @@ export async function prepareSharedWorkspaceSetup( let beforeAllOutput: string | undefined; const hookExecutions: WorkspaceSetupHookExecution[] = []; + const environmentSetupExecutions: EnvironmentSetupExecution[] = []; let extensionState: ExtensionRuntimeState | undefined; let repoManager: RepoManager | undefined; @@ -484,6 +647,28 @@ export async function prepareSharedWorkspaceSetup( if (useStaticWorkspace && configuredStaticPath) { setupLog(`reusing existing static workspace: ${configuredStaticPath}`); sharedWorkspacePath = configuredStaticPath; + } else if (selectedSuiteEnvironment) { + setupLog(`preparing shared host environment: ${selectedSuiteEnvironment.environment.workdir}`); + try { + const prepared = await prepareHostEnvironmentForSetup( + selectedSuiteEnvironment.environment, + '__environment_setup__', + ); + sharedWorkspacePath = prepared.workdir; + environmentSetupExecutions.push(prepared.execution); + setupLog(`shared host environment ready at: ${sharedWorkspacePath}`); + } catch (error) { + if (error instanceof WorkspaceSetupError) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + throw new WorkspaceSetupError(`Failed to prepare host environment: ${message}`, { + failureStage: 'setup', + failureReasonCode: 'environment_setup_error', + hookExecutions, + cause: error, + }); + } } else if (!isAttemptWorkspace && workspaceTemplate) { setupLog(`creating shared workspace from template: ${workspaceTemplate}`); try { @@ -721,6 +906,9 @@ export async function prepareSharedWorkspaceSetup( ...(selectedSuiteWorkspace?.key !== undefined && { sharedWorkspaceOwnerKey: selectedSuiteWorkspace.key, }), + ...(selectedSuiteEnvironment?.key !== undefined && { + sharedEnvironmentOwnerKey: selectedSuiteEnvironment.key, + }), sharedWorkspaceAppliesToAllCases, ...(sharedWorkspacePath !== undefined && { sharedWorkspacePath }), ...(sharedBaselineCommit !== undefined && { sharedBaselineCommit }), @@ -730,6 +918,7 @@ export async function prepareSharedWorkspaceSetup( useStaticWorkspace, configuredMode, hookExecutions, + environmentSetupExecutions, ...(extensionState !== undefined && { extensionState }), }; } @@ -752,24 +941,55 @@ export async function prepareEvalCaseWorkspace( } = options; const runtimeWorkspace = effectiveRuntimeWorkspace(evalCase); + if (unsupportedDockerEnvironment(evalCase)) { + throw new WorkspaceSetupError( + `Docker environment recipes are not implemented for runtime execution yet: test '${evalCase.id}'.`, + { + failureStage: 'setup', + failureReasonCode: 'unsupported_environment', + }, + ); + } + const runtimeEnvironment = hostEnvironment(evalCase); let workspacePath: string | undefined = isAttemptScopedWorkspace(runtimeWorkspace) ? undefined : sharedWorkspacePath; const inheritedSuiteWorkspaceFile = workspacePath ? suiteWorkspaceFile : undefined; let beforeAllOutput: string | undefined; let beforeEachOutput: string | undefined; - const isSharedWorkspace = !!workspacePath; + let isSharedWorkspace = !!workspacePath; let caseWorkspaceFile: string | undefined; const caseHooksEnabled = hooksEnabled(runtimeWorkspace); const hookExecutions: WorkspaceSetupHookExecution[] = []; + const environmentSetupExecutions: EnvironmentSetupExecution[] = []; let extensionState = sharedExtensionState; if (!workspacePath) { + if (runtimeEnvironment) { + try { + const prepared = await prepareHostEnvironmentForSetup(runtimeEnvironment, evalCase.id); + workspacePath = prepared.workdir; + isSharedWorkspace = true; + environmentSetupExecutions.push(prepared.execution); + } catch (error) { + if (error instanceof WorkspaceSetupError) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + throw new WorkspaceSetupError(`Failed to prepare host environment: ${message}`, { + failureStage: 'setup', + failureReasonCode: 'environment_setup_error', + hookExecutions, + cause: error, + }); + } + } + const rawCaseTemplate = runtimeWorkspace?.template; const resolvedCaseTemplate = await resolveWorkspaceTemplate(rawCaseTemplate); const caseWorkspaceTemplate = resolvedCaseTemplate?.dir; caseWorkspaceFile = resolvedCaseTemplate?.workspaceFile; - if (caseWorkspaceTemplate && evalRunId) { + if (!workspacePath && caseWorkspaceTemplate && evalRunId) { try { workspacePath = await createTempWorkspace(caseWorkspaceTemplate, evalRunId, evalCase.id); } catch (error) { @@ -1179,6 +1399,7 @@ export async function prepareEvalCaseWorkspace( ...(baselineCommit !== undefined && { baselineCommit }), isSharedWorkspace, hookExecutions, + environmentSetupExecutions, ...(extensionState !== undefined && { extensionState }), }; } diff --git a/packages/core/test/evaluation/environment-host-runtime.test.ts b/packages/core/test/evaluation/environment-host-runtime.test.ts new file mode 100644 index 000000000..f6fe1af7b --- /dev/null +++ b/packages/core/test/evaluation/environment-host-runtime.test.ts @@ -0,0 +1,189 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { prepareHostEnvironment } from '../../src/evaluation/environment/host.js'; +import { type RunEvaluationOptions, runEvaluation } from '../../src/evaluation/orchestrator.js'; +import type { + Provider, + ProviderRequest, + ProviderResponse, +} from '../../src/evaluation/providers/types.js'; +import type { EvalTest } from '../../src/evaluation/types.js'; + +const tempDirs: string[] = []; + +function tempDir(prefix: string): string { + const dir = mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function baseEvalCase(overrides: Partial): EvalTest { + return { + id: 'case-1', + suite: 'environment-host-runtime', + question: 'Return ok', + input: [{ role: 'user', content: 'Return ok' }], + expected_output: [], + reference_answer: '', + file_paths: [], + criteria: '', + assertions: [{ name: 'contains-ok', type: 'contains', value: 'ok' }], + ...overrides, + }; +} + +function providerFactory(captured: ProviderRequest[]): RunEvaluationOptions['providerFactory'] { + return (target): Provider => ({ + id: `${target.kind}:${target.name}`, + kind: target.kind, + targetName: target.name, + async invoke(request): Promise { + captured.push(request); + return { + output: [{ role: 'assistant', content: 'ok' }], + durationMs: 1, + }; + }, + }); +} + +describe('host environment runtime', () => { + it('creates host workdir and passes deterministic setup args on stdin from recipe source cwd', async () => { + const root = tempDir('agentv-host-env-'); + const sourceDir = path.join(root, '.agentv/environments'); + const workdir = path.join(sourceDir, 'checkout'); + await mkdir(sourceDir, { recursive: true }); + const setupScript = path.join(sourceDir, 'setup.mjs'); + await writeFile( + setupScript, + [ + "import { readFileSync, writeFileSync } from 'node:fs';", + "import path from 'node:path';", + "const payload = JSON.parse(readFileSync(0, 'utf8'));", + "writeFileSync(path.join(payload.environment.workdir, 'setup-payload.json'), JSON.stringify({ payload, cwd: process.cwd(), envWorkdir: process.env.AGENTV_ENVIRONMENT_WORKDIR }, null, 2));", + ].join('\n'), + ); + + const result = await prepareHostEnvironment({ + type: 'host', + workdir, + sourceDir, + setup: { + command: ['node', 'setup.mjs'], + args: { + repo: 'example/repo', + nested: { count: 2, enabled: true }, + }, + }, + }); + + const payload = JSON.parse(await readFile(path.join(workdir, 'setup-payload.json'), 'utf8')); + expect(result.status).toBe('success'); + expect(existsSync(workdir)).toBe(true); + expect(payload.cwd).toBe(sourceDir); + expect(payload.envWorkdir).toBe(workdir); + expect(payload.payload).toEqual({ + args: { + repo: 'example/repo', + nested: { count: 2, enabled: true }, + }, + environment: { + type: 'host', + workdir, + }, + }); + }); + + it('returns structured setup failure details for non-zero setup commands', async () => { + const root = tempDir('agentv-host-env-fail-'); + await expect( + prepareHostEnvironment({ + type: 'host', + workdir: path.join(root, 'checkout'), + sourceDir: root, + setup: { + command: ['node', '-e', "console.error('setup failed'); process.exit(7)"], + args: { reason: 'test' }, + }, + }), + ).rejects.toMatchObject({ + result: { + status: 'failed', + exitCode: 7, + stderr: 'setup failed\n', + args: { reason: 'test' }, + }, + }); + }); + + it('passes environment workdir to generic provider requests and script graders', async () => { + const root = tempDir('agentv-host-env-eval-'); + const workdir = path.join(root, 'workdir'); + const sourceDir = path.join(root, '.agentv/environments'); + const legacyTemplate = path.join(root, 'legacy-template'); + await mkdir(sourceDir, { recursive: true }); + await mkdir(legacyTemplate, { recursive: true }); + const setupScript = path.join(sourceDir, 'setup.mjs'); + const graderScript = path.join(root, 'grader.mjs'); + await writeFile( + setupScript, + "import { writeFileSync } from 'node:fs'; writeFileSync(process.env.AGENTV_ENVIRONMENT_WORKDIR + '/ready.txt', 'ready');\n", + ); + await writeFile( + graderScript, + [ + "import { readFileSync } from 'node:fs';", + "const payload = JSON.parse(readFileSync(0, 'utf8'));", + "const pass = process.cwd() === payload.workspace_path && payload.workspace_path.endsWith('/workdir');", + "console.log(JSON.stringify({ pass, score: pass ? 1 : 0, reason: process.cwd(), checks: [{ text: 'cwd matches environment workdir', pass, reason: payload.workspace_path }] }));", + ].join('\n'), + ); + + const capturedRequests: ProviderRequest[] = []; + const results = await runEvaluation({ + testFilePath: path.join(root, 'suite.eval.yaml'), + repoRoot: root, + target: { kind: 'cli', name: 'generic-cli', config: {} }, + providerFactory: providerFactory(capturedRequests), + evalCases: [ + baseEvalCase({ + environment: { + type: 'host', + workdir, + sourceDir, + setup: { command: ['node', 'setup.mjs'] }, + }, + workspace: { + template: legacyTemplate, + }, + assertions: [ + { + name: 'script-cwd', + type: 'script', + command: ['node', graderScript], + }, + ], + }), + ], + maxConcurrency: 1, + retainOnSuccess: 'cleanup', + }); + + expect(capturedRequests).toHaveLength(1); + expect(capturedRequests[0].cwd).toBe(workdir); + expect(results[0].score).toBe(1); + expect(results[0].scores?.[0]?.reason).toBe(workdir); + expect(existsSync(path.join(workdir, 'ready.txt'))).toBe(true); + expect(existsSync(workdir)).toBe(true); + }); +}); diff --git a/packages/core/test/evaluation/loaders/environment-recipe.test.ts b/packages/core/test/evaluation/loaders/environment-recipe.test.ts index fe48635da..d8f238016 100644 --- a/packages/core/test/evaluation/loaders/environment-recipe.test.ts +++ b/packages/core/test/evaluation/loaders/environment-recipe.test.ts @@ -59,6 +59,7 @@ describe('environment recipe loading', () => { expect(tests[0].environment).toEqual({ type: 'host', workdir: path.join(dir, 'workspaces/app'), + sourceDir: dir, setup: { command: './scripts/setup.sh', args: { @@ -127,6 +128,7 @@ describe('environment recipe loading', () => { context: path.join(dir, 'environment'), dockerfile: path.join(dir, 'Dockerfile'), workdir: '/app', + sourceDir: dir, env: { NODE_ENV: 'test' }, resources: { cpus: 2, memory: '4g' }, mounts: [{ source: path.join(dir, 'fixtures'), target: '/fixtures', access: 'ro' }], diff --git a/packages/core/test/evaluation/providers/pi-runtime.test.ts b/packages/core/test/evaluation/providers/pi-runtime.test.ts index 55dcbfdbe..dd292510e 100644 --- a/packages/core/test/evaluation/providers/pi-runtime.test.ts +++ b/packages/core/test/evaluation/providers/pi-runtime.test.ts @@ -38,6 +38,7 @@ describe('Pi coding-agent runtime providers', () => { expect(extractLastAssistantContent(response.output)).toBe('cli ok'); expect(captured?.command.slice(0, 3)).toEqual(['pi-shim', '--profile', 'clean']); + expect(captured?.cwd).toBe(workspace); expect(captured?.command).toContain('--mode'); expect(captured?.command).toContain('json'); expect(captured?.env.HOME).toBe('/tmp/pi-profile'); From 4b07ef358e1367ca58228b56fbd0f625d019d0d9 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 11:54:26 +0200 Subject: [PATCH 2/4] test(runtime): use real workspace for script grader cwd --- packages/core/test/evaluation/graders.test.ts | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index b6b5a5330..4d0c2c5f6 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, spyOn } from 'bun:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -1464,29 +1466,34 @@ describe('ScriptGrader', () => { it('passes workspace_path to script grader via payload and env var', async () => { const graderProvider = new StubProvider(textResponse('{}')); + const workspacePath = await mkdtemp(join(tmpdir(), 'agentv-script-grader-workspace-')); const __dirname = dirname(fileURLToPath(import.meta.url)); const script = ['node', join(__dirname, '../fixtures/test-grader-workspace.cjs')]; const evaluator = new ScriptGrader({ command: script }); - const result = await evaluator.evaluate({ - evalCase: baseTestCase, - candidate: 'Test candidate', - target: baseTarget, - provider: graderProvider, - attempt: 0, - promptInputs: { question: '' }, - now: new Date(), - workspacePath: '/tmp/test-workspace', - }); - - expect(result.score).toBe(1); - expect(result.verdict).toBe('pass'); - const passedTexts2 = result.assertions.filter((a) => a.passed).map((a) => a.text); - expect(passedTexts2).toContain('workspace_path present in payload'); - expect(passedTexts2).toContain('AGENTV_WORKSPACE_PATH env var set'); - expect(passedTexts2).toContain('payload and env var match'); + try { + const result = await evaluator.evaluate({ + evalCase: baseTestCase, + candidate: 'Test candidate', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + workspacePath, + }); + + expect(result.score).toBe(1); + expect(result.verdict).toBe('pass'); + const passedTexts2 = result.assertions.filter((a) => a.passed).map((a) => a.text); + expect(passedTexts2).toContain('workspace_path present in payload'); + expect(passedTexts2).toContain('AGENTV_WORKSPACE_PATH env var set'); + expect(passedTexts2).toContain('payload and env var match'); + } finally { + await rm(workspacePath, { recursive: true, force: true }); + } }); it('omits details when not returned by script grader', async () => { From c5d333b938935a726cf9909e9ee03f8cd15958ec Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 11:58:45 +0200 Subject: [PATCH 3/4] fix(cli): omit host recipe source metadata from bundles --- apps/cli/src/commands/eval/task-bundle.ts | 6 +++++- apps/cli/test/commands/eval/bundle.test.ts | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index a2c1c19fb..2a1473db3 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -811,7 +811,11 @@ function serializeEnvironment( environment: EnvironmentRecipe, rewrites: ReadonlyMap, ): Record { - const { recipeFilePath: _recipeFilePath, ...portableEnvironment } = environment; + const { + recipeFilePath: _recipeFilePath, + sourceDir: _sourceDir, + ...portableEnvironment + } = environment; return rewritePathsDeep(portableEnvironment, rewrites) as Record; } diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index bfa7f6bc4..92d1b01f9 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -150,6 +150,7 @@ tests: ../data/cases.yaml workdir: 'workspaces/workspace-template', setup: { command: ['bun', 'scripts/scripts/setup.ts'] }, }); + expect(testCase.environment).not.toHaveProperty('source_dir'); expect(bundledEval.prompts).toEqual(['{{ input }}']); const input = (testCase.vars as Record).input as Array<{ content: Array>; From a83057bbd84cb94a4bee293e0f5a1fc3bb13d934 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 12:11:25 +0200 Subject: [PATCH 4/4] fix(runtime): disable batching for environment cases --- packages/core/src/evaluation/orchestrator.ts | 6 +- .../core/test/evaluation/orchestrator.test.ts | 66 +++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 4e942f531..07a89c73b 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -910,10 +910,12 @@ export async function runEvaluation( const requiresWorkspaceDispatch = workspacePath !== undefined || legacyWorkspacePath !== undefined || - filteredEvalCases.some((evalCase) => evalCase.workspace !== undefined); + filteredEvalCases.some( + (evalCase) => evalCase.workspace !== undefined || evalCase.environment !== undefined, + ); if (providerSupportsBatch && requiresWorkspaceDispatch) { if (verbose) { - console.warn('Warning: Batch mode is disabled for workspace-enabled evals.'); + console.warn('Warning: Batch mode is disabled for workspace- or environment-enabled evals.'); } providerSupportsBatch = false; batchingDisabledByRuntimePolicy = true; diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 73b9aeb8b..444d1dbda 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -967,6 +967,72 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, } }); + it('disables request batching when cases require environment setup', async () => { + class BatchCapableProvider implements Provider { + readonly id = 'batch:environment'; + readonly kind = 'mock' as const; + readonly targetName = 'environment'; + readonly supportsBatch = true; + batchCalls = 0; + invokeRequests: ProviderRequest[] = []; + + async invoke(request: ProviderRequest): Promise { + this.invokeRequests.push(request); + return { + output: [{ role: 'assistant', content: 'OK' }], + }; + } + + async invokeBatch(): Promise { + this.batchCalls += 1; + throw new Error('batch should not be used for environment cases'); + } + } + + const sourceDir = mkdtempSync(path.join(tmpdir(), 'agentv-batch-environment-source-')); + const workdir = path.join(sourceDir, 'workdir'); + writeFileSync( + path.join(sourceDir, 'setup.mjs'), + "import { writeFileSync } from 'node:fs'; writeFileSync(process.env.AGENTV_ENVIRONMENT_WORKDIR + '/ready.txt', 'ok');\n", + 'utf8', + ); + const provider = new BatchCapableProvider(); + + try { + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: { + ...baseTarget, + providerBatching: true, + workers: 1, + }, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases: [ + { + ...baseTestCase, + environment: { + type: 'host', + workdir, + sourceDir, + setup: { command: ['node', 'setup.mjs'] }, + }, + }, + ], + }); + + expect(results).toHaveLength(1); + expect(provider.batchCalls).toBe(0); + expect(provider.invokeRequests).toHaveLength(1); + expect(provider.invokeRequests[0]?.cwd).toBe(workdir); + expect(existsSync(path.join(workdir, 'ready.txt'))).toBe(true); + } finally { + const { rm } = await import('node:fs/promises'); + await rm(sourceDir, { recursive: true, force: true }); + } + }); + it('disables request batching when repeat attempts are configured', async () => { class BatchCapableProvider implements Provider { readonly id = 'batch:repeat';