feat(runtime): execute host environment recipes

apps/cli/src/commands/eval/task-bundle.ts

@@ -811,7 +811,11 @@ function serializeEnvironment(
   environment: EnvironmentRecipe,
   rewrites: ReadonlyMap<string, string>,
 ): Record<string, unknown> {
-  const { recipeFilePath: _recipeFilePath, ...portableEnvironment } = environment;
+  const {
+    recipeFilePath: _recipeFilePath,
+    sourceDir: _sourceDir,
+    ...portableEnvironment
+  } = environment;
   return rewritePathsDeep(portableEnvironment, rewrites) as Record<string, unknown>;
 }
 

apps/cli/test/commands/eval/bundle.test.ts

@@ -150,6 +150,7 @@ tests: ../data/cases.yaml
       workdir: 'workspaces/workspace-template',
       setup: { command: ['bun', 'scripts/scripts/setup.ts'] },
     });
+    expect(testCase.environment).not.toHaveProperty('source_dir');
     expect(bundledEval.prompts).toEqual(['{{ input }}']);
     const input = (testCase.vars as Record<string, unknown>).input as Array<{
       content: Array<Record<string, unknown>>;

packages/core/src/evaluation/environment/host.ts

@@ -0,0 +1,129 @@
+import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
+
+import { execFileWithStdin, execShellWithStdin } from '../../runtime/exec.js';
+import type {
+  EnvironmentSetupConfig,
+  HostEnvironmentRecipe,
+} from '../loaders/environment-recipe.js';
+import type { JsonObject } from '../types.js';
+
+export type HostEnvironmentSetupStatus = 'skipped' | 'success' | 'failed';
+
+export interface HostEnvironmentSetupResult {
+  readonly type: 'host';
+  readonly workdir: string;
+  readonly status: HostEnvironmentSetupStatus;
+  readonly command?: readonly string[] | string;
+  readonly cwd?: string;
+  readonly args?: JsonObject;
+  readonly stdout?: string;
+  readonly stderr?: string;
+  readonly exitCode?: number;
+}
+
+export class HostEnvironmentSetupError extends Error {
+  readonly result: HostEnvironmentSetupResult;
+
+  constructor(message: string, result: HostEnvironmentSetupResult) {
+    super(message);
+    this.name = 'HostEnvironmentSetupError';
+    this.result = result;
+  }
+}
+
+function timeoutMs(setup: EnvironmentSetupConfig): number | undefined {
+  return setup.timeout_seconds !== undefined ? setup.timeout_seconds * 1000 : undefined;
+}
+
+function setupPayload(recipe: HostEnvironmentRecipe): JsonObject {
+  return {
+    args: recipe.setup?.args ?? {},
+    environment: {
+      type: 'host',
+      workdir: recipe.workdir,
+    },
+  };
+}
+
+function formatSetupFailure(result: HostEnvironmentSetupResult): string {
+  const command =
+    typeof result.command === 'string' ? result.command : (result.command ?? []).join(' ');
+  const stderr = result.stderr?.trim();
+  const stdout = result.stdout?.trim();
+  const details = stderr || stdout;
+  return details
+    ? `environment.setup failed with exit code ${result.exitCode ?? 1} (${command}): ${details}`
+    : `environment.setup failed with exit code ${result.exitCode ?? 1} (${command})`;
+}
+
+export async function prepareHostEnvironment(
+  recipe: HostEnvironmentRecipe,
+): Promise<HostEnvironmentSetupResult> {
+  const workdir = path.resolve(recipe.workdir);
+  await mkdir(workdir, { recursive: true });
+
+  const setup = recipe.setup;
+  if (!setup) {
+    return {
+      type: 'host',
+      workdir,
+      status: 'skipped',
+    };
+  }
+
+  const cwd = path.resolve(recipe.sourceDir);
+  const env = {
+    ...(recipe.env ?? {}),
+    ...(setup.env ?? {}),
+    AGENTV_ENVIRONMENT_WORKDIR: workdir,
+  };
+  const payload = JSON.stringify(setupPayload({ ...recipe, workdir }), null, 2);
+
+  let result: { readonly stdout: string; readonly stderr: string; readonly exitCode: number };
+  try {
+    result =
+      typeof setup.command === 'string'
+        ? await execShellWithStdin(setup.command, payload, {
+            cwd,
+            env,
+            timeoutMs: timeoutMs(setup),
+          })
+        : await execFileWithStdin(setup.command, payload, {
+            cwd,
+            env,
+            timeoutMs: timeoutMs(setup),
+          });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const failure: HostEnvironmentSetupResult = {
+      type: 'host',
+      workdir,
+      status: 'failed',
+      command: setup.command,
+      cwd,
+      ...(setup.args !== undefined ? { args: setup.args } : {}),
+      stderr: message,
+      exitCode: 1,
+    };
+    throw new HostEnvironmentSetupError(formatSetupFailure(failure), failure);
+  }
+
+  const setupResult: HostEnvironmentSetupResult = {
+    type: 'host',
+    workdir,
+    status: result.exitCode === 0 ? 'success' : 'failed',
+    command: setup.command,
+    cwd,
+    ...(setup.args !== undefined ? { args: setup.args } : {}),
+    stdout: result.stdout,
+    stderr: result.stderr,
+    exitCode: result.exitCode,
+  };
+
+  if (result.exitCode !== 0) {
+    throw new HostEnvironmentSetupError(formatSetupFailure(setupResult), setupResult);
+  }
+
+  return setupResult;
+}

packages/core/src/evaluation/graders/script-grader.ts

@@ -310,6 +310,8 @@ export class ScriptGrader implements Grader {
       getProxyUsage = proxy.getUsageMetadata;
     }
 
+    const effectiveCwd = this.cwd ?? context.workspacePath;
+
     // Build workspace env if workspace path is available
     const workspaceEnv = context.workspacePath
       ? { AGENTV_WORKSPACE_PATH: context.workspacePath }
@@ -339,7 +341,7 @@ export class ScriptGrader implements Grader {
           this.command,
           inputPayload,
           this.agentTimeoutMs,
-          this.cwd,
+          effectiveCwd,
           env,
         );
         exitCode = result.exitCode;
@@ -385,7 +387,7 @@ export class ScriptGrader implements Grader {
       const proxyUsage = getProxyUsage?.();
       const graderRawRequest: JsonObject = {
         command: this.command,
-        ...(this.cwd ? { cwd: this.cwd } : {}),
+        ...(effectiveCwd ? { cwd: effectiveCwd } : {}),
         ...(proxyUsage
           ? {
               target_proxy: {
@@ -426,7 +428,7 @@ export class ScriptGrader implements Grader {
         expectedAspectCount: 1,
         graderRawRequest: {
           command: this.command,
-          ...(this.cwd ? { cwd: this.cwd } : {}),
+          ...(effectiveCwd ? { cwd: effectiveCwd } : {}),
           ...(proxyUsage
             ? {
                 target_proxy: {

packages/core/src/evaluation/loaders/environment-recipe.ts

@@ -15,13 +15,17 @@ export type EnvironmentSetupConfig = {
   readonly timeout_seconds?: number;
 };
 
+type EnvironmentRecipeSource = {
+  readonly recipeFilePath?: string;
+  readonly sourceDir: string;
+};
+
 export type HostEnvironmentRecipe = {
   readonly type: 'host';
   readonly workdir: string;
   readonly setup?: EnvironmentSetupConfig;
   readonly env?: Readonly<Record<string, string>>;
-  readonly recipeFilePath?: string;
-};
+} & EnvironmentRecipeSource;
 
 export type DockerEnvironmentMount = {
   readonly source: string;
@@ -48,8 +52,7 @@ export type DockerEnvironmentRecipe = {
   readonly mounts?: readonly DockerEnvironmentMount[];
   readonly secrets?: Readonly<Record<string, string>>;
   readonly setup?: EnvironmentSetupConfig;
-  readonly recipeFilePath?: string;
-};
+} & EnvironmentRecipeSource;
 
 export type EnvironmentRecipe = HostEnvironmentRecipe | DockerEnvironmentRecipe;
 
@@ -116,6 +119,7 @@ function parseEnvironmentRecipe(
     return {
       type,
       workdir: resolveHostPath(workdir, baseDir),
+      sourceDir: baseDir,
       ...(setup !== undefined && { setup }),
       ...(env !== undefined && { env }),
       ...(recipeFilePath !== undefined && { recipeFilePath }),
@@ -144,6 +148,7 @@ function parseEnvironmentRecipe(
   return {
     type,
     workdir,
+    sourceDir: baseDir,
     ...(context !== undefined && { context: resolveHostPath(context, baseDir) }),
     ...(dockerfile !== undefined && { dockerfile: resolveHostPath(dockerfile, baseDir) }),
     ...(image !== undefined && { image }),

packages/core/src/evaluation/orchestrator.ts

@@ -910,10 +910,12 @@ export async function runEvaluation(
   const requiresWorkspaceDispatch =
     workspacePath !== undefined ||
     legacyWorkspacePath !== undefined ||
-    filteredEvalCases.some((evalCase) => evalCase.workspace !== undefined);
+    filteredEvalCases.some(
+      (evalCase) => evalCase.workspace !== undefined || evalCase.environment !== undefined,
+    );
   if (providerSupportsBatch && requiresWorkspaceDispatch) {
     if (verbose) {
-      console.warn('Warning: Batch mode is disabled for workspace-enabled evals.');
+      console.warn('Warning: Batch mode is disabled for workspace- or environment-enabled evals.');
     }
     providerSupportsBatch = false;
     batchingDisabledByRuntimePolicy = true;