feat(eval): add environment recipe schema loader (av-noh3.2.2)

apps/cli/src/commands/eval/task-bundle.ts

@@ -3,6 +3,7 @@ import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import {
+  type EnvironmentRecipe,
   type EvalSourceReference,
   type EvalTest,
   type JsonObject,
@@ -116,6 +117,8 @@ export interface MaterializedEvalBundlePaths {
 type BundleReferenceKind =
   | EvalSourceReference['kind']
   | 'expected_output_file'
+  | 'environment_workdir'
+  | 'environment_setup_command'
   | 'workspace_template'
   | 'workspace_hook_command';
 
@@ -188,7 +191,13 @@ function referenceBucket(
   if (reference.kind === 'workspace_template') {
     return BUNDLE_WORKSPACES_DIRNAME;
   }
-  if (reference.kind === 'workspace_hook_command') {
+  if (reference.kind === 'environment_workdir') {
+    return BUNDLE_WORKSPACES_DIRNAME;
+  }
+  if (
+    reference.kind === 'workspace_hook_command' ||
+    reference.kind === 'environment_setup_command'
+  ) {
     return BUNDLE_SCRIPTS_DIRNAME;
   }
   return TASK_GRADERS_DIRNAME;
@@ -798,6 +807,14 @@ function serializeWorkspace(
   return rewritePathsDeep(portableWorkspace, rewrites) as Record<string, unknown>;
 }
 
+function serializeEnvironment(
+  environment: EnvironmentRecipe,
+  rewrites: ReadonlyMap<string, string>,
+): Record<string, unknown> {
+  const { recipeFilePath: _recipeFilePath, ...portableEnvironment } = environment;
+  return rewritePathsDeep(portableEnvironment, rewrites) as Record<string, unknown>;
+}
+
 function buildPortableEvalCase(
   test: EvalTest,
   rewrites: ReadonlyMap<string, string>,
@@ -820,6 +837,9 @@ function buildPortableEvalCase(
   if (test.workspace) {
     testCase.workspace = serializeWorkspace(test.workspace, rewrites);
   }
+  if (test.environment) {
+    testCase.environment = serializeEnvironment(test.environment, rewrites);
+  }
   if (test.metadata && Object.keys(test.metadata).length > 0) {
     testCase.metadata = rewritePathsDeep(test.metadata, rewrites);
   }
@@ -894,6 +914,58 @@ async function maybeWorkspaceHookCommandReference(options: {
   };
 }
 
+function environmentBaseDir(environment: EnvironmentRecipe, evalFileDir: string): string {
+  return environment.recipeFilePath ? path.dirname(environment.recipeFilePath) : evalFileDir;
+}
+
+async function collectEnvironmentReferences(
+  tests: readonly EvalTest[],
+  evalFileDir: string,
+): Promise<readonly BundleSourceReference[]> {
+  const references: BundleSourceReference[] = [];
+
+  for (const test of tests) {
+    const environment = test.environment;
+    if (!environment) {
+      continue;
+    }
+
+    if (environment.type === 'host') {
+      references.push({
+        kind: 'environment_workdir',
+        displayPath: environment.workdir,
+        resolvedPath: environment.workdir,
+        location: `environment.workdir for test "${test.id}"`,
+      });
+    }
+
+    const command = environment.setup?.command;
+    if (!command) {
+      continue;
+    }
+
+    const baseDir = environmentBaseDir(environment, evalFileDir);
+    const commandArgs = Array.isArray(command) ? command : [command];
+    for (const arg of commandArgs) {
+      const reference = await maybeWorkspaceHookCommandReference({
+        arg,
+        baseDir,
+        testId: test.id,
+        hookName: 'setup',
+      });
+      if (reference) {
+        references.push({
+          ...reference,
+          kind: 'environment_setup_command',
+          location: `environment.setup.command for test "${test.id}"`,
+        });
+      }
+    }
+  }
+
+  return references;
+}
+
 async function collectWorkspaceReferences(
   tests: readonly EvalTest[],
   evalFileDir: string,
@@ -1104,9 +1176,11 @@ export async function materializeEvalBundle(
 
   const evalFileDir = path.dirname(path.resolve(options.evalFilePath));
   const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir);
+  const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir);
   const references: BundleSourceReference[] = [
     ...options.tests.flatMap((test) => test.source?.references ?? []),
     ...collectExpectedOutputReferences(options.tests),
+    ...environmentReferences,
     ...workspaceReferences.references,
   ];
 

apps/cli/test/commands/eval/bundle.test.ts

@@ -93,19 +93,19 @@ await Bun.write(\`\${payload.workspace_path}/hook-ran.txt\`, 'ok\\n');
     const evalPath = path.join(sourceDir, 'evals', 'demo.eval.yaml');
     const sourceEvalBefore = `name: portable-demo
 target: inherited
-workspace:
-  template: ../workspace-template
-  hooks:
-    before_each:
-      command: ["bun", "../scripts/setup.ts"]
+environment:
+  type: host
+  workdir: ../workspace-template
+  setup:
+    command: ["bun", "../scripts/setup.ts"]
 tests: ../data/cases.yaml
 `;
     await writeFile(evalPath, sourceEvalBefore, 'utf8');
 
     return { sourceDir, bundleDir, evalPath, sourceEvalBefore };
   }
 
-  it('bundles inherited targets, relative data, workspace templates, and scripts into a runnable directory', async () => {
+  it('bundles inherited targets, relative data, environment workdirs, and scripts into a runnable directory', async () => {
     const { sourceDir, bundleDir, evalPath, sourceEvalBefore } =
       await createPortableSourceFixture();
 
@@ -145,9 +145,10 @@ tests: ../data/cases.yaml
     expect(bundledEval.execution).toBeUndefined();
     const [testCase] = bundledEval.tests as Record<string, unknown>[];
     expect(testCase.id).toBe('case-alpha');
-    expect(testCase.workspace).toMatchObject({
-      template: 'workspaces/workspace-template',
-      hooks: { before_each: { command: ['bun', 'scripts/scripts/setup.ts'] } },
+    expect(testCase.environment).toMatchObject({
+      type: 'host',
+      workdir: 'workspaces/workspace-template',
+      setup: { command: ['bun', 'scripts/scripts/setup.ts'] },
     });
     expect(bundledEval.prompts).toEqual(['{{ input }}']);
     const input = (testCase.vars as Record<string, unknown>).input as Array<{
@@ -212,7 +213,7 @@ tests:
     expect(bundledTargets).toContain('inline bundled response');
   }, 30_000);
 
-  it('reports unbundleable workspace references with their eval location', async () => {
+  it('reports unbundleable environment references with their eval location', async () => {
     const sourceDir = path.join(tempDir, 'missing-source');
     const bundleDir = path.join(tempDir, 'missing-bundle');
     await mkdir(path.join(sourceDir, '.agentv'), { recursive: true });
@@ -227,8 +228,9 @@ tests:
     );
     await writeFile(
       path.join(sourceDir, 'evals', 'missing-template.eval.yaml'),
-      `workspace:
-  template: ../does-not-exist
+      `environment:
+  type: host
+  workdir: ../does-not-exist
 prompts:
   - "{{ input }}"
 tests:
@@ -253,7 +255,7 @@ tests:
     expect(result.exitCode).toBe(1);
     const output = `${result.stdout}\n${result.stderr}`;
     expect(output).toContain('Cannot bundle eval');
-    expect(output).toContain('workspace.template for test "missing-template"');
+    expect(output).toContain('environment.workdir for test "missing-template"');
     expect(output).toContain('not found');
   }, 30_000);
 });

apps/cli/test/commands/grade/grade-prepared.test.ts

@@ -92,8 +92,9 @@ targets:
   await writeFile(
     evalPath,
     `
-workspace:
-  template: ../template
+environment:
+  type: host
+  workdir: ../template
 assert:
 ${assertionYaml
   .trim()

apps/cli/test/commands/prepare/prepare.test.ts

@@ -68,19 +68,9 @@ targets:
   );
   await writeFile(
     evalPath,
-    `workspace:
-  template: ../template
-  hooks:
-    before_all:
-      command:
-        - bun
-        - ../scripts/hook.ts
-        - workspace_before_all
-    before_each:
-      command:
-        - bun
-        - ../scripts/hook.ts
-        - workspace_before_each
+    `environment:
+  type: host
+  workdir: ../template
 target:
   extends: codex
   hooks:
@@ -163,22 +153,12 @@ describe('agentv prepare', () => {
     expect(await exists(promptPath)).toBe(true);
     expect(await exists(manifestPath)).toBe(true);
 
-    for (const step of [
-      'workspace_before_all',
-      'target_before_all',
-      'workspace_before_each',
-      'target_before_each',
-    ]) {
+    for (const step of ['target_before_all', 'target_before_each']) {
       expect(await exists(path.join(workspacePath, `${step}.txt`))).toBe(true);
     }
     expect(
       (await readFile(path.join(workspacePath, 'hook-order.txt'), 'utf8')).trim().split('\n'),
-    ).toEqual([
-      'workspace_before_all',
-      'target_before_all',
-      'workspace_before_each',
-      'target_before_each',
-    ]);
+    ).toEqual(['target_before_all', 'target_before_each']);
 
     expect(await exists(targetMarker)).toBe(false);
     expect(await exists(graderMarker)).toBe(false);
@@ -283,8 +263,9 @@ targets:
   - id: agentv:agent-rules
     hook: beforeAll
     rules: ../rules/AGENTS.md
-workspace:
-  template: ../template
+environment:
+  type: host
+  workdir: ../template
 prompts:
   - "{{ input }}"
 tests:

apps/cli/test/commands/workspace/deps.test.ts

@@ -13,7 +13,7 @@ const __dirname = path.dirname(__filename);
 const projectRoot = path.resolve(__dirname, '../../../../..');
 const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts');
 
-describe('workspace deps', () => {
+describe('workspace deps legacy compatibility command', () => {
   it('exits non-zero when an eval uses removed repo schema fields', async () => {
     const tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-workspace-deps-test-'));
     try {

examples/contract/evals/release-gate.eval.yaml

@@ -1,8 +1,8 @@
 name: release-contract
 description: Lightweight release gate for npm latest promotion.
-workspace:
-  scope: suite
-  template: ../workspace-template
+environment:
+  type: host
+  workdir: ../workspace-template
 target: github-models-contract
 prompts:
   - "{{ input }}"

examples/contract/evals/repo-materialization.eval.yaml

@@ -1,10 +1,13 @@
 name: repo-materialization-contract
 description: Release gate for public repo materialization, previous-commit checkout, and file input
   substitution.
-workspace:
-  scope: suite
-  repos:
-    - path: ./fixture
+environment:
+  type: host
+  workdir: ./fixture
+  setup:
+    command: ../scripts/materialize-repo.sh
+    args:
+      path: ./fixture
       repo: EntityProcess/agentv-contract-fixture
       commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb
 target: github-models-contract

examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml

@@ -3,9 +3,9 @@ tags:
   - skill-trigger
 extensions:
   - agentv:agent-rules
-workspace:
-  scope: suite
-  template: workspace/
+environment:
+  type: host
+  workdir: workspace/
 prompts:
   - "{{ input }}"
 tests:

examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml

@@ -1,8 +1,8 @@
 extensions:
   - agentv:agent-rules
-workspace:
-  scope: suite
-  template: workspace/
+environment:
+  type: host
+  workdir: workspace/
 prompts:
   - "{{ input }}"
 tests:

examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml

@@ -3,9 +3,9 @@ tags:
 target: copilot-transcript-replay
 extensions:
   - file://../scripts/copilot-replay-workspace.mjs:beforeAll
-workspace:
-  scope: suite
-  template: ../workspace/
+environment:
+  type: host
+  workdir: ../workspace/
 prompts:
   - "{{ input }}"
 tests:

examples/features/docker-workspace/README.md

@@ -9,7 +9,7 @@ When evaluating coding agents (e.g., SWE-bench), the grader script needs to:
 2. Run tests inside the repository's environment
 3. Report pass/fail results
 
-Docker workspaces let you run this grading process inside a pre-built container
+Docker environments let you run this grading process inside a pre-built container
 image that has the repository, dependencies, and test infrastructure ready.
 
 ## How It Works
@@ -29,28 +29,29 @@ image that has the repository, dependencies, and test infrastructure ready.
 ## YAML Schema
 
 ```yaml
-workspace:
-  scope: suite
-  docker:
-    image: swebench/sweb.eval.x86_64.django__django-15180
-    timeout: 1800    # seconds (default: 1800)
+environment:
+  type: docker
+  image: swebench/sweb.eval.x86_64.django__django-15180
+  workdir: /testbed
+  resources:
     memory: 4g       # optional Docker memory limit
     cpus: 2          # optional Docker CPU limit
 ```
 
-For evals that need a repo pinned to a dataset snapshot, use `workspace.repos[].commit`:
+For evals that need a repo pinned to a dataset snapshot, keep that metadata in `environment.setup.args`:
 
 ```yaml
-workspace:
-  scope: suite
-  docker:
-    image: swebench/sweb.eval.x86_64.django__django-15180
-  repos:
-    - path: /testbed
+environment:
+  type: docker
+  image: swebench/sweb.eval.x86_64.django__django-15180
+  workdir: /testbed
+  setup:
+    command: ./setup.sh
+    args:
       commit: abc123def
 ```
 
-Repos defined without `repo` are assumed to already exist inside the container (e.g., SWE-bench prebuilt images).
+Prebuilt images can carry the repository inside the container, while the setup args record the dataset snapshot being evaluated.
 
 ## Running