From a8a5625c2c69979d791f39fbe4b789d8e5c889bf Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 10:01:00 +0200 Subject: [PATCH 1/2] feat(eval): add environment recipe schema loader --- .../src/evaluation/loaders/config-graph.ts | 3 + .../evaluation/loaders/environment-recipe.ts | 323 +++++ .../core/src/evaluation/providers/targets.ts | 17 + packages/core/src/evaluation/types.ts | 3 + .../evaluation/validation/eval-file.schema.ts | 90 +- .../evaluation/validation/eval-validator.ts | 124 +- packages/core/src/evaluation/yaml-parser.ts | 127 +- .../evaluation/loaders/config-loader.test.ts | 14 + .../loaders/environment-recipe.test.ts | 241 ++++ .../references/eval.schema.json | 1244 ++++++++++++++--- 10 files changed, 1892 insertions(+), 294 deletions(-) create mode 100644 packages/core/src/evaluation/loaders/environment-recipe.ts create mode 100644 packages/core/test/evaluation/loaders/environment-recipe.test.ts diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index 6eec6f2fc..448fb875f 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -32,6 +32,9 @@ const REMOVED_TARGET_FIELDS = new Map([ ['binary', "put process argv under 'config.command'."], ['args', "put process argv under 'config.command'."], ['arguments', "put process argv under 'config.command'."], + ['environment', 'environment recipes belong at suite/test/case scope, not under targets.'], + ['container', 'container/testbed setup belongs in an environment recipe, not under targets.'], + ['install', 'install/setup steps belong in environment.setup, not under targets.'], ['grader_target', "grader selection belongs in 'defaults.grader' or evaluator config."], ['workers', "target-level 'workers' is not general run policy; use 'execution.max_concurrency'."], [ diff --git a/packages/core/src/evaluation/loaders/environment-recipe.ts b/packages/core/src/evaluation/loaders/environment-recipe.ts new file mode 100644 index 000000000..352e4238a --- /dev/null +++ b/packages/core/src/evaluation/loaders/environment-recipe.ts @@ -0,0 +1,323 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import type { JsonObject, JsonValue } from '../types.js'; +import { isJsonObject } from '../types.js'; +import { parseYamlValue } from '../yaml-loader.js'; + +const FILE_PROTOCOL = 'file://'; + +export type EnvironmentSetupConfig = { + readonly command: string | readonly string[]; + readonly args?: JsonObject; + readonly env?: Readonly>; + readonly timeout_seconds?: number; +}; + +export type HostEnvironmentRecipe = { + readonly type: 'host'; + readonly workdir: string; + readonly setup?: EnvironmentSetupConfig; + readonly env?: Readonly>; + readonly recipeFilePath?: string; +}; + +export type DockerEnvironmentMount = { + readonly source: string; + readonly target: string; + readonly access?: 'ro' | 'rw'; + readonly read_only?: boolean; +}; + +export type DockerEnvironmentResources = { + readonly cpus?: number; + readonly memory?: string; + readonly disk?: string; + readonly gpu?: boolean | string; +}; + +export type DockerEnvironmentRecipe = { + readonly type: 'docker'; + readonly workdir: string; + readonly context?: string; + readonly dockerfile?: string; + readonly image?: string; + readonly env?: Readonly>; + readonly resources?: DockerEnvironmentResources; + readonly mounts?: readonly DockerEnvironmentMount[]; + readonly secrets?: Readonly>; + readonly setup?: EnvironmentSetupConfig; + readonly recipeFilePath?: string; +}; + +export type EnvironmentRecipe = HostEnvironmentRecipe | DockerEnvironmentRecipe; + +export async function resolveEnvironmentRecipe( + raw: unknown, + evalFileDir: string, + location = 'environment', +): Promise { + if (raw === undefined) { + return undefined; + } + + if (typeof raw === 'string') { + if (!raw.startsWith(FILE_PROTOCOL)) { + throw new Error( + `${location} must be an inline recipe object or a file:// reference to a recipe file.`, + ); + } + const recipePath = resolveReferencePath(raw, evalFileDir); + let parsed: unknown; + try { + parsed = parseYamlValue(await readFile(recipePath, 'utf8')); + } catch (error) { + throw new Error( + `${location} recipe file not found or unreadable: ${raw} (${(error as Error).message})`, + ); + } + if (isJsonObject(parsed) && Object.prototype.hasOwnProperty.call(parsed, 'environment')) { + throw new Error( + `${location} recipe file ${recipePath} must contain the environment recipe directly, not an object wrapped in 'environment'.`, + ); + } + return parseEnvironmentRecipe(parsed, path.dirname(recipePath), location, recipePath); + } + + return parseEnvironmentRecipe(raw, evalFileDir, location); +} + +function resolveReferencePath(reference: string, evalFileDir: string): string { + const filePath = reference.slice(FILE_PROTOCOL.length); + return path.isAbsolute(filePath) ? filePath : path.resolve(evalFileDir, filePath); +} + +function parseEnvironmentRecipe( + raw: unknown, + baseDir: string, + location: string, + recipeFilePath?: string, +): EnvironmentRecipe { + if (!isJsonObject(raw)) { + throw new Error(`${location} must be an object with type: host|docker and workdir.`); + } + const type = raw.type; + if (type !== 'host' && type !== 'docker') { + throw new Error(`${location}.type must be 'host' or 'docker'.`); + } + + const workdir = readRequiredString(raw.workdir, `${location}.workdir`); + const setup = parseSetup(raw.setup, `${location}.setup`); + const env = parseStringRecord(raw.env, `${location}.env`); + + if (type === 'host') { + assertNoUnknownFields(raw, location, ['type', 'workdir', 'setup', 'env']); + return { + type, + workdir: resolveHostPath(workdir, baseDir), + ...(setup !== undefined && { setup }), + ...(env !== undefined && { env }), + ...(recipeFilePath !== undefined && { recipeFilePath }), + }; + } + + assertNoUnknownFields(raw, location, [ + 'type', + 'workdir', + 'context', + 'dockerfile', + 'image', + 'env', + 'resources', + 'mounts', + 'secrets', + 'setup', + ]); + const context = readOptionalString(raw.context, `${location}.context`); + const dockerfile = readOptionalString(raw.dockerfile, `${location}.dockerfile`); + const image = readOptionalString(raw.image, `${location}.image`); + const secrets = parseStringRecord(raw.secrets, `${location}.secrets`); + if (!context && !image) { + throw new Error(`${location} docker recipes must define either 'image' or 'context'.`); + } + return { + type, + workdir, + ...(context !== undefined && { context: resolveHostPath(context, baseDir) }), + ...(dockerfile !== undefined && { dockerfile: resolveHostPath(dockerfile, baseDir) }), + ...(image !== undefined && { image }), + ...(env !== undefined && { env }), + ...(parseResources(raw.resources, `${location}.resources`) ?? {}), + ...(parseMounts(raw.mounts, `${location}.mounts`, baseDir) ?? {}), + ...(secrets !== undefined && { secrets }), + ...(setup !== undefined && { setup }), + ...(recipeFilePath !== undefined && { recipeFilePath }), + }; +} + +function resolveHostPath(value: string, baseDir: string): string { + return path.isAbsolute(value) ? value : path.resolve(baseDir, value); +} + +function parseSetup( + raw: JsonValue | undefined, + location: string, +): EnvironmentSetupConfig | undefined { + if (raw === undefined) { + return undefined; + } + if (!isJsonObject(raw)) { + throw new Error(`${location} must be an object.`); + } + assertNoUnknownFields(raw, location, ['command', 'args', 'env', 'timeout_seconds']); + const command = raw.command; + if ( + !( + typeof command === 'string' || + (Array.isArray(command) && + command.length > 0 && + command.every((entry) => typeof entry === 'string' && entry.trim().length > 0)) + ) + ) { + throw new Error(`${location}.command must be a non-empty string or argv array.`); + } + const args = raw.args; + if (args !== undefined && !isJsonObject(args)) { + throw new Error(`${location}.args must be an object when provided.`); + } + const timeoutSeconds = raw.timeout_seconds; + if (timeoutSeconds !== undefined && (typeof timeoutSeconds !== 'number' || timeoutSeconds <= 0)) { + throw new Error(`${location}.timeout_seconds must be a positive number.`); + } + const env = parseStringRecord(raw.env, `${location}.env`); + return { + command, + ...(isJsonObject(args) ? { args } : {}), + ...(env !== undefined && { env }), + ...(typeof timeoutSeconds === 'number' ? { timeout_seconds: timeoutSeconds } : {}), + }; +} + +function parseResources( + raw: JsonValue | undefined, + location: string, +): { readonly resources?: DockerEnvironmentResources } | undefined { + if (raw === undefined) { + return undefined; + } + if (!isJsonObject(raw)) { + throw new Error(`${location} must be an object.`); + } + assertNoUnknownFields(raw, location, ['cpus', 'memory', 'disk', 'gpu']); + const resources: DockerEnvironmentResources = { + ...(typeof raw.cpus === 'number' && raw.cpus > 0 ? { cpus: raw.cpus } : {}), + ...(typeof raw.memory === 'string' && raw.memory.trim().length > 0 + ? { memory: raw.memory.trim() } + : {}), + ...(typeof raw.disk === 'string' && raw.disk.trim().length > 0 + ? { disk: raw.disk.trim() } + : {}), + ...(typeof raw.gpu === 'boolean' || typeof raw.gpu === 'string' ? { gpu: raw.gpu } : {}), + }; + if (raw.cpus !== undefined && resources.cpus === undefined) { + throw new Error(`${location}.cpus must be a positive number.`); + } + if (raw.memory !== undefined && resources.memory === undefined) { + throw new Error(`${location}.memory must be a non-empty string.`); + } + if (raw.disk !== undefined && resources.disk === undefined) { + throw new Error(`${location}.disk must be a non-empty string.`); + } + if (raw.gpu !== undefined && resources.gpu === undefined) { + throw new Error(`${location}.gpu must be a boolean or string.`); + } + return Object.keys(resources).length > 0 ? { resources } : undefined; +} + +function parseMounts( + raw: JsonValue | undefined, + location: string, + baseDir: string, +): { readonly mounts?: readonly DockerEnvironmentMount[] } | undefined { + if (raw === undefined) { + return undefined; + } + if (!Array.isArray(raw)) { + throw new Error(`${location} must be an array.`); + } + const mounts = raw.map((entry, index) => { + const entryLocation = `${location}[${index}]`; + if (!isJsonObject(entry)) { + throw new Error(`${entryLocation} must be an object.`); + } + assertNoUnknownFields(entry, entryLocation, ['source', 'target', 'access', 'read_only']); + const source = readRequiredString(entry.source, `${entryLocation}.source`); + const target = readRequiredString(entry.target, `${entryLocation}.target`); + const access = entry.access; + if (access !== undefined && access !== 'ro' && access !== 'rw') { + throw new Error(`${entryLocation}.access must be 'ro' or 'rw'.`); + } + const normalizedAccess: DockerEnvironmentMount['access'] = + access === 'ro' || access === 'rw' ? access : undefined; + if (entry.read_only !== undefined && typeof entry.read_only !== 'boolean') { + throw new Error(`${entryLocation}.read_only must be a boolean.`); + } + return { + source: resolveHostPath(source, baseDir), + target, + ...(normalizedAccess !== undefined && { access: normalizedAccess }), + ...(typeof entry.read_only === 'boolean' && { read_only: entry.read_only }), + }; + }); + return { mounts }; +} + +function parseStringRecord( + raw: JsonValue | undefined, + location: string, +): Readonly> | undefined { + if (raw === undefined) { + return undefined; + } + if (!isJsonObject(raw)) { + throw new Error(`${location} must be an object of string values.`); + } + const result: Record = {}; + for (const [key, value] of Object.entries(raw)) { + if (typeof value !== 'string') { + throw new Error(`${location}.${key} must be a string.`); + } + result[key] = value; + } + return result; +} + +function readRequiredString(value: JsonValue | undefined, location: string): string { + const result = readOptionalString(value, location); + if (result === undefined) { + throw new Error(`${location} must be a non-empty string.`); + } + return result; +} + +function readOptionalString(value: JsonValue | undefined, location: string): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${location} must be a non-empty string.`); + } + return value.trim(); +} + +function assertNoUnknownFields( + raw: JsonObject, + location: string, + allowed: readonly string[], +): void { + const allowedFields = new Set(allowed); + const unknown = Object.keys(raw).find((key) => !allowedFields.has(key)); + if (unknown) { + throw new Error(`${location}.${unknown} is not supported in environment recipes.`); + } +} diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 98b5aadcd..c7a9b88aa 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -702,6 +702,7 @@ export function normalizeTargetDefinition( if (!isRecord(definition)) { throw new Error('Target definition must be an object'); } + assertNoTargetTestbedFields(definition); const rawId = definition.id; const rawLabel = definition.label; @@ -726,6 +727,22 @@ export function normalizeTargetDefinition( } as unknown as TargetDefinition; } +function assertNoTargetTestbedFields(definition: Record): void { + if (definition.environment !== undefined) { + throw new Error( + 'Target definitions cannot include environment; author environment at suite/test/case scope.', + ); + } + if (definition.container !== undefined) { + throw new Error( + 'Target definitions cannot include container setup; use an environment recipe.', + ); + } + if (definition.install !== undefined) { + throw new Error('Target definitions cannot include install steps; use environment.setup.'); + } +} + function collectDeprecatedCamelCaseWarnings( value: unknown, location: string, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index c9e34d24d..65593ad0c 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1,3 +1,4 @@ +import type { EnvironmentRecipe } from './loaders/environment-recipe.js'; import type { TransformSpec } from './output-transform.js'; import type { TargetExecutionEnvelope } from './providers/types.js'; import type { TokenUsage, ToolTrajectoryGraderConfig, Trace } from './trace.js'; @@ -1077,6 +1078,8 @@ export interface EvalTest { readonly preprocessors?: readonly ContentPreprocessorConfig[]; /** Promptfoo-style lifecycle extensions inherited from the suite. */ readonly extensions?: readonly AgentVExtensionConfig[]; + /** AgentV-authored host/Docker testbed recipe inherited from suite or authored per case. */ + readonly environment?: EnvironmentRecipe; /** Workspace configuration (merged from suite-level and case-level) */ readonly workspace?: WorkspaceConfig; /** Arbitrary metadata passed to workspace scripts via stdin */ diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 55b29adda..1d93239c0 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -387,16 +387,6 @@ const ExtensionSchema = z.union([ // Repo lifecycle // --------------------------------------------------------------------------- -const RepoSchema = z - .object({ - path: z.string().optional(), - repo: z.string().min(1).optional(), - commit: z.string().min(1).optional(), - ancestor: z.number().int().min(0).optional(), - sparse: z.array(z.string()).optional(), - }) - .strict(); - const WorkspaceHookSchema = z .object({ command: z.union([z.string(), z.array(z.string())]).optional(), @@ -417,13 +407,6 @@ const WorkspaceHooksSchema = z }) .strict(); -const DockerWorkspaceSchema = z.object({ - image: z.string(), - timeout: z.number().int().min(1).optional(), - memory: z.string().optional(), - cpus: z.number().min(0.1).optional(), -}); - const WorkspaceEnvSchema = z .object({ required_commands: z.array(z.string().min(1)).optional(), @@ -433,15 +416,72 @@ const WorkspaceEnvSchema = z const WorkspaceSchema = z .object({ - template: z.string().optional(), - scope: z.enum(['suite', 'attempt']).optional(), - repos: z.array(RepoSchema).optional(), + template: z.never().optional(), + scope: z.never().optional(), + repos: z.never().optional(), hooks: WorkspaceHooksSchema.optional(), - docker: DockerWorkspaceSchema.optional(), + docker: z.never().optional(), env: WorkspaceEnvSchema.optional(), }) .strict(); +const EnvironmentSetupSchema = z + .object({ + command: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]), + args: JsonObjectSchema.optional(), + env: z.record(z.string()).optional(), + timeout_seconds: z.number().gt(0).optional(), + }) + .strict(); + +const EnvironmentBaseSchema = z.object({ + workdir: z.string().min(1), + setup: EnvironmentSetupSchema.optional(), + env: z.record(z.string()).optional(), +}); + +const HostEnvironmentSchema = EnvironmentBaseSchema.extend({ + type: z.literal('host'), +}).strict(); + +const DockerEnvironmentMountSchema = z + .object({ + source: z.string().min(1), + target: z.string().min(1), + access: z.enum(['ro', 'rw']).optional(), + read_only: z.boolean().optional(), + }) + .strict(); + +const DockerEnvironmentResourcesSchema = z + .object({ + cpus: z.number().gt(0).optional(), + memory: z.string().min(1).optional(), + disk: z.string().min(1).optional(), + gpu: z.union([z.boolean(), z.string().min(1)]).optional(), + }) + .strict(); + +const DockerEnvironmentSchema = EnvironmentBaseSchema.extend({ + type: z.literal('docker'), + context: z.string().min(1).optional(), + dockerfile: z.string().min(1).optional(), + image: z.string().min(1).optional(), + resources: DockerEnvironmentResourcesSchema.optional(), + mounts: z.array(DockerEnvironmentMountSchema).optional(), + secrets: z.record(z.string()).optional(), +}) + .strict() + .refine((value) => value.context !== undefined || value.image !== undefined, { + message: "Docker environment recipes must define either 'image' or 'context'.", + }); + +const EnvironmentSchema = z.union([ + z.string().regex(/^\s*file:\/\//, 'environment string must start with file://'), + HostEnvironmentSchema, + DockerEnvironmentSchema, +]); + // --------------------------------------------------------------------------- // Target hooks (eval-level per-target customization) // --------------------------------------------------------------------------- @@ -475,6 +515,9 @@ const EvalLocalTargetSchema = z transform: z.union([z.string(), JsonObjectSchema]).optional(), delay: z.number().min(0).optional(), env: z.record(z.string()).optional(), + environment: z.never().optional(), + container: z.never().optional(), + install: z.never().optional(), reasoning_effort: z.string().min(1).optional(), hooks: TargetHooksSchema.optional(), }) @@ -589,6 +632,7 @@ const EvalTestSchema = z.object({ threshold: z.number().min(0).max(1).optional(), execution: TestExecutionSchema.optional(), run: RunOverrideSchema.optional(), + environment: EnvironmentSchema.optional(), workspace: WorkspaceSchema.optional(), metadata: z.record(z.unknown()).optional(), conversation_id: z.string().optional(), @@ -667,6 +711,9 @@ const ConfigTargetSchema = z provider: z.string().min(1), runtime: ConfigRuntimeSchema, config: JsonRecordSchema.optional(), + environment: z.never().optional(), + container: z.never().optional(), + install: z.never().optional(), }) .strict(); @@ -768,6 +815,7 @@ export const EvalFileSchema: z.ZodType = z budget_usd: z.never().optional(), threshold: z.number().min(0).max(1).optional(), default_test: z.union([DefaultTestReferenceSchema, DefaultTestSchema]).optional(), + environment: EnvironmentSchema.optional(), scenarios: z.array(ScenarioSchema).optional(), derived_metrics: z.array(DerivedMetricSchema).optional(), output_path: z.union([z.string().min(1), z.array(z.string().min(1))]).optional(), diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 8a1708d95..364cb839c 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -4,6 +4,7 @@ import fg from 'fast-glob'; import { interpolateEnv } from '../interpolation.js'; import { loadCasesFromDirectory, loadCasesFromFile } from '../loaders/case-file-loader.js'; +import { resolveEnvironmentRecipe } from '../loaders/environment-recipe.js'; import { buildSearchRoots } from '../loaders/file-resolver.js'; import { loadPromptMdFallback } from '../loaders/prompt-md-fallback.js'; import { isGraderKind } from '../types.js'; @@ -150,6 +151,7 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'tests', 'graders', 'defaults', + 'environment', 'target', 'targets', 'model', @@ -283,6 +285,7 @@ const KNOWN_TEST_FIELDS = new Set([ 'threshold', 'execution', 'run', + 'environment', 'workspace', 'metadata', 'conversation_id', @@ -472,7 +475,10 @@ export async function validateEvalFile(filePath: string): Promise { + if (environment === undefined) { + return; + } + try { + await resolveEnvironmentRecipe(environment, path.dirname(evalFilePath), location); + } catch (error) { + errors.push({ + severity: 'error', + filePath: evalFilePath, + location, + message: (error as Error).message, + }); + } +} + +function validateTargetTestbedFields( + target: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (!isObject(target)) { + return; + } + for (const field of ['environment', 'container', 'install'] as const) { + if (target[field] === undefined) { + continue; + } + errors.push({ + severity: 'error', + filePath, + location: `${location}.${field}`, + message: + field === 'environment' + ? 'Target definitions cannot include environment; author environment at suite/test/case scope.' + : `Target definitions cannot include ${field}; use an environment recipe for testbed setup.`, + }); + } +} + +function validateTargetsTestbedFields( + targets: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + const entries = Array.isArray(targets) ? targets : targets === undefined ? [] : [targets]; + entries.forEach((entry, index) => { + validateTargetTestbedFields(entry, `${location}[${index}]`, filePath, errors); + }); +} + function validateTestExecutionFields( caseExecution: JsonObject, filePath: string, @@ -1029,7 +1099,9 @@ async function validateCompositionDiagnostics( filePath, location, message: - 'Parent workspace is not allowed when an eval imports suites with type: suite. A wrapper eval owns target and run controls, while imported suites own task environment. Move workspace into the child suite, or import raw cases with type: tests when you intentionally want parent workspace context.', + location === 'environment' + ? 'Parent environment is not allowed when an eval imports suites with type: suite. Imported suites own task environment. Move environment into the child suite, or import raw cases with type: tests when you intentionally want parent environment context.' + : 'Parent workspace is not allowed when an eval imports suites with type: suite. A wrapper eval owns target and run controls, while imported suites own task environment. Move workspace into the child suite, or import raw cases with type: tests when you intentionally want parent workspace context.', }); } } @@ -1069,7 +1141,7 @@ async function validateCompositionDiagnostics( severity: 'warning', filePath, location: entry.location, - message: `imports.tests imports raw cases from eval suite '${resolvedSuite.displayPath}' and drops suite context, including child workspace, input, assertions, metadata, target, and run controls. Parent suite context applies. Use imports.suites to preserve child test and workspace semantics.`, + message: `imports.tests imports raw cases from eval suite '${resolvedSuite.displayPath}' and drops suite context, including child environment, workspace, input, assertions, metadata, target, and run controls. Parent suite context applies. Use imports.suites to preserve child test and environment semantics.`, }); } } @@ -1112,6 +1184,9 @@ function parentWorkspaceLocations(parsed: JsonObject): readonly string[] { if (parsed.workspace !== undefined) { locations.push('workspace'); } + if (parsed.environment !== undefined) { + locations.push('environment'); + } if (isObject(parsed.execution) && parsed.execution.workspace !== undefined) { locations.push('execution.workspace'); } @@ -1711,6 +1786,51 @@ function validateWorkspaceRepoConfig( const scope = workspace.scope; const docker = workspace.docker; + let hasRemovedTestbedField = false; + + if ('template' in workspace) { + hasRemovedTestbedField = true; + errors.push({ + severity: 'error', + filePath, + location: `${location}.template`, + message: + 'workspace.template has been removed from public eval YAML. Use environment.workdir and environment.setup for authored testbed setup.', + }); + } + if ('repos' in workspace) { + hasRemovedTestbedField = true; + errors.push({ + severity: 'error', + filePath, + location: `${location}.repos`, + message: + 'workspace.repos has been removed from public eval YAML. Use an environment recipe with setup args to materialize repositories.', + }); + } + if ('scope' in workspace) { + hasRemovedTestbedField = true; + errors.push({ + severity: 'error', + filePath, + location: `${location}.scope`, + message: + 'workspace.scope has been removed from public eval YAML. Use environment at suite/test scope and let runtime manage workspace lifetime.', + }); + } + if ('docker' in workspace) { + hasRemovedTestbedField = true; + errors.push({ + severity: 'error', + filePath, + location: `${location}.docker`, + message: + 'workspace.docker has been removed from public eval YAML. Use environment.type: docker with image or context.', + }); + } + if (hasRemovedTestbedField) { + return; + } if ('mode' in workspace) { errors.push({ diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 38b8e976d..39fd8dd79 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -40,6 +40,7 @@ import { loadConfig, parseTargetHooks, } from './loaders/config-loader.js'; +import { resolveEnvironmentRecipe } from './loaders/environment-recipe.js'; import { buildSearchRoots, resolveFileReference, @@ -70,7 +71,6 @@ import type { ConversationAggregation, ConversationMode, ConversationTurn, - DockerWorkspaceConfig, EvalGraderSource, EvalPromptIdentity, EvalRunOverride, @@ -81,7 +81,6 @@ import type { GraderConfig, JsonObject, JsonValue, - RepoConfig, TargetHooksConfig, TestMessage, TestMessageContent, @@ -93,7 +92,6 @@ import type { WorkspaceScriptConfig, } from './types.js'; import { isJsonObject, isTestMessage } from './types.js'; -import { parseRepoConfig } from './workspace/repo-config-parser.js'; import { parseYamlValue } from './yaml-loader.js'; // Re-export public APIs from modules @@ -213,6 +211,7 @@ type RawTestSuite = JsonObject & { readonly budget_usd?: JsonValue; readonly threshold?: JsonValue; readonly default_test?: JsonValue; + readonly environment?: JsonValue; readonly workspace?: JsonValue; readonly assert?: JsonValue; readonly preprocessors?: JsonValue; @@ -250,6 +249,7 @@ type RawEvalCase = JsonObject & { readonly execution?: JsonValue; readonly run?: JsonValue; readonly assert?: JsonValue; + readonly environment?: JsonValue; readonly workspace?: JsonValue; readonly metadata?: JsonValue; readonly depends_on?: JsonValue; @@ -1247,6 +1247,7 @@ async function loadTestsFromParsedYamlValue( } rejectAuthoredWorkers(interpolated); rejectAuthoredDirectInput(interpolated); + rejectTargetTestbedFields(interpolated); if (options?.allowInternalExpectedOutput !== true) { rejectAuthoredExpectedOutput(interpolated); } @@ -1284,10 +1285,16 @@ async function loadTestsFromParsedYamlValue( const globalEvaluator = coerceEvaluator(suite.evaluator, 'global'); const defaultTestRubricPrompt = extractDefaultTestRubricPrompt(suite); const suiteExtensions = parseExtensions(suite.extensions, evalFileDir); + const suiteEnvironment = await resolveEnvironmentRecipe( + suite.environment, + evalFileDir, + 'environment', + ); const importedSuiteTests: EvalTest[] = []; const nunjucksFilters = await loadNunjucksFilters(suite.nunjucks_filters, evalFileDir); const parentWorkspace = parentWorkspaceLocation(suite); + const parentEnvironment = parentEnvironmentLocation(suite); const importEntries = readImports(suite.imports); const expandedImports = await expandImportEntries({ entries: importEntries, @@ -1295,6 +1302,7 @@ async function loadTestsFromParsedYamlValue( repoRoot, suiteMetadataPayload, parentWorkspaceLocation: parentWorkspace, + parentEnvironmentLocation: parentEnvironment, options, }); importedSuiteTests.push(...expandedImports.importedSuiteTests); @@ -1313,6 +1321,7 @@ async function loadTestsFromParsedYamlValue( repoRoot, suiteMetadataPayload, parentWorkspaceLocation: parentWorkspace, + parentEnvironmentLocation: parentEnvironment, options, }); expandedTestCases = [...expandedImports.rawCases, ...expanded.rawCases]; @@ -1576,6 +1585,12 @@ async function loadTestsFromParsedYamlValue( // Parse per-case workspace config and merge with suite-level const caseWorkspace = await resolveWorkspaceConfig(renderedCase.workspace, evalFileDir); const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace); + const caseEnvironment = await resolveEnvironmentRecipe( + renderedCase.environment, + evalFileDir, + `test '${id ?? 'unknown'}'.environment`, + ); + const environment = caseEnvironment ?? suiteEnvironment; // Parse per-case metadata, then merge suite-level metadata payload. // Arrays concatenate (suite-first, deduplicated), scalars on the case win. @@ -1649,6 +1664,7 @@ async function loadTestsFromParsedYamlValue( ...(caseVars ? { vars: caseVars } : {}), ...(outputTransform ? { outputTransform } : {}), ...(suiteExtensions.length > 0 ? { extensions: suiteExtensions } : {}), + ...(environment ? { environment } : {}), workspace: mergedWorkspace, metadata, ...(caseRun?.threshold !== undefined ? { threshold: caseRun.threshold } : {}), @@ -1803,6 +1819,38 @@ function rejectAuthoredDirectInput(parsed: JsonObject): void { } } +function rejectTargetTestbedFields(parsed: JsonObject): void { + rejectSingleTargetTestbedFields((parsed as RawTestSuite).target, 'target'); + const rawTargets = (parsed as RawTestSuite).targets; + const targets = Array.isArray(rawTargets) + ? rawTargets + : rawTargets === undefined + ? [] + : [rawTargets]; + targets.forEach((target, index) => { + rejectSingleTargetTestbedFields(target, `targets[${index}]`); + }); +} + +function rejectSingleTargetTestbedFields(rawTarget: JsonValue | undefined, location: string): void { + if (!isJsonObject(rawTarget)) { + return; + } + if (rawTarget.environment !== undefined) { + throw new Error( + `${location}.environment is not supported. Author environment at suite/test/case scope, not under targets.`, + ); + } + if (rawTarget.container !== undefined) { + throw new Error( + `${location}.container is not supported. Use an environment recipe for testbed setup.`, + ); + } + if (rawTarget.install !== undefined) { + throw new Error(`${location}.install is not supported. Use environment.setup.`); + } +} + function rejectAuthoredExpectedOutput(parsed: JsonObject): void { if (parsed.expected_output !== undefined) { throw new Error( @@ -2218,6 +2266,7 @@ async function expandImportEntries(params: { readonly repoRoot: URL | string; readonly suiteMetadataPayload?: Record; readonly parentWorkspaceLocation?: string; + readonly parentEnvironmentLocation?: string; readonly options?: LoadOptions; }): Promise { const rawCases: JsonValue[] = []; @@ -2233,6 +2282,11 @@ async function expandImportEntries(params: { `Parent workspace is not allowed when importing eval suites (${params.parentWorkspaceLocation}): ${entry.path}. Move workspace into the child suite, or import raw cases with imports.tests when you intentionally want parent workspace context.`, ); } + if (params.parentEnvironmentLocation) { + throw new Error( + `Parent environment is not allowed when importing eval suites (${params.parentEnvironmentLocation}): ${entry.path}. Imported suites own task environment. Move environment into the child suite, or import raw cases with imports.tests when you intentionally want parent environment context.`, + ); + } const suite = await loadTestSuite(resolvedPath, params.repoRoot, { ...params.options, filter: entry.select?.testIds, @@ -2333,6 +2387,7 @@ async function expandInlineTestEntries(params: { readonly repoRoot: URL | string; readonly suiteMetadataPayload?: Record; readonly parentWorkspaceLocation?: string; + readonly parentEnvironmentLocation?: string; readonly options?: LoadOptions; }): Promise { const withFileReferences = await expandFileReferences(params.entries, params.evalFileDir); @@ -2356,6 +2411,7 @@ async function expandInlineTestEntries(params: { repoRoot: params.repoRoot, suiteMetadataPayload: params.suiteMetadataPayload, parentWorkspaceLocation: params.parentWorkspaceLocation, + parentEnvironmentLocation: params.parentEnvironmentLocation, options: params.options, }); rawCases.push(...expanded.rawCases); @@ -2373,6 +2429,14 @@ function parentWorkspaceLocation(suite: RawTestSuite): string | undefined { return undefined; } +function parentEnvironmentLocation(suite: RawTestSuite): string | undefined { + if (suite.environment !== undefined) { + return 'environment'; + } + + return undefined; +} + function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonObject | undefined { if (suite.experiment !== undefined && typeof suite.experiment !== 'string') { throw new Error( @@ -3090,39 +3154,38 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi 'workspace.path has been removed from eval YAML. Put existing workspace paths in .agentv/config.local.yaml execution.workspace_path or pass --workspace-path.', ); } - - let template = typeof obj.template === 'string' ? obj.template : undefined; - if (template && !path.isAbsolute(template)) { - template = path.resolve(evalFileDir, template); + if ('template' in obj) { + throw new Error( + 'workspace.template has been removed from public eval YAML. Use environment.workdir and environment.setup for authored testbed setup.', + ); + } + if ('repos' in obj) { + throw new Error( + 'workspace.repos has been removed from public eval YAML. Use an environment recipe with setup args to materialize repositories.', + ); + } + if ('docker' in obj) { + throw new Error( + 'workspace.docker has been removed from public eval YAML. Use environment.type: docker with image or context.', + ); + } + if ('scope' in obj) { + throw new Error( + 'workspace.scope has been removed from public eval YAML. Use environment at suite/test scope and let runtime manage workspace lifetime.', + ); } if ('isolation' in obj) { throw new Error('workspace.isolation has been removed. Use workspace.scope: suite|attempt.'); } - if (obj.scope !== undefined && obj.scope !== 'suite' && obj.scope !== 'attempt') { - throw new Error("workspace.scope must be 'suite' or 'attempt'."); - } - const scope = obj.scope === 'suite' || obj.scope === 'attempt' ? obj.scope : undefined; - - const repos = Array.isArray(obj.repos) - ? ((obj.repos as Record[]) - .map(parseRepoConfig) - .filter(Boolean) as RepoConfig[]) - : undefined; const hooks = parseWorkspaceHooksConfig(obj.hooks, evalFileDir); - - const docker = parseDockerWorkspaceConfig(obj.docker); const env = parseWorkspaceEnvConfig(obj.env); - if (!template && !scope && !repos && !hooks && !docker && !env) return undefined; + if (!hooks && !env) return undefined; return { - ...(template !== undefined && { template }), - ...(scope !== undefined && { scope }), - ...(repos !== undefined && { repos }), ...(hooks !== undefined && { hooks }), - ...(docker !== undefined && { docker }), ...(env !== undefined && { env }), }; } @@ -3146,22 +3209,6 @@ function parseWorkspaceEnvConfig(raw: unknown): WorkspaceEnvConfig | undefined { }; } -/** - * Parse a DockerWorkspaceConfig from raw YAML value. - */ -function parseDockerWorkspaceConfig(raw: unknown): DockerWorkspaceConfig | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - if (typeof obj.image !== 'string') return undefined; - - return { - image: obj.image, - ...(typeof obj.timeout === 'number' && { timeout: obj.timeout }), - ...(typeof obj.memory === 'string' && { memory: obj.memory }), - ...(typeof obj.cpus === 'number' && { cpus: obj.cpus }), - }; -} - /** * Merge case-level workspace config with suite-level defaults. * Strategy: case-level fields replace suite-level fields. diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 351954bfb..41f5b2a21 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -328,6 +328,20 @@ describe('loadConfig', () => { ].join('\n'), message: /workers/, }, + { + name: 'target-environment', + yaml: [ + 'targets:', + ' - id: codex-local', + ' provider: codex-cli', + ' runtime: host', + ' environment:', + ' type: host', + ' workdir: ./workspace', + '', + ].join('\n'), + message: /environment recipes belong at suite\/test\/case scope/, + }, ]; for (const testCase of invalidCases) { diff --git a/packages/core/test/evaluation/loaders/environment-recipe.test.ts b/packages/core/test/evaluation/loaders/environment-recipe.test.ts new file mode 100644 index 000000000..fe48635da --- /dev/null +++ b/packages/core/test/evaluation/loaders/environment-recipe.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { validateEvalFile } from '../../../src/evaluation/validation/eval-validator.js'; +import { loadTestSuite, loadTests } from '../../../src/evaluation/yaml-parser.js'; + +function withTempDir(prefix: string, fn: (dir: string) => Promise): Promise { + const tempDir = mkdtempSync(path.join(os.tmpdir(), prefix)); + return fn(tempDir).finally(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); +} + +function writeEval(dir: string, body: string): string { + const evalPath = path.join(dir, 'suite.eval.yaml'); + writeFileSync(evalPath, body); + return evalPath; +} + +const baseCase = [ + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: case-1', + ' vars:', + ' input: Fix the bug', + ' assert:', + ' - type: contains', + ' value: fixed', + '', +].join('\n'); + +describe('environment recipe loading', () => { + it('accepts inline host environment recipes', async () => { + await withTempDir('agentv-env-host-', async (dir) => { + const evalPath = writeEval( + dir, + [ + 'environment:', + ' type: host', + ' workdir: ./workspaces/app', + ' setup:', + ' command: ./scripts/setup.sh', + ' args:', + ' repo: https://github.com/example/app.git', + ' commit: abc123', + ' env:', + ' SETUP_MODE: test', + ' env:', + ' NODE_ENV: test', + baseCase, + ].join('\n'), + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests[0].environment).toEqual({ + type: 'host', + workdir: path.join(dir, 'workspaces/app'), + setup: { + command: './scripts/setup.sh', + args: { + repo: 'https://github.com/example/app.git', + commit: 'abc123', + }, + env: { SETUP_MODE: 'test' }, + }, + env: { NODE_ENV: 'test' }, + }); + }); + }); + + it('loads file:// environment recipes relative to the recipe file', async () => { + await withTempDir('agentv-env-file-', async (dir) => { + const recipeDir = path.join(dir, '.agentv/environments'); + mkdirSync(recipeDir, { recursive: true }); + writeFileSync( + path.join(recipeDir, 'host.yaml'), + ['type: host', 'workdir: ./checkout', 'setup:', ' command: ./setup.sh', ''].join('\n'), + ); + const evalPath = writeEval( + dir, + ['environment: file://.agentv/environments/host.yaml', baseCase].join('\n'), + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests[0].environment).toMatchObject({ + type: 'host', + workdir: path.join(recipeDir, 'checkout'), + recipeFilePath: path.join(recipeDir, 'host.yaml'), + }); + }); + }); + + it('accepts inline docker environment recipes at schema level', async () => { + await withTempDir('agentv-env-docker-', async (dir) => { + const evalPath = writeEval( + dir, + [ + 'environment:', + ' type: docker', + ' context: ./environment', + ' dockerfile: Dockerfile', + ' workdir: /app', + ' env:', + ' NODE_ENV: test', + ' resources:', + ' cpus: 2', + ' memory: 4g', + ' mounts:', + ' - source: ./fixtures', + ' target: /fixtures', + ' access: ro', + ' secrets:', + ' OPENAI_API_KEY: placeholder', + baseCase, + ].join('\n'), + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests[0].environment).toEqual({ + type: 'docker', + context: path.join(dir, 'environment'), + dockerfile: path.join(dir, 'Dockerfile'), + workdir: '/app', + env: { NODE_ENV: 'test' }, + resources: { cpus: 2, memory: '4g' }, + mounts: [{ source: path.join(dir, 'fixtures'), target: '/fixtures', access: 'ro' }], + secrets: { OPENAI_API_KEY: 'placeholder' }, + }); + }); + }); + + it('inherits suite environment and lets case environment replace it', async () => { + await withTempDir('agentv-env-override-', async (dir) => { + const evalPath = writeEval( + dir, + [ + 'environment:', + ' type: host', + ' workdir: ./suite-workdir', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: inherited', + ' vars: { input: A }', + ' assert:', + ' - type: contains', + ' value: fixed', + ' - id: overridden', + ' environment:', + ' type: host', + ' workdir: ./case-workdir', + ' vars: { input: B }', + ' assert:', + ' - type: contains', + ' value: fixed', + '', + ].join('\n'), + ); + + const tests = await loadTests(evalPath, dir); + + expect(tests.find((test) => test.id === 'inherited')?.environment?.workdir).toBe( + path.join(dir, 'suite-workdir'), + ); + expect(tests.find((test) => test.id === 'overridden')?.environment?.workdir).toBe( + path.join(dir, 'case-workdir'), + ); + }); + }); + + it('rejects target-level environment', async () => { + await withTempDir('agentv-env-target-', async (dir) => { + const evalPath = writeEval( + dir, + [ + 'targets:', + ' - id: codex', + ' provider: codex-cli', + ' environment:', + ' type: host', + ' workdir: ./workspace', + baseCase, + ].join('\n'), + ); + + await expect(loadTests(evalPath, dir)).rejects.toThrow(/targets\[0\]\.environment/); + }); + }); + + it('rejects public workspace testbed fields with environment guidance', async () => { + await withTempDir('agentv-env-workspace-', async (dir) => { + const evalPath = writeEval( + dir, + ['workspace:', ' repos:', ' - repo: example/app', ' path: app', baseCase].join( + '\n', + ), + ); + + await expect(loadTests(evalPath, dir)).rejects.toThrow( + /workspace\.repos.*environment recipe/, + ); + }); + }); + + it('keeps top-level env and extensions distinct from environment', async () => { + await withTempDir('agentv-env-distinct-', async (dir) => { + writeFileSync(path.join(dir, 'hooks.mjs'), 'export function beforeAll() {}\n'); + const evalPath = writeEval( + dir, + [ + 'env:', + ' PROVIDER_FLAG: enabled', + 'environment:', + ' type: host', + ' workdir: ./workdir', + ' env:', + ' TESTBED_FLAG: enabled', + 'extensions:', + ' - file://hooks.mjs:beforeAll', + baseCase, + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, dir); + const validation = await validateEvalFile(evalPath); + + expect(validation.valid).toBe(true); + expect(suite.tests[0].environment?.env).toEqual({ TESTBED_FLAG: 'enabled' }); + expect(suite.tests[0].extensions?.[0]).toMatchObject({ + hook: 'beforeAll', + path: path.join(dir, 'hooks.mjs'), + }); + }); + }); +}); diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index 6b15cfad3..b510193cd 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -998,6 +998,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -1332,6 +1341,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -1666,6 +1684,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -2154,45 +2181,218 @@ }, "additionalProperties": false }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, "workspace": { "type": "object", "properties": { "template": { - "type": "string" + "not": {} }, "scope": { - "type": "string", - "enum": ["suite", "attempt"] + "not": {} }, "repos": { - "type": "array", - "items": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "repo": { - "type": "string", - "minLength": 1 - }, - "commit": { - "type": "string", - "minLength": 1 - }, - "ancestor": { - "type": "integer", - "minimum": 0 - }, - "sparse": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - } + "not": {} }, "hooks": { "type": "object", @@ -2332,25 +2532,7 @@ "additionalProperties": false }, "docker": { - "type": "object", - "properties": { - "image": { - "type": "string" - }, - "timeout": { - "type": "integer", - "minimum": 1 - }, - "memory": { - "type": "string" - }, - "cpus": { - "type": "number", - "minimum": 0.1 - } - }, - "required": ["image"], - "additionalProperties": false + "not": {} }, "env": { "type": "object", @@ -2926,6 +3108,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -3260,6 +3451,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -3594,6 +3794,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -4082,45 +4291,218 @@ }, "additionalProperties": false }, - "workspace": { - "type": "object", - "properties": { - "template": { - "type": "string" - }, - "scope": { + "environment": { + "anyOf": [ + { "type": "string", - "enum": ["suite", "attempt"] + "pattern": "^\\s*file:\\/\\/" }, - "repos": { - "type": "array", - "items": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "repo": { - "type": "string", - "minLength": 1 - }, - "commit": { - "type": "string", - "minLength": 1 - }, - "ancestor": { - "type": "integer", - "minimum": 0 - }, - "sparse": { - "type": "array", - "items": { - "type": "string" - } + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" } }, - "additionalProperties": false - } + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "workspace": { + "type": "object", + "properties": { + "template": { + "not": {} + }, + "scope": { + "not": {} + }, + "repos": { + "not": {} }, "hooks": { "type": "object", @@ -4260,25 +4642,7 @@ "additionalProperties": false }, "docker": { - "type": "object", - "properties": { - "image": { - "type": "string" - }, - "timeout": { - "type": "integer", - "minimum": 1 - }, - "memory": { - "type": "string" - }, - "cpus": { - "type": "number", - "minimum": 0.1 - } - }, - "required": ["image"], - "additionalProperties": false + "not": {} }, "env": { "type": "object", @@ -4781,6 +5145,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -5115,6 +5488,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -5449,6 +5831,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -5926,6 +6317,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -6260,6 +6660,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -6594,6 +7003,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -6984,41 +7402,242 @@ } ] }, - "scenarios": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "config": { - "type": "array", - "items": { + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { "type": "object", "properties": { - "vars": { - "type": "object", - "properties": {}, - "additionalProperties": {} - }, - "provider": { + "command": { "anyOf": [ { "type": "string", "minLength": 1 }, { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1 - }, - "extends": { - "type": "string", - "minLength": 1 - }, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, + "scenarios": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "vars": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "provider": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "extends": { + "type": "string", + "minLength": 1 + }, "provider": { "type": "string", "minLength": 1 @@ -7191,6 +7810,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -7525,6 +8153,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -7859,6 +8496,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -8438,6 +9084,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -8772,6 +9427,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -9106,6 +9770,15 @@ "type": "string" } }, + "environment": { + "not": {} + }, + "container": { + "not": {} + }, + "install": { + "not": {} + }, "reasoning_effort": { "type": "string", "minLength": 1 @@ -9594,45 +10267,218 @@ }, "additionalProperties": false }, + "environment": { + "anyOf": [ + { + "type": "string", + "pattern": "^\\s*file:\\/\\/" + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "host" + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "workdir": { + "type": "string", + "minLength": 1 + }, + "setup": { + "type": "object", + "properties": { + "command": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + } + ] + }, + "args": { + "type": "object", + "properties": {}, + "additionalProperties": {} + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + } + }, + "required": ["command"], + "additionalProperties": false + }, + "env": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "type": { + "type": "string", + "const": "docker" + }, + "context": { + "type": "string", + "minLength": 1 + }, + "dockerfile": { + "type": "string", + "minLength": 1 + }, + "image": { + "type": "string", + "minLength": 1 + }, + "resources": { + "type": "object", + "properties": { + "cpus": { + "type": "number", + "exclusiveMinimum": true, + "minimum": 0 + }, + "memory": { + "type": "string", + "minLength": 1 + }, + "disk": { + "type": "string", + "minLength": 1 + }, + "gpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + } + }, + "additionalProperties": false + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "minLength": 1 + }, + "target": { + "type": "string", + "minLength": 1 + }, + "access": { + "type": "string", + "enum": ["ro", "rw"] + }, + "read_only": { + "type": "boolean" + } + }, + "required": ["source", "target"], + "additionalProperties": false + } + }, + "secrets": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": ["workdir", "type"], + "additionalProperties": false + } + ] + }, "workspace": { "type": "object", "properties": { "template": { - "type": "string" + "not": {} }, "scope": { - "type": "string", - "enum": ["suite", "attempt"] + "not": {} }, "repos": { - "type": "array", - "items": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "repo": { - "type": "string", - "minLength": 1 - }, - "commit": { - "type": "string", - "minLength": 1 - }, - "ancestor": { - "type": "integer", - "minimum": 0 - }, - "sparse": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - } + "not": {} }, "hooks": { "type": "object", @@ -9772,25 +10618,7 @@ "additionalProperties": false }, "docker": { - "type": "object", - "properties": { - "image": { - "type": "string" - }, - "timeout": { - "type": "integer", - "minimum": 1 - }, - "memory": { - "type": "string" - }, - "cpus": { - "type": "number", - "minimum": 0.1 - } - }, - "required": ["image"], - "additionalProperties": false + "not": {} }, "env": { "type": "object", @@ -10218,41 +11046,13 @@ "type": "object", "properties": { "template": { - "type": "string" + "not": {} }, "scope": { - "type": "string", - "enum": ["suite", "attempt"] + "not": {} }, "repos": { - "type": "array", - "items": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "repo": { - "type": "string", - "minLength": 1 - }, - "commit": { - "type": "string", - "minLength": 1 - }, - "ancestor": { - "type": "integer", - "minimum": 0 - }, - "sparse": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - } + "not": {} }, "hooks": { "type": "object", @@ -10392,25 +11192,7 @@ "additionalProperties": false }, "docker": { - "type": "object", - "properties": { - "image": { - "type": "string" - }, - "timeout": { - "type": "integer", - "minimum": 1 - }, - "memory": { - "type": "string" - }, - "cpus": { - "type": "number", - "minimum": 0.1 - } - }, - "required": ["image"], - "additionalProperties": false + "not": {} }, "env": { "type": "object", From 88a798b67318854144f3f6bcc70af30efc9d48ce Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Jul 2026 10:56:18 +0200 Subject: [PATCH 2/2] fix(eval): migrate public workspace fixtures to environment --- apps/cli/src/commands/eval/task-bundle.ts | 76 +++++++- apps/cli/test/commands/eval/bundle.test.ts | 28 +-- .../commands/grade/grade-prepared.test.ts | 5 +- .../cli/test/commands/prepare/prepare.test.ts | 35 +--- apps/cli/test/commands/workspace/deps.test.ts | 2 +- .../contract/evals/release-gate.eval.yaml | 6 +- .../evals/repo-materialization.eval.yaml | 11 +- .../agent-skills-evals/csv-analyzer.EVAL.yaml | 6 +- .../multi-provider-skill-trigger.EVAL.yaml | 6 +- .../evals/skill-trigger.EVAL.yaml | 6 +- examples/features/docker-workspace/README.md | 29 +-- .../evals/docker-example.EVAL.yaml | 10 +- .../file-changes-graders/evals/suite.yaml | 6 +- .../file-changes-with-repos/evals/suite.yaml | 6 +- .../features/file-changes/evals/suite.yaml | 6 +- .../functional-grading/evals/suite.yaml | 6 +- .../features/repo-lifecycle/evals/suite.yaml | 11 +- .../sdk-eval-authoring/evals/greeting.eval.ts | 17 +- .../tool-calls-template/evals/suite.yaml | 6 +- .../vitest-workspace-grader/evals/suite.yaml | 6 +- .../graders/welcome-banner.test.ts | 14 +- .../workspace-artifact/evals/suite.yaml | 6 +- .../workspace-multi-repo/evals/suite.yaml | 26 +-- .../features/workspace-setup-script/README.md | 20 +- .../evals/dataset-vscode.eval.yaml | 15 +- .../workspace-setup-script/evals/suite.yaml | 12 +- .../evals/accuracy/suite.yaml | 6 +- .../evals/regression/suite.yaml | 6 +- .../workspace-shared-config/workspace.yaml | 13 +- .../evals/bug-fixes.eval.yaml | 14 +- examples/showcase/cross-repo-sync/README.md | 8 +- .../showcase/cross-repo-sync/evals/suite.yaml | 6 +- .../evaluation/loaders/environment-recipe.ts | 3 +- .../core/src/evaluation/prepared-workspace.ts | 2 +- .../core/src/evaluation/workspace/setup.ts | 74 ++++--- packages/core/src/index.ts | 5 + .../evaluation/eval-inline-experiment.test.ts | 60 +++--- .../core/test/evaluation/extensions.test.ts | 45 +++-- .../interpolation-integration.test.ts | 48 +++-- .../loaders/case-file-loader.test.ts | 2 +- .../core/test/evaluation/orchestrator.test.ts | 10 +- .../evaluation/prepared-workspace.test.ts | 2 +- .../evaluation/repo-schema-validation.test.ts | 75 ++++---- .../validation/eval-validator.test.ts | 181 +++++++----------- .../workspace-path-validator.test.ts | 18 +- .../workspace-config-parsing.test.ts | 108 +++++------ .../evaluation/workspace/deps-scanner.test.ts | 8 +- packages/sdk/src/eval.ts | 46 +++++ packages/sdk/src/index.ts | 6 + packages/sdk/test/eval-authoring.test.ts | 36 ++-- 50 files changed, 622 insertions(+), 537 deletions(-) diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 397ee2f30..a2c1c19fb 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { + type EnvironmentRecipe, type EvalSourceReference, type EvalTest, type JsonObject, @@ -116,6 +117,8 @@ export interface MaterializedEvalBundlePaths { type BundleReferenceKind = | EvalSourceReference['kind'] | 'expected_output_file' + | 'environment_workdir' + | 'environment_setup_command' | 'workspace_template' | 'workspace_hook_command'; @@ -188,7 +191,13 @@ function referenceBucket( if (reference.kind === 'workspace_template') { return BUNDLE_WORKSPACES_DIRNAME; } - if (reference.kind === 'workspace_hook_command') { + if (reference.kind === 'environment_workdir') { + return BUNDLE_WORKSPACES_DIRNAME; + } + if ( + reference.kind === 'workspace_hook_command' || + reference.kind === 'environment_setup_command' + ) { return BUNDLE_SCRIPTS_DIRNAME; } return TASK_GRADERS_DIRNAME; @@ -798,6 +807,14 @@ function serializeWorkspace( return rewritePathsDeep(portableWorkspace, rewrites) as Record; } +function serializeEnvironment( + environment: EnvironmentRecipe, + rewrites: ReadonlyMap, +): Record { + const { recipeFilePath: _recipeFilePath, ...portableEnvironment } = environment; + return rewritePathsDeep(portableEnvironment, rewrites) as Record; +} + function buildPortableEvalCase( test: EvalTest, rewrites: ReadonlyMap, @@ -820,6 +837,9 @@ function buildPortableEvalCase( if (test.workspace) { testCase.workspace = serializeWorkspace(test.workspace, rewrites); } + if (test.environment) { + testCase.environment = serializeEnvironment(test.environment, rewrites); + } if (test.metadata && Object.keys(test.metadata).length > 0) { testCase.metadata = rewritePathsDeep(test.metadata, rewrites); } @@ -894,6 +914,58 @@ async function maybeWorkspaceHookCommandReference(options: { }; } +function environmentBaseDir(environment: EnvironmentRecipe, evalFileDir: string): string { + return environment.recipeFilePath ? path.dirname(environment.recipeFilePath) : evalFileDir; +} + +async function collectEnvironmentReferences( + tests: readonly EvalTest[], + evalFileDir: string, +): Promise { + const references: BundleSourceReference[] = []; + + for (const test of tests) { + const environment = test.environment; + if (!environment) { + continue; + } + + if (environment.type === 'host') { + references.push({ + kind: 'environment_workdir', + displayPath: environment.workdir, + resolvedPath: environment.workdir, + location: `environment.workdir for test "${test.id}"`, + }); + } + + const command = environment.setup?.command; + if (!command) { + continue; + } + + const baseDir = environmentBaseDir(environment, evalFileDir); + const commandArgs = Array.isArray(command) ? command : [command]; + for (const arg of commandArgs) { + const reference = await maybeWorkspaceHookCommandReference({ + arg, + baseDir, + testId: test.id, + hookName: 'setup', + }); + if (reference) { + references.push({ + ...reference, + kind: 'environment_setup_command', + location: `environment.setup.command for test "${test.id}"`, + }); + } + } + } + + return references; +} + async function collectWorkspaceReferences( tests: readonly EvalTest[], evalFileDir: string, @@ -1104,9 +1176,11 @@ export async function materializeEvalBundle( const evalFileDir = path.dirname(path.resolve(options.evalFilePath)); const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir); + const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir); const references: BundleSourceReference[] = [ ...options.tests.flatMap((test) => test.source?.references ?? []), ...collectExpectedOutputReferences(options.tests), + ...environmentReferences, ...workspaceReferences.references, ]; diff --git a/apps/cli/test/commands/eval/bundle.test.ts b/apps/cli/test/commands/eval/bundle.test.ts index 484bf0865..bfa7f6bc4 100644 --- a/apps/cli/test/commands/eval/bundle.test.ts +++ b/apps/cli/test/commands/eval/bundle.test.ts @@ -93,11 +93,11 @@ await Bun.write(\`\${payload.workspace_path}/hook-ran.txt\`, 'ok\\n'); const evalPath = path.join(sourceDir, 'evals', 'demo.eval.yaml'); const sourceEvalBefore = `name: portable-demo target: inherited -workspace: - template: ../workspace-template - hooks: - before_each: - command: ["bun", "../scripts/setup.ts"] +environment: + type: host + workdir: ../workspace-template + setup: + command: ["bun", "../scripts/setup.ts"] tests: ../data/cases.yaml `; await writeFile(evalPath, sourceEvalBefore, 'utf8'); @@ -105,7 +105,7 @@ tests: ../data/cases.yaml return { sourceDir, bundleDir, evalPath, sourceEvalBefore }; } - it('bundles inherited targets, relative data, workspace templates, and scripts into a runnable directory', async () => { + it('bundles inherited targets, relative data, environment workdirs, and scripts into a runnable directory', async () => { const { sourceDir, bundleDir, evalPath, sourceEvalBefore } = await createPortableSourceFixture(); @@ -145,9 +145,10 @@ tests: ../data/cases.yaml expect(bundledEval.execution).toBeUndefined(); const [testCase] = bundledEval.tests as Record[]; expect(testCase.id).toBe('case-alpha'); - expect(testCase.workspace).toMatchObject({ - template: 'workspaces/workspace-template', - hooks: { before_each: { command: ['bun', 'scripts/scripts/setup.ts'] } }, + expect(testCase.environment).toMatchObject({ + type: 'host', + workdir: 'workspaces/workspace-template', + setup: { command: ['bun', 'scripts/scripts/setup.ts'] }, }); expect(bundledEval.prompts).toEqual(['{{ input }}']); const input = (testCase.vars as Record).input as Array<{ @@ -212,7 +213,7 @@ tests: expect(bundledTargets).toContain('inline bundled response'); }, 30_000); - it('reports unbundleable workspace references with their eval location', async () => { + it('reports unbundleable environment references with their eval location', async () => { const sourceDir = path.join(tempDir, 'missing-source'); const bundleDir = path.join(tempDir, 'missing-bundle'); await mkdir(path.join(sourceDir, '.agentv'), { recursive: true }); @@ -227,8 +228,9 @@ tests: ); await writeFile( path.join(sourceDir, 'evals', 'missing-template.eval.yaml'), - `workspace: - template: ../does-not-exist + `environment: + type: host + workdir: ../does-not-exist prompts: - "{{ input }}" tests: @@ -253,7 +255,7 @@ tests: expect(result.exitCode).toBe(1); const output = `${result.stdout}\n${result.stderr}`; expect(output).toContain('Cannot bundle eval'); - expect(output).toContain('workspace.template for test "missing-template"'); + expect(output).toContain('environment.workdir for test "missing-template"'); expect(output).toContain('not found'); }, 30_000); }); diff --git a/apps/cli/test/commands/grade/grade-prepared.test.ts b/apps/cli/test/commands/grade/grade-prepared.test.ts index f3388ae5f..8959a8821 100644 --- a/apps/cli/test/commands/grade/grade-prepared.test.ts +++ b/apps/cli/test/commands/grade/grade-prepared.test.ts @@ -92,8 +92,9 @@ targets: await writeFile( evalPath, ` -workspace: - template: ../template +environment: + type: host + workdir: ../template assert: ${assertionYaml .trim() diff --git a/apps/cli/test/commands/prepare/prepare.test.ts b/apps/cli/test/commands/prepare/prepare.test.ts index 9aa7ac7d3..1a9a2ed57 100644 --- a/apps/cli/test/commands/prepare/prepare.test.ts +++ b/apps/cli/test/commands/prepare/prepare.test.ts @@ -68,19 +68,9 @@ targets: ); await writeFile( evalPath, - `workspace: - template: ../template - hooks: - before_all: - command: - - bun - - ../scripts/hook.ts - - workspace_before_all - before_each: - command: - - bun - - ../scripts/hook.ts - - workspace_before_each + `environment: + type: host + workdir: ../template target: extends: codex hooks: @@ -163,22 +153,12 @@ describe('agentv prepare', () => { expect(await exists(promptPath)).toBe(true); expect(await exists(manifestPath)).toBe(true); - for (const step of [ - 'workspace_before_all', - 'target_before_all', - 'workspace_before_each', - 'target_before_each', - ]) { + for (const step of ['target_before_all', 'target_before_each']) { expect(await exists(path.join(workspacePath, `${step}.txt`))).toBe(true); } expect( (await readFile(path.join(workspacePath, 'hook-order.txt'), 'utf8')).trim().split('\n'), - ).toEqual([ - 'workspace_before_all', - 'target_before_all', - 'workspace_before_each', - 'target_before_each', - ]); + ).toEqual(['target_before_all', 'target_before_each']); expect(await exists(targetMarker)).toBe(false); expect(await exists(graderMarker)).toBe(false); @@ -283,8 +263,9 @@ targets: - id: agentv:agent-rules hook: beforeAll rules: ../rules/AGENTS.md -workspace: - template: ../template +environment: + type: host + workdir: ../template prompts: - "{{ input }}" tests: diff --git a/apps/cli/test/commands/workspace/deps.test.ts b/apps/cli/test/commands/workspace/deps.test.ts index d8cbcaac8..12a34957e 100644 --- a/apps/cli/test/commands/workspace/deps.test.ts +++ b/apps/cli/test/commands/workspace/deps.test.ts @@ -13,7 +13,7 @@ const __dirname = path.dirname(__filename); const projectRoot = path.resolve(__dirname, '../../../../..'); const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); -describe('workspace deps', () => { +describe('workspace deps legacy compatibility command', () => { it('exits non-zero when an eval uses removed repo schema fields', async () => { const tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-workspace-deps-test-')); try { diff --git a/examples/contract/evals/release-gate.eval.yaml b/examples/contract/evals/release-gate.eval.yaml index 9e5e80c93..58f3fcf2e 100644 --- a/examples/contract/evals/release-gate.eval.yaml +++ b/examples/contract/evals/release-gate.eval.yaml @@ -1,8 +1,8 @@ name: release-contract description: Lightweight release gate for npm latest promotion. -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: github-models-contract prompts: - "{{ input }}" diff --git a/examples/contract/evals/repo-materialization.eval.yaml b/examples/contract/evals/repo-materialization.eval.yaml index 0c1f85897..cd545d688 100644 --- a/examples/contract/evals/repo-materialization.eval.yaml +++ b/examples/contract/evals/repo-materialization.eval.yaml @@ -1,10 +1,13 @@ name: repo-materialization-contract description: Release gate for public repo materialization, previous-commit checkout, and file input substitution. -workspace: - scope: suite - repos: - - path: ./fixture +environment: + type: host + workdir: ./fixture + setup: + command: ../scripts/materialize-repo.sh + args: + path: ./fixture repo: EntityProcess/agentv-contract-fixture commit: 21a34daed7ebcfe36cbed053607622a55e5e94cb target: github-models-contract diff --git a/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml b/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml index 2a90bea23..5e1e255a1 100644 --- a/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml +++ b/examples/features/agent-skills-evals/csv-analyzer.EVAL.yaml @@ -3,9 +3,9 @@ tags: - skill-trigger extensions: - agentv:agent-rules -workspace: - scope: suite - template: workspace/ +environment: + type: host + workdir: workspace/ prompts: - "{{ input }}" tests: diff --git a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml index 9bb4fb60e..8e44d427d 100644 --- a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml +++ b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml @@ -1,8 +1,8 @@ extensions: - agentv:agent-rules -workspace: - scope: suite - template: workspace/ +environment: + type: host + workdir: workspace/ prompts: - "{{ input }}" tests: diff --git a/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml b/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml index 2d1841b23..b0c6c0756 100644 --- a/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml +++ b/examples/features/copilot-transcript-replay/evals/skill-trigger.EVAL.yaml @@ -3,9 +3,9 @@ tags: target: copilot-transcript-replay extensions: - file://../scripts/copilot-replay-workspace.mjs:beforeAll -workspace: - scope: suite - template: ../workspace/ +environment: + type: host + workdir: ../workspace/ prompts: - "{{ input }}" tests: diff --git a/examples/features/docker-workspace/README.md b/examples/features/docker-workspace/README.md index 9096dc89b..2b9dd4820 100644 --- a/examples/features/docker-workspace/README.md +++ b/examples/features/docker-workspace/README.md @@ -9,7 +9,7 @@ When evaluating coding agents (e.g., SWE-bench), the grader script needs to: 2. Run tests inside the repository's environment 3. Report pass/fail results -Docker workspaces let you run this grading process inside a pre-built container +Docker environments let you run this grading process inside a pre-built container image that has the repository, dependencies, and test infrastructure ready. ## How It Works @@ -29,28 +29,29 @@ image that has the repository, dependencies, and test infrastructure ready. ## YAML Schema ```yaml -workspace: - scope: suite - docker: - image: swebench/sweb.eval.x86_64.django__django-15180 - timeout: 1800 # seconds (default: 1800) +environment: + type: docker + image: swebench/sweb.eval.x86_64.django__django-15180 + workdir: /testbed + resources: memory: 4g # optional Docker memory limit cpus: 2 # optional Docker CPU limit ``` -For evals that need a repo pinned to a dataset snapshot, use `workspace.repos[].commit`: +For evals that need a repo pinned to a dataset snapshot, keep that metadata in `environment.setup.args`: ```yaml -workspace: - scope: suite - docker: - image: swebench/sweb.eval.x86_64.django__django-15180 - repos: - - path: /testbed +environment: + type: docker + image: swebench/sweb.eval.x86_64.django__django-15180 + workdir: /testbed + setup: + command: ./setup.sh + args: commit: abc123def ``` -Repos defined without `repo` are assumed to already exist inside the container (e.g., SWE-bench prebuilt images). +Prebuilt images can carry the repository inside the container, while the setup args record the dataset snapshot being evaluated. ## Running diff --git a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml index 4eecdc29d..ab22ff1a1 100644 --- a/examples/features/docker-workspace/evals/docker-example.EVAL.yaml +++ b/examples/features/docker-workspace/evals/docker-example.EVAL.yaml @@ -1,11 +1,11 @@ name: docker-workspace-example description: Example eval using Docker workspace for grading target: mock_agent -workspace: - scope: suite - docker: - image: python:3.11-slim - timeout: 300 +environment: + type: docker + image: python:3.11-slim + workdir: /workspace + resources: memory: 2g cpus: 1 prompts: diff --git a/examples/features/file-changes-graders/evals/suite.yaml b/examples/features/file-changes-graders/evals/suite.yaml index 3dbbdefb0..898961c0d 100644 --- a/examples/features/file-changes-graders/evals/suite.yaml +++ b/examples/features/file-changes-graders/evals/suite.yaml @@ -1,7 +1,7 @@ description: Verify file_changes diffs are accessible to LLM grader (llm-rubric, built-in, and copilot-cli) -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_agent prompts: - "{{ input }}" diff --git a/examples/features/file-changes-with-repos/evals/suite.yaml b/examples/features/file-changes-with-repos/evals/suite.yaml index fe63b020d..1e183940d 100644 --- a/examples/features/file-changes-with-repos/evals/suite.yaml +++ b/examples/features/file-changes-with-repos/evals/suite.yaml @@ -1,8 +1,8 @@ name: file-changes-with-repos description: Verify file_changes captures workspace-root files AND changes inside nested repos -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template extensions: - file://../scripts/setup-nested-repo.mjs:beforeAll target: mock_agent diff --git a/examples/features/file-changes/evals/suite.yaml b/examples/features/file-changes/evals/suite.yaml index 4ac0bf0d1..4da0dd020 100644 --- a/examples/features/file-changes/evals/suite.yaml +++ b/examples/features/file-changes/evals/suite.yaml @@ -1,8 +1,8 @@ name: file-changes description: Verify file_changes captures edits, creates, and deletes across multiple tests -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_agent prompts: - "{{ input }}" diff --git a/examples/features/functional-grading/evals/suite.yaml b/examples/features/functional-grading/evals/suite.yaml index f162f23c4..371c427d7 100644 --- a/examples/features/functional-grading/evals/suite.yaml +++ b/examples/features/functional-grading/evals/suite.yaml @@ -1,8 +1,8 @@ name: functional-grading description: Functional grading with workspace_path — deploy-and-test pattern -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_agent prompts: - "{{ input }}" diff --git a/examples/features/repo-lifecycle/evals/suite.yaml b/examples/features/repo-lifecycle/evals/suite.yaml index ca9abe72b..f045a59d5 100644 --- a/examples/features/repo-lifecycle/evals/suite.yaml +++ b/examples/features/repo-lifecycle/evals/suite.yaml @@ -1,11 +1,14 @@ description: "Demonstrates workspace repo lifecycle: clone a git repo into the workspace, check out a specific commit, and have the agent work on it." -workspace: - scope: suite - repos: - - path: ./repo +environment: + type: host + workdir: ./repo + setup: + command: ../scripts/materialize-repo.sh + args: repo: https://github.com/EntityProcess/agentv.git commit: main + path: ./repo tags: - agent prompts: diff --git a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts index 8aa011910..86af6a410 100644 --- a/examples/features/sdk-eval-authoring/evals/greeting.eval.ts +++ b/examples/features/sdk-eval-authoring/evals/greeting.eval.ts @@ -5,12 +5,9 @@ export default defineEval({ description: 'YAML-aligned TypeScript eval authoring with @agentv/sdk', inputFiles: ['../fixtures/shared-context.md'], target: 'mock-sdk', - workspace: { - hooks: { - beforeAll: { - command: ['echo', 'suite-start'], - }, - }, + environment: { + type: 'host', + workdir: '../fixtures', }, tests: [ { @@ -22,14 +19,6 @@ export default defineEval({ graders.contains('Hello', { name: 'mentions-hello' }), graders.regex(/mock target/i, { name: 'mentions-mock-target' }), ], - workspace: { - hooks: { - beforeEach: { - command: ['echo', 'per-test-setup'], - timeoutMs: 1_000, - }, - }, - }, }, ], }); diff --git a/examples/features/tool-calls-template/evals/suite.yaml b/examples/features/tool-calls-template/evals/suite.yaml index d593211f2..ddcfd87c9 100644 --- a/examples/features/tool-calls-template/evals/suite.yaml +++ b/examples/features/tool-calls-template/evals/suite.yaml @@ -2,9 +2,9 @@ name: tool-calls-template description: Rubric assertions with {{ tool_calls }} for skill verification extensions: - agentv:agent-rules -workspace: - scope: suite - template: ../workspace/ +environment: + type: host + workdir: ../workspace/ prompts: - "{{ input }}" tests: diff --git a/examples/features/vitest-workspace-grader/evals/suite.yaml b/examples/features/vitest-workspace-grader/evals/suite.yaml index a79fff3e3..527363cae 100644 --- a/examples/features/vitest-workspace-grader/evals/suite.yaml +++ b/examples/features/vitest-workspace-grader/evals/suite.yaml @@ -1,8 +1,8 @@ name: vitest-workspace-grader description: Deterministic workspace grading with a Vitest verifier file. -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_agent prompts: - "{{ input }}" diff --git a/examples/features/vitest-workspace-grader/graders/welcome-banner.test.ts b/examples/features/vitest-workspace-grader/graders/welcome-banner.test.ts index 12fb09f7a..675a67681 100644 --- a/examples/features/vitest-workspace-grader/graders/welcome-banner.test.ts +++ b/examples/features/vitest-workspace-grader/graders/welcome-banner.test.ts @@ -2,14 +2,18 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; +const workspacePath = process.env.AGENTV_WORKSPACE_PATH; + function readWorkspaceFile(relativePath: string) { - return readFileSync( - join(process.env.AGENTV_WORKSPACE_PATH ?? process.cwd(), relativePath), - 'utf8', - ); + if (!workspacePath) { + throw new Error('AGENTV_WORKSPACE_PATH is required'); + } + return readFileSync(join(workspacePath, relativePath), 'utf8'); } -describe('welcome banner', () => { +const describeWithWorkspace = workspacePath ? describe : describe.skip; + +describeWithWorkspace('welcome banner', () => { const page = () => readWorkspaceFile('app/page.tsx'); it('shows ready status text', () => { diff --git a/examples/features/workspace-artifact/evals/suite.yaml b/examples/features/workspace-artifact/evals/suite.yaml index f5b0fa3c1..ada8130ac 100644 --- a/examples/features/workspace-artifact/evals/suite.yaml +++ b/examples/features/workspace-artifact/evals/suite.yaml @@ -1,8 +1,8 @@ name: workspace-artifact description: Verify file_changes captures generated artifacts (CSV) under workspace_path -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_csv_agent prompts: - "{{ input }}" diff --git a/examples/features/workspace-multi-repo/evals/suite.yaml b/examples/features/workspace-multi-repo/evals/suite.yaml index f669a6be9..5cba5f5b2 100644 --- a/examples/features/workspace-multi-repo/evals/suite.yaml +++ b/examples/features/workspace-multi-repo/evals/suite.yaml @@ -1,18 +1,18 @@ description: Demonstrates a multi-repo workspace. Two repos (agentv and allagents) are cloned into the workspace. -workspace: - scope: suite - template: ../workspace-template - hooks: - after_each: - reset: fast - repos: - - path: ./agentv - repo: https://github.com/EntityProcess/agentv.git - commit: main - - path: ./allagents - repo: https://github.com/EntityProcess/allagents.git - commit: main +environment: + type: host + workdir: ../workspace-template + setup: + command: ../scripts/materialize-repos.sh + args: + repos: + - path: ./agentv + repo: https://github.com/EntityProcess/agentv.git + commit: main + - path: ./allagents + repo: https://github.com/EntityProcess/allagents.git + commit: main tags: - agent prompts: diff --git a/examples/features/workspace-setup-script/README.md b/examples/features/workspace-setup-script/README.md index 2b65b4519..113c047a5 100644 --- a/examples/features/workspace-setup-script/README.md +++ b/examples/features/workspace-setup-script/README.md @@ -8,7 +8,7 @@ Demonstrates using a `beforeAll` lifecycle extension to clean and re-initialize ## Solution -A Node.js lifecycle extension exports `beforeAll(context)`. AgentV runs it after `workspace.template` and `workspace.repos` materialize, so the extension can safely prepare local configuration without owning repo provisioning. +A Node.js lifecycle extension exports `beforeAll(context)`. AgentV runs it after the authored `environment` recipe is prepared, so the extension can safely prepare local configuration without owning repo provisioning. ``` workspace-setup-script/ @@ -32,17 +32,19 @@ workspace-setup-script/ ## Eval YAML -Use top-level `extensions` for executable setup and keep repos under `workspace.repos`: +Use top-level `extensions` for lifecycle hooks and keep authored testbed setup under `environment`: ```yaml extensions: - file://../scripts/workspace-setup.mjs:beforeAll -workspace: - scope: suite - template: ../workspace-template - repos: - - path: ./my-repo +environment: + type: host + workdir: ../workspace-template + setup: + command: ../scripts/materialize-repo.sh + args: + path: ./my-repo repo: https://github.com/EntityProcess/agentv.git commit: main ``` @@ -69,8 +71,8 @@ The `type: file` path is resolved from the eval file's directory up to the repo ## How It Works -1. AgentV copies `workspace-template/` to the suite workspace. -2. AgentV clones `workspace.repos`. +1. AgentV prepares the authored `environment`. +2. The environment setup materializes `my-repo/`. 3. The `beforeAll` extension removes stale `.allagents/` config and runs `npx allagents workspace init`. 4. The extension registers the local marketplace with `--scope project`. 5. `allagents workspace sync` installs `my-plugin@workspace-setup-script-marketplace`. diff --git a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml index 86f06fe18..e25f3cc59 100644 --- a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml @@ -2,14 +2,13 @@ description: Demonstrates using a beforeAll lifecycle extension with the VSCode suite.yaml but uses vscode instead of copilot. extensions: - file://../scripts/workspace-setup.mjs:beforeAll -workspace: - scope: suite - template: ../workspace-template - hooks: - after_each: - reset: fast - repos: - - path: ./my-repo +environment: + type: host + workdir: ../workspace-template + setup: + command: ../scripts/materialize-repo.sh + args: + path: ./my-repo repo: https://github.com/EntityProcess/agentv.git commit: main tags: diff --git a/examples/features/workspace-setup-script/evals/suite.yaml b/examples/features/workspace-setup-script/evals/suite.yaml index fad03c4f8..fec3ff0ae 100644 --- a/examples/features/workspace-setup-script/evals/suite.yaml +++ b/examples/features/workspace-setup-script/evals/suite.yaml @@ -2,11 +2,13 @@ description: Demonstrates using a beforeAll lifecycle extension to clean and re- allagents workspace before evaluation runs. extensions: - file://../scripts/workspace-setup.mjs:beforeAll -workspace: - scope: suite - template: ../workspace-template - repos: - - path: ./my-repo +environment: + type: host + workdir: ../workspace-template + setup: + command: ../scripts/materialize-repo.sh + args: + path: ./my-repo repo: https://github.com/EntityProcess/agentv.git commit: main tags: diff --git a/examples/features/workspace-shared-config/evals/accuracy/suite.yaml b/examples/features/workspace-shared-config/evals/accuracy/suite.yaml index 06f91a720..769edfc84 100644 --- a/examples/features/workspace-shared-config/evals/accuracy/suite.yaml +++ b/examples/features/workspace-shared-config/evals/accuracy/suite.yaml @@ -1,6 +1,6 @@ -description: Accuracy eval that references a shared workspace config file. The workspace is defined - once in workspace.yaml and reused across eval files. -workspace: ../../workspace.yaml +description: Accuracy eval that references a shared environment recipe file. The environment is + defined once in workspace.yaml and reused across eval files. +environment: file://../../workspace.yaml tags: - agent prompts: diff --git a/examples/features/workspace-shared-config/evals/regression/suite.yaml b/examples/features/workspace-shared-config/evals/regression/suite.yaml index 67d93653f..38c74e265 100644 --- a/examples/features/workspace-shared-config/evals/regression/suite.yaml +++ b/examples/features/workspace-shared-config/evals/regression/suite.yaml @@ -1,6 +1,6 @@ -description: Regression eval that references the same shared workspace config file. Demonstrates - workspace config reuse across eval files in different directories. -workspace: ../../workspace.yaml +description: Regression eval that references the same shared environment recipe file. Demonstrates + environment recipe reuse across eval files in different directories. +environment: file://../../workspace.yaml tags: - agent prompts: diff --git a/examples/features/workspace-shared-config/workspace.yaml b/examples/features/workspace-shared-config/workspace.yaml index a74989a77..8dbe65776 100644 --- a/examples/features/workspace-shared-config/workspace.yaml +++ b/examples/features/workspace-shared-config/workspace.yaml @@ -1,9 +1,8 @@ -scope: suite -template: ./workspace-template -hooks: - after_each: - reset: fast -repos: - - path: ./agentv +type: host +workdir: ./workspace-template +setup: + command: ./scripts/materialize-repo.sh + args: + path: ./agentv repo: https://github.com/EntityProcess/agentv.git commit: main diff --git a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml index 8f0712f11..ad5ea8d01 100644 --- a/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml +++ b/examples/showcase/bug-fix-benchmark/evals/bug-fixes.eval.yaml @@ -2,15 +2,15 @@ description: | Evaluate coding agents on real bug fixes from public GitHub repositories. Compare baseline performance against plugin-augmented workflows: superpowers (obra), compound-engineering (Every Inc), agent-skills (Addy Osmani). -workspace: - scope: suite - repos: - - path: ./repo +environment: + type: host + workdir: ./repo + setup: + command: ../scripts/materialize-repo.sh + args: + path: ./repo repo: https://github.com/EntityProcess/agentv commit: 6e446b722627e9df017b22e391fa63320362d8c7 - hooks: - before_each: - reset: fast target: name: claude-baseline extends: "{{ env.AGENT_TARGET }}" diff --git a/examples/showcase/cross-repo-sync/README.md b/examples/showcase/cross-repo-sync/README.md index f7a2cc849..d7c664d98 100644 --- a/examples/showcase/cross-repo-sync/README.md +++ b/examples/showcase/cross-repo-sync/README.md @@ -6,13 +6,13 @@ Evaluates whether a coding agent can keep two public repos in sync after one cha When **AgentV** (EntityProcess/agentv) ships a feature, the **agentevals** (agentevals/agentevals) spec docs must be updated to reflect the change. This eval measures how well an agent handles that cross-repo synchronization. -## Workspace Features Demonstrated +## Environment Features Demonstrated | Feature | Usage | |---------|-------| -| `workspace.template` | AGENTS.md + skills dir copied to workspace | -| `workspace.before_each` | Clones agentevals at "before" state per test | -| `workspace.after_each` | Resets git state between tests | +| `environment.workdir` | AGENTS.md + skills dir available to the task environment | +| `extensions.beforeEach` | Clones agentevals at "before" state per test | +| `extensions.afterEach` | Resets git state between tests | | `metadata` | Commit SHAs passed to setup via stdin JSON | | `fileChanges` | Unified diff of agent's edits | diff --git a/examples/showcase/cross-repo-sync/evals/suite.yaml b/examples/showcase/cross-repo-sync/evals/suite.yaml index 01249201f..4dd24d377 100644 --- a/examples/showcase/cross-repo-sync/evals/suite.yaml +++ b/examples/showcase/cross-repo-sync/evals/suite.yaml @@ -8,9 +8,9 @@ tags: extensions: - file://../scripts/setup.ts:beforeEach - file://../scripts/reset.ts:afterEach -workspace: - scope: suite - template: ../workspace-template +environment: + type: host + workdir: ../workspace-template target: mock_agent prompts: - "{{ input }}" diff --git a/packages/core/src/evaluation/loaders/environment-recipe.ts b/packages/core/src/evaluation/loaders/environment-recipe.ts index 352e4238a..1d163b44c 100644 --- a/packages/core/src/evaluation/loaders/environment-recipe.ts +++ b/packages/core/src/evaluation/loaders/environment-recipe.ts @@ -1,6 +1,7 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; +import { interpolateEnv } from '../interpolation.js'; import type { JsonObject, JsonValue } from '../types.js'; import { isJsonObject } from '../types.js'; import { parseYamlValue } from '../yaml-loader.js'; @@ -70,7 +71,7 @@ export async function resolveEnvironmentRecipe( const recipePath = resolveReferencePath(raw, evalFileDir); let parsed: unknown; try { - parsed = parseYamlValue(await readFile(recipePath, 'utf8')); + parsed = interpolateEnv(parseYamlValue(await readFile(recipePath, 'utf8')), process.env); } catch (error) { throw new Error( `${location} recipe file not found or unreadable: ${raw} (${(error as Error).message})`, diff --git a/packages/core/src/evaluation/prepared-workspace.ts b/packages/core/src/evaluation/prepared-workspace.ts index 786514c9d..3ed8f72d7 100644 --- a/packages/core/src/evaluation/prepared-workspace.ts +++ b/packages/core/src/evaluation/prepared-workspace.ts @@ -202,7 +202,7 @@ export async function prepareEvalWorkspace( if (!caseSetup.workspacePath) { throw new Error( - `No workspace was materialized for test "${evalCase.id}". Add workspace.template, workspace.repos, or workspace.hooks before preparing an external attempt.`, + `No workspace was materialized for test "${evalCase.id}". Add environment.workdir before preparing an external attempt, or use an internal workspace override for private runtime plumbing.`, ); } diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 487133083..725e62a68 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -182,7 +182,8 @@ export function caseUsesSharedWorkspaceSetup( evalCase: EvalTest, setup: Pick, ): boolean { - if (isAttemptScopedWorkspace(evalCase.workspace)) { + const workspace = effectiveRuntimeWorkspace(evalCase); + if (isAttemptScopedWorkspace(workspace)) { return false; } if (setup.sharedWorkspaceAppliesToAllCases) { @@ -190,7 +191,7 @@ export function caseUsesSharedWorkspaceSetup( } return ( setup.sharedWorkspaceOwnerKey !== undefined && - workspaceNeedsSharedSetup(evalCase.workspace) && + workspaceNeedsSharedSetup(workspace) && sharedWorkspaceOwnerKey(evalCase) === setup.sharedWorkspaceOwnerKey ); } @@ -210,6 +211,17 @@ function workspaceNeedsSharedSetup( ); } +function effectiveRuntimeWorkspace(evalCase: EvalTest): WorkspaceConfig | undefined { + const workspace = evalCase.workspace; + if (evalCase.environment?.type !== 'host') { + return workspace; + } + return { + ...(workspace ?? {}), + template: workspace?.template ?? evalCase.environment.workdir, + }; +} + function stableWorkspaceValue(value: unknown): string { if (value === undefined) { return 'undefined'; @@ -246,7 +258,7 @@ function sharedWorkspaceOwnerKey(evalCase: EvalTest): string { : source?.evalFileAbsolutePath ? `parent:${source.evalFileAbsolutePath}` : 'programmatic'; - return `${sourceKey}:${stableWorkspaceValue(evalCase.workspace)}`; + return `${sourceKey}:${stableWorkspaceValue(effectiveRuntimeWorkspace(evalCase))}`; } interface SelectedSharedWorkspace { @@ -261,7 +273,8 @@ function selectSuiteWorkspace(evalCases: readonly EvalTest[]): SelectedSharedWor >(); for (const evalCase of evalCases) { - if (!workspaceNeedsSharedSetup(evalCase.workspace)) { + const workspace = effectiveRuntimeWorkspace(evalCase); + if (!workspaceNeedsSharedSetup(workspace)) { continue; } const key = sharedWorkspaceOwnerKey(evalCase); @@ -271,7 +284,7 @@ function selectSuiteWorkspace(evalCases: readonly EvalTest[]): SelectedSharedWor continue; } candidates.set(key, { - workspace: evalCase.workspace, + workspace, owner: describeWorkspaceOwner(evalCase), testIds: [evalCase.id], }); @@ -298,7 +311,7 @@ function selectSuiteExtensions(evalCases: readonly EvalTest[]): readonly AgentVE const candidates = new Map(); for (const evalCase of evalCases) { const extensions = evalCase.extensions ?? []; - if (extensions.length === 0 || isAttemptScopedWorkspace(evalCase.workspace)) { + if (extensions.length === 0 || isAttemptScopedWorkspace(effectiveRuntimeWorkspace(evalCase))) { continue; } candidates.set(stableWorkspaceValue(extensions), extensions); @@ -433,7 +446,7 @@ export async function prepareSharedWorkspaceSetup( if ( useStaticWorkspace && - evalCases.some((evalCase) => isAttemptScopedWorkspace(evalCase.workspace)) + evalCases.some((evalCase) => isAttemptScopedWorkspace(effectiveRuntimeWorkspace(evalCase))) ) { throw new Error( 'static workspace mode is incompatible with workspace.scope: attempt. Use workspace.scope: suite or omit the static workspace override.', @@ -738,7 +751,8 @@ export async function prepareEvalCaseWorkspace( sharedExtensionState, } = options; - let workspacePath: string | undefined = isAttemptScopedWorkspace(evalCase.workspace) + const runtimeWorkspace = effectiveRuntimeWorkspace(evalCase); + let workspacePath: string | undefined = isAttemptScopedWorkspace(runtimeWorkspace) ? undefined : sharedWorkspacePath; const inheritedSuiteWorkspaceFile = workspacePath ? suiteWorkspaceFile : undefined; @@ -746,12 +760,12 @@ export async function prepareEvalCaseWorkspace( let beforeEachOutput: string | undefined; const isSharedWorkspace = !!workspacePath; let caseWorkspaceFile: string | undefined; - const caseHooksEnabled = hooksEnabled(evalCase.workspace); + const caseHooksEnabled = hooksEnabled(runtimeWorkspace); const hookExecutions: WorkspaceSetupHookExecution[] = []; let extensionState = sharedExtensionState; if (!workspacePath) { - const rawCaseTemplate = evalCase.workspace?.template; + const rawCaseTemplate = runtimeWorkspace?.template; const resolvedCaseTemplate = await resolveWorkspaceTemplate(rawCaseTemplate); const caseWorkspaceTemplate = resolvedCaseTemplate?.dir; caseWorkspaceFile = resolvedCaseTemplate?.workspaceFile; @@ -781,24 +795,22 @@ export async function prepareEvalCaseWorkspace( if ( !workspacePath && - (evalCase.workspace?.hooks || - evalCase.workspace?.repos?.length || - evalCase.extensions?.length) && + (runtimeWorkspace?.hooks || runtimeWorkspace?.repos?.length || evalCase.extensions?.length) && evalRunId ) { workspacePath = getWorkspacePath(evalRunId, evalCase.id); await mkdir(workspacePath, { recursive: true }); } - if (evalCase.workspace?.repos?.length && workspacePath) { + if (runtimeWorkspace?.repos?.length && workspacePath) { const perCaseRepoManager = new RepoManager(setupDebug, { projectConfigDir: evalDir }); try { if (setupDebug) { console.log( - `[setup] test=${evalCase.id} materializing ${evalCase.workspace.repos.length} attempt repo(s) into ${workspacePath}`, + `[setup] test=${evalCase.id} materializing ${runtimeWorkspace.repos.length} attempt repo(s) into ${workspacePath}`, ); } - await perCaseRepoManager.materializeAll(evalCase.workspace.repos, workspacePath); + await perCaseRepoManager.materializeAll(runtimeWorkspace.repos, workspacePath); if (setupDebug) { console.log(`[setup] test=${evalCase.id} attempt repo materialization complete`); } @@ -839,7 +851,7 @@ export async function prepareEvalCaseWorkspace( } } - const caseDockerConfig = evalCase.workspace?.docker; + const caseDockerConfig = runtimeWorkspace?.docker; if (caseDockerConfig) { try { await prepareDockerWorkspace(caseDockerConfig, (message) => { @@ -861,7 +873,7 @@ export async function prepareEvalCaseWorkspace( } } - const caseEnvConfig = evalCase.workspace?.env; + const caseEnvConfig = runtimeWorkspace?.env; if (caseEnvConfig) { try { await runPreflightChecks(caseEnvConfig, workspacePath ?? evalDir, (message) => { @@ -896,7 +908,7 @@ export async function prepareEvalCaseWorkspace( case_input: evalCase.question, case_metadata: evalCase.metadata, eval_dir: evalDir ?? process.cwd(), - workspace_file_dir: evalCase.workspace?.workspaceFileDir, + workspace_file_dir: runtimeWorkspace?.workspaceFileDir, }, state: extensionState, }); @@ -915,7 +927,7 @@ export async function prepareEvalCaseWorkspace( } } - const caseBeforeAllHook = evalCase.workspace?.hooks?.before_all; + const caseBeforeAllHook = runtimeWorkspace?.hooks?.before_all; if (workspacePath && caseHooksEnabled && hasHookCommand(caseBeforeAllHook)) { const beforeAllHook = caseBeforeAllHook; const beforeAllCommand = (beforeAllHook.command ?? []).join(' '); @@ -931,7 +943,7 @@ export async function prepareEvalCaseWorkspace( caseInput: evalCase.question, caseMetadata: evalCase.metadata, evalDir, - workspaceFileDir: evalCase.workspace?.workspaceFileDir, + workspaceFileDir: runtimeWorkspace?.workspaceFileDir, }; try { beforeAllOutput = await executeWorkspaceScript( @@ -983,20 +995,20 @@ export async function prepareEvalCaseWorkspace( if ( caseHooksEnabled && workspacePath && - evalCase.workspace?.hooks?.before_each?.reset && - evalCase.workspace.hooks.before_each.reset !== 'none' + runtimeWorkspace?.hooks?.before_each?.reset && + runtimeWorkspace.hooks.before_each.reset !== 'none' ) { try { - if (repoManager && evalCase.workspace.repos?.length) { + if (repoManager && runtimeWorkspace.repos?.length) { await repoManager.reset( - evalCase.workspace.repos, + runtimeWorkspace.repos, workspacePath, - evalCase.workspace.hooks.before_each.reset, + runtimeWorkspace.hooks.before_each.reset, ); } else { await resetWorkspaceRoot( workspacePath, - evalCase.workspace.hooks.before_each.reset, + runtimeWorkspace.hooks.before_each.reset, sharedBaselineCommit, ); } @@ -1011,7 +1023,7 @@ export async function prepareEvalCaseWorkspace( } } - const caseBeforeEachHook = evalCase.workspace?.hooks?.before_each; + const caseBeforeEachHook = runtimeWorkspace?.hooks?.before_each; if (workspacePath && evalCase.extensions && evalCase.extensions.length > 0) { try { beforeEachNeedsFreshBaseline = hasExtensionHook(evalCase.extensions, 'beforeEach'); @@ -1026,7 +1038,7 @@ export async function prepareEvalCaseWorkspace( case_input: evalCase.question, case_metadata: evalCase.metadata, eval_dir: evalDir ?? process.cwd(), - workspace_file_dir: evalCase.workspace?.workspaceFileDir, + workspace_file_dir: runtimeWorkspace?.workspaceFileDir, }, state: extensionState, }); @@ -1055,7 +1067,7 @@ export async function prepareEvalCaseWorkspace( caseInput: evalCase.question, caseMetadata: evalCase.metadata, evalDir, - workspaceFileDir: evalCase.workspace?.workspaceFileDir, + workspaceFileDir: runtimeWorkspace?.workspaceFileDir, }; try { beforeEachOutput = await executeWorkspaceScript( @@ -1105,7 +1117,7 @@ export async function prepareEvalCaseWorkspace( caseInput: evalCase.question, caseMetadata: evalCase.metadata, evalDir, - workspaceFileDir: evalCase.workspace?.workspaceFileDir, + workspaceFileDir: runtimeWorkspace?.workspaceFileDir, }; try { await executeWorkspaceScript( diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 793eb4ba8..2f4035ecf 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -29,6 +29,11 @@ export { loadTsEvalFile, type TsEvalResult, } from './evaluation/loaders/ts-eval-loader.js'; +export type { + DockerEnvironmentRecipe, + EnvironmentRecipe, + HostEnvironmentRecipe, +} from './evaluation/loaders/environment-recipe.js'; export { transpileEvalYaml, transpileEvalYamlFile, diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 109aa9c15..d0bdbb45c 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -1031,8 +1031,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { 'timeout_seconds: 10', 'evaluate_options:', ' budget_usd: 0.5', - 'workspace:', - ' template: ./child-workspace', + 'environment:', + ' type: host', + ' workdir: ./child-workspace', 'assert:', ' - type: contains', ' value: child', @@ -1081,7 +1082,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig?.repeat).toMatchObject({ count: 3, strategy: 'pass_any' }); expect(test.run).toBeUndefined(); expect(test.suite).toBe('child-suite'); - expect(test.workspace?.template).toBe(path.join(tempDir, 'child-workspace')); + expect(test.environment?.workdir).toBe(path.join(tempDir, 'child-workspace')); expect(test.input.map((message) => message.content)).toEqual([ 'child shared input', 'child case input', @@ -1143,13 +1144,14 @@ describe('eval.yaml flat runtime controls and tests imports', () => { }); }); - it('rejects parent workspace when importing eval suites with type: suite', async () => { + it('rejects parent environment when importing eval suites with type: suite', async () => { await writeFile( path.join(tempDir, 'child.eval.yaml'), [ 'name: child-suite', - 'workspace:', - ' path: ./child-workspace', + 'environment:', + ' type: host', + ' workdir: ./child-workspace', 'prompts:', ' - "{{ input }}"', 'tests:', @@ -1164,8 +1166,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'workspace:', - ' template: ./parent-workspace', + 'environment:', + ' type: host', + ' workdir: ./parent-workspace', 'tests:', ' - include: child.eval.yaml', ' type: suite', @@ -1174,7 +1177,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ); await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow( - /Parent workspace is not allowed/, + /Parent environment is not allowed/, ); }); @@ -1432,8 +1435,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { path.join(tempDir, 'child.eval.yaml'), [ 'name: child-suite', - 'workspace:', - ' template: ./child-workspace', + 'environment:', + ' type: host', + ' workdir: ./child-workspace', 'assert:', ' - type: contains', ' value: child', @@ -1479,7 +1483,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.experimentConfig).toMatchObject({ target: 'codex-gpt5', threshold: 0.8 }); expect(byId.get('child-case')?.suite).toBe('child-suite'); expect(byId.get('child-case')?.source?.importedSuiteName).toBe('child-suite'); - expect(byId.get('child-case')?.workspace?.template).toBe(path.join(tempDir, 'child-workspace')); + expect(byId.get('child-case')?.environment?.workdir).toBe( + path.join(tempDir, 'child-workspace'), + ); expect(byId.get('child-case')?.input.map((message) => message.content)).toEqual([ 'child shared input', 'child case input', @@ -1506,8 +1512,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'workspace:', - ' template: ./parent-workspace', + 'environment:', + ' type: host', + ' workdir: ./parent-workspace', 'assert:', ' - type: contains', ' value: parent', @@ -1534,7 +1541,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.tests.map((test) => test.id)).toEqual(['jsonl-case', 'yaml-case', 'inline-case']); for (const id of ['jsonl-case', 'yaml-case', 'inline-case']) { expect(byId.get(id)?.suite).toBe('parent-suite'); - expect(byId.get(id)?.workspace?.template).toBe(path.join(tempDir, 'parent-workspace')); + expect(byId.get(id)?.environment?.workdir).toBe(path.join(tempDir, 'parent-workspace')); expect(byId.get(id)?.assertions?.[0]).toMatchObject({ type: 'contains', value: 'parent' }); } expect(byId.get('jsonl-case')?.input.map((message) => message.content)).toEqual([ @@ -1557,8 +1564,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'workspace:', - ' template: ./parent-workspace', + 'environment:', + ' type: host', + ' workdir: ./parent-workspace', 'imports:', ' tests:', ' - path: imported.jsonl', @@ -1573,12 +1581,12 @@ describe('eval.yaml flat runtime controls and tests imports', () => { expect(suite.tests.map((test) => test.id)).toEqual(['imported-case', 'local-case']); expect(byId.get('imported-case')?.suite).toBe('parent-suite'); expect(byId.get('local-case')?.suite).toBe('parent-suite'); - expect(byId.get('imported-case')?.workspace?.template).toBe( + expect(byId.get('imported-case')?.environment?.workdir).toBe( path.join(tempDir, 'parent-workspace'), ); }); - it('rejects parent workspace when imports.suites preserves child workspaces', async () => { + it('rejects parent environment when imports.suites preserves child environments', async () => { await writeFile( path.join(tempDir, 'child.eval.yaml'), [ @@ -1597,8 +1605,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'workspace:', - ' template: ./parent-workspace', + 'environment:', + ' type: host', + ' workdir: ./parent-workspace', 'imports:', ' suites:', ' - path: child.eval.yaml', @@ -1607,7 +1616,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ); await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow( - /Parent workspace is not allowed/, + /Parent environment is not allowed/, ); }); @@ -1716,8 +1725,9 @@ describe('eval.yaml flat runtime controls and tests imports', () => { parentPath, [ 'name: parent-suite', - 'workspace:', - ' template: ./parent-workspace', + 'environment:', + ' type: host', + ' workdir: ./parent-workspace', 'assert:', ' - type: contains', ' value: parent', @@ -1740,7 +1750,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { 'parent shared input', 'raw case input', ]); - expect(test.workspace?.template).toBe(path.join(tempDir, 'parent-workspace')); + expect(test.environment?.workdir).toBe(path.join(tempDir, 'parent-workspace')); expect(test.assertions?.[0]).toMatchObject({ type: 'contains', value: 'parent' }); }); }); diff --git a/packages/core/test/evaluation/extensions.test.ts b/packages/core/test/evaluation/extensions.test.ts index 541468c13..61909e73b 100644 --- a/packages/core/test/evaluation/extensions.test.ts +++ b/packages/core/test/evaluation/extensions.test.ts @@ -123,7 +123,7 @@ tests: ]); }); - it('runs lifecycle file hooks and exposes staged agent-rules paths to providers and results', async () => { + it('runs lifecycle file hooks with an internal runtime workspace and exposes staged agent-rules paths', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-extensions-run-')); tempDirs.push(dir); await mkdir(path.join(dir, 'template'), { recursive: true }); @@ -172,8 +172,6 @@ export function afterAll(context) { hook: beforeAll skills: rules/skills rules: rules/AGENTS.md -workspace: - template: template prompts: - "{{ input }}" tests: @@ -185,6 +183,10 @@ tests: 'utf8', ); const suite = await loadTestSuite(path.join(dir, 'suite.eval.yaml'), dir); + const evalCases = suite.tests.map((test) => ({ + ...test, + workspace: { template: path.join(dir, 'template') }, + })); const provider = new CapturingProvider(); const results = await runEvaluation({ @@ -193,7 +195,7 @@ tests: target, providerFactory: () => provider, evaluators: passEvaluators, - evalCases: suite.tests, + evalCases, maxConcurrency: 1, }); @@ -221,7 +223,7 @@ tests: expect(results[0].afterAllOutput).toContain('afterAll output'); }); - it('runs afterEach extensions and preserves extension metadata for conversation cases', async () => { + it('runs afterEach extensions with an internal runtime workspace for conversation cases', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-extensions-conversation-')); tempDirs.push(dir); await mkdir(path.join(dir, 'template'), { recursive: true }); @@ -246,8 +248,6 @@ export function afterEach(context) { hook: beforeAll skills: rules/skills - file://hooks.mjs:afterEach -workspace: - template: template prompts: - "{{ input }}" tests: @@ -261,6 +261,10 @@ tests: 'utf8', ); const suite = await loadTestSuite(path.join(dir, 'suite.eval.yaml'), dir); + const evalCases = suite.tests.map((test) => ({ + ...test, + workspace: { template: path.join(dir, 'template') }, + })); const provider = new CapturingProvider(); const results = await runEvaluation({ @@ -269,7 +273,7 @@ tests: target, providerFactory: () => provider, evaluators: passEvaluators, - evalCases: suite.tests, + evalCases, maxConcurrency: 1, }); @@ -282,7 +286,7 @@ tests: expect(results[0].afterEachOutput).toContain('conversation afterEach output'); }); - it('scopes beforeAll extension state to the suite workspace', async () => { + it('scopes beforeAll extension state to the internal suite workspace', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-extensions-suite-')); tempDirs.push(dir); const previousDataDir = process.env.AGENTV_DATA_DIR; @@ -298,11 +302,6 @@ tests: - id: agentv:agent-rules hook: beforeAll skills: rules/skills -workspace: - repos: - - path: ./repo-a - repo: file://${sourceRepo} - commit: ${commit} prompts: - "{{ input }}" tests: @@ -318,6 +317,12 @@ tests: 'utf8', ); const suite = await loadTestSuite(path.join(dir, 'suite.eval.yaml'), dir); + const evalCases = suite.tests.map((test) => ({ + ...test, + workspace: { + repos: [{ path: './repo-a', repo: `file://${sourceRepo}`, commit }], + }, + })); const requests: ProviderRequest[] = []; const provider = new CapturingProvider((request) => { requests.push(request); @@ -329,7 +334,7 @@ tests: target, providerFactory: () => provider, evaluators: passEvaluators, - evalCases: suite.tests, + evalCases, maxConcurrency: 2, }); @@ -353,7 +358,7 @@ tests: } }, 30_000); - it('refreshes the baseline after beforeEach extensions mutate files without state', async () => { + it('refreshes the internal workspace baseline after beforeEach extensions mutate files without state', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'agentv-extensions-baseline-')); tempDirs.push(dir); await mkdir(path.join(dir, 'template'), { recursive: true }); @@ -372,8 +377,6 @@ export function beforeEach(context) { path.join(dir, 'suite.eval.yaml'), `extensions: - file://hooks.mjs:beforeEach -workspace: - template: template prompts: - "{{ input }}" tests: @@ -385,6 +388,10 @@ tests: 'utf8', ); const suite = await loadTestSuite(path.join(dir, 'suite.eval.yaml'), dir); + const evalCases = suite.tests.map((test) => ({ + ...test, + workspace: { template: path.join(dir, 'template') }, + })); const provider = new CapturingProvider((request) => { if (!request.cwd) { throw new Error('cwd was not provided'); @@ -398,7 +405,7 @@ tests: target, providerFactory: () => provider, evaluators: passEvaluators, - evalCases: suite.tests, + evalCases, maxConcurrency: 1, }); diff --git a/packages/core/test/evaluation/interpolation-integration.test.ts b/packages/core/test/evaluation/interpolation-integration.test.ts index a5f191bdb..2c8af6245 100644 --- a/packages/core/test/evaluation/interpolation-integration.test.ts +++ b/packages/core/test/evaluation/interpolation-integration.test.ts @@ -41,9 +41,12 @@ describe('env interpolation in YAML loading', () => { await writeFile( evalFile, [ - 'workspace:', - ' repos:', - ' - path: ./RepoA', + 'environment:', + ' type: host', + ' workdir: ./RepoA', + ' setup:', + ' command: ./setup.sh', + ' args:', ' repo: "{{ env.AGENTV_TEST_PATH }}"', 'prompts:', ' - "{{ input }}"', @@ -57,17 +60,20 @@ describe('env interpolation in YAML loading', () => { const cases = await loadTests(evalFile, testDir); expect(cases[0].criteria).toBe('Must return correct answer'); - expect(cases[0].workspace?.repos?.[0]?.repo).toBe('https://github.com/org/from-env.git'); + expect(cases[0].environment?.setup?.args?.repo).toBe('https://github.com/org/from-env.git'); }); - it('interpolates {{ env.VAR }} in workspace repo identity', async () => { - const evalFile = path.join(testDir, 'interp-workspace.eval.yaml'); + it('interpolates {{ env.VAR }} in environment setup args', async () => { + const evalFile = path.join(testDir, 'interp-environment.eval.yaml'); await writeFile( evalFile, [ - 'workspace:', - ' repos:', - ' - path: ./RepoA', + 'environment:', + ' type: host', + ' workdir: ./RepoA', + ' setup:', + ' command: ./setup.sh', + ' args:', ' repo: "{{ env.AGENTV_TEST_PATH }}"', 'prompts:', ' - "{{ input }}"', @@ -79,20 +85,28 @@ describe('env interpolation in YAML loading', () => { ].join('\n'), ); const cases = await loadTests(evalFile, testDir); - expect(cases[0].workspace?.repos?.[0]?.repo).toBe('https://github.com/org/from-env.git'); + expect(cases[0].environment?.setup?.args?.repo).toBe('https://github.com/org/from-env.git'); }); - it('interpolates {{ env.VAR }} in external workspace YAML file', async () => { - const workspaceFile = path.join(testDir, 'workspace.yaml'); + it('interpolates {{ env.VAR }} in external environment YAML file', async () => { + const environmentFile = path.join(testDir, 'environment.yaml'); await writeFile( - workspaceFile, - ['repos:', ' - path: ./RepoB', ' repo: "{{ env.AGENTV_TEST_PATH }}"', ''].join('\n'), + environmentFile, + [ + 'type: host', + 'workdir: ./RepoB', + 'setup:', + ' command: ./setup.sh', + ' args:', + ' repo: "{{ env.AGENTV_TEST_PATH }}"', + '', + ].join('\n'), ); - const evalFile = path.join(testDir, 'interp-ext-workspace.eval.yaml'); + const evalFile = path.join(testDir, 'interp-ext-environment.eval.yaml'); await writeFile( evalFile, [ - 'workspace: workspace.yaml', + 'environment: file://environment.yaml', 'prompts:', ' - "{{ input }}"', 'tests:', @@ -103,7 +117,7 @@ describe('env interpolation in YAML loading', () => { ].join('\n'), ); const cases = await loadTests(evalFile, testDir); - expect(cases[0].workspace?.repos?.[0]?.repo).toBe('https://github.com/org/from-env.git'); + expect(cases[0].environment?.setup?.args?.repo).toBe('https://github.com/org/from-env.git'); }); it('interpolates {{ env.VAR }} in external YAML case files', async () => { diff --git a/packages/core/test/evaluation/loaders/case-file-loader.test.ts b/packages/core/test/evaluation/loaders/case-file-loader.test.ts index 9b763153e..4efc5ab7a 100644 --- a/packages/core/test/evaluation/loaders/case-file-loader.test.ts +++ b/packages/core/test/evaluation/loaders/case-file-loader.test.ts @@ -815,7 +815,7 @@ input: "Input" expect(ws.template).toBe(path.join(casesDir, 'my-case', 'workspace')); }); - it('does not override explicit workspace in case.yaml', async () => { + it('does not override explicit legacy internal workspace in case.yaml', async () => { const casesDir = path.join(tempDir, 'ws-explicit'); await mkdir(path.join(casesDir, 'my-case', 'workspace'), { recursive: true }); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index c83bf3e87..73b9aeb8b 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -2949,7 +2949,7 @@ describe('required gates', () => { }); }); -describe('workspace.template .code-workspace resolution', () => { +describe('internal workspace.template .code-workspace resolution', () => { let testDir: string; afterEach(async () => { @@ -2959,7 +2959,7 @@ describe('workspace.template .code-workspace resolution', () => { } }); - it('threads workspaceFile to provider when workspace.template is a .code-workspace file', async () => { + it('threads workspaceFile to provider when internal workspace.template is a .code-workspace file', async () => { const { mkdtemp, writeFile, mkdir, rm } = await import('node:fs/promises'); testDir = await mkdtemp(path.join(tmpdir(), 'agentv-orch-ws-resolve-')); @@ -2974,7 +2974,7 @@ describe('workspace.template .code-workspace resolution', () => { output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }], }); - // Point workspace.template at the .code-workspace FILE (not directory) + // Point internal workspace.template at the .code-workspace FILE (not directory) const evalCase: EvalTest = { ...baseTestCase, workspace: { @@ -3030,7 +3030,7 @@ describe('workspace.template .code-workspace resolution', () => { output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }], }); - // Point workspace.template at the DIRECTORY (should auto-detect the single .code-workspace) + // Point internal workspace.template at the DIRECTORY (should auto-detect the single .code-workspace) const evalCase: EvalTest = { ...baseTestCase, workspace: { @@ -3794,7 +3794,7 @@ fs.writeFileSync(path.join(payload.workspace_path, 'hook.txt'), payload.test_id const missingRepoBSource = path.join(testDir, 'missing-repo-b-source'); // Runtime workspacePath points at an existing machine-local workspace. It is used - // as-is and does not materialize workspace.repos from the portable eval contract. + // as-is and does not materialize internal workspace.repos. const evalCase: EvalTest = { ...baseTestCase, workspace: { diff --git a/packages/core/test/evaluation/prepared-workspace.test.ts b/packages/core/test/evaluation/prepared-workspace.test.ts index cc0895fe5..37ba97cca 100644 --- a/packages/core/test/evaluation/prepared-workspace.test.ts +++ b/packages/core/test/evaluation/prepared-workspace.test.ts @@ -139,7 +139,7 @@ describe('prepareEvalWorkspace', () => { expect(prepared.promptSource.question).toBe('Implement the requested change'); }); - it('materializes workspace.repos using the same repo setup path', async () => { + it('materializes internal workspace.repos using the same repo setup path', async () => { const repoDir = path.join(tmpDir, 'source-repo'); const commit = createTestRepo(repoDir, { 'README.md': '# source repo\n', diff --git a/packages/core/test/evaluation/repo-schema-validation.test.ts b/packages/core/test/evaluation/repo-schema-validation.test.ts index 4a6222c0e..9b7ef9045 100644 --- a/packages/core/test/evaluation/repo-schema-validation.test.ts +++ b/packages/core/test/evaluation/repo-schema-validation.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { EvalFileSchema } from '../../src/evaluation/validation/eval-file.schema.js'; -describe('repo lifecycle schema validation', () => { +describe('environment recipe schema validation', () => { const baseEval = { description: 'test', prompts: ['{{ input }}'], @@ -14,46 +14,55 @@ describe('repo lifecycle schema validation', () => { ], }; - it('accepts workspace repos with provenance fields', () => { + it('accepts host environment setup args with repo provenance fields', () => { const result = EvalFileSchema.safeParse({ ...baseEval, - workspace: { - repos: [ - { - path: './repo-a', + environment: { + type: 'host', + workdir: './repo-a', + setup: { + command: './setup.sh', + args: { repo: 'https://github.com/org/repo.git', commit: 'main', ancestor: 1, sparse: ['src', 'package.json'], }, - ], + }, }, }); expect(result.success).toBe(true); }); - it('accepts GitHub org/name shorthand', () => { + it('accepts GitHub org/name shorthand in environment setup args', () => { const result = EvalFileSchema.safeParse({ ...baseEval, - workspace: { - repos: [ - { - path: './repo-a', + environment: { + type: 'host', + workdir: './repo-a', + setup: { + command: './setup.sh', + args: { repo: 'org/repo', commit: '4a1b2c3d', }, - ], + }, }, }); expect(result.success).toBe(true); }); - it('accepts Docker repo hints without repo identity', () => { + it('accepts Docker environment setup args without repo identity', () => { const result = EvalFileSchema.safeParse({ ...baseEval, - workspace: { - docker: { image: 'swebench/sweb.eval.django__django:latest' }, - repos: [{ path: '/testbed', commit: 'abc123' }], + environment: { + type: 'docker', + image: 'swebench/sweb.eval.django__django:latest', + workdir: '/testbed', + setup: { + command: './setup.sh', + args: { commit: 'abc123' }, + }, }, }); expect(result.success).toBe(true); @@ -124,52 +133,36 @@ describe('repo lifecycle schema validation', () => { expect(result.success).toBe(false); }); - it('rejects negative ancestor', () => { + it('rejects unknown environment fields', () => { const result = EvalFileSchema.safeParse({ ...baseEval, - workspace: { - repos: [ - { - path: './repo-a', - repo: 'https://github.com/org/repo.git', - ancestor: -1, - }, - ], + environment: { + type: 'host', + workdir: './repo-a', + repos: [{ repo: 'https://github.com/org/repo.git' }], }, }); expect(result.success).toBe(false); }); - it('accepts workspace with hooks after_each reset config', () => { + it('accepts internal workspace hooks after_each reset config', () => { const result = EvalFileSchema.safeParse({ ...baseEval, workspace: { - repos: [ - { - path: './repo-a', - repo: 'https://github.com/org/repo.git', - }, - ], hooks: { after_each: { reset: 'fast' } }, }, }); expect(result.success).toBe(true); }); - it('accepts workspace with scope field', () => { + it('rejects public workspace scope field', () => { const result = EvalFileSchema.safeParse({ ...baseEval, workspace: { scope: 'attempt', - repos: [ - { - path: './repo-a', - repo: 'https://github.com/org/repo.git', - }, - ], }, }); - expect(result.success).toBe(true); + expect(result.success).toBe(false); }); it('rejects removed workspace isolation per_test value', () => { diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index a6cbd6f09..98a34b231 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -942,8 +942,9 @@ tests: it('warns when imports.tests-style raw imports drop eval suite context', async () => { await writeFile( path.join(tempDir, 'composition-child-tests-import.eval.yaml'), - `workspace: - template: ./child-workspace + `environment: + type: host + workdir: ./child-workspace input: child suite input assert: - type: contains @@ -957,8 +958,9 @@ tests: const filePath = path.join(tempDir, 'composition-parent-tests-import.eval.yaml'); await writeFile( filePath, - `workspace: - template: ./parent-workspace + `environment: + type: host + workdir: ./parent-workspace tests: - include: composition-child-tests-import.eval.yaml type: tests @@ -2199,17 +2201,15 @@ tests: }); }); - describe('workspace repo validation', () => { - it('errors when legacy source is set', async () => { - const filePath = path.join(tempDir, 'workspace-legacy-source-error.yaml'); + describe('environment recipe validation and workspace hard-deprecation', () => { + it('errors when public suite workspace repos are authored', async () => { + const filePath = path.join(tempDir, 'workspace-repos-error.yaml'); await writeFile( filePath, `workspace: repos: - path: ./repo - source: - type: git - url: https://github.com/org/repo.git + repo: https://github.com/org/repo.git tests: - id: test-1 criteria: Goal @@ -2224,13 +2224,13 @@ tests: result.errors.some( (e) => e.severity === 'error' && - e.message.includes('workspace.repos[].source has been removed'), + e.message.includes('workspace.repos has been removed from public eval YAML'), ), ).toBe(true); }); - it('errors when legacy checkout is set in a per-case workspace', async () => { - const filePath = path.join(tempDir, 'workspace-legacy-checkout-error.yaml'); + it('errors when public per-case workspace repos are authored', async () => { + const filePath = path.join(tempDir, 'case-workspace-repos-error.yaml'); await writeFile( filePath, `tests: @@ -2241,8 +2241,6 @@ tests: repos: - path: ./repo repo: https://github.com/org/repo.git - checkout: - ref: main `, ); @@ -2253,91 +2251,76 @@ tests: result.errors.some( (e) => e.severity === 'error' && - e.message.includes('workspace.repos[].checkout has been removed'), + e.message.includes('workspace.repos has been removed from public eval YAML'), ), ).toBe(true); }); - it('errors when legacy clone is set', async () => { - const filePath = path.join(tempDir, 'workspace-legacy-clone-error.yaml'); + it('accepts host environment setup args with repo provenance', async () => { + const filePath = path.join(tempDir, 'environment-host.yaml'); await writeFile( filePath, - `workspace: - repos: - - path: ./repo + `prompts: + - "{{ prompt }}" +environment: + type: host + workdir: ./repo + setup: + command: ./setup.sh + args: repo: https://github.com/org/repo.git - clone: - depth: 1 + commit: main + sparse: + include: src/** tests: - id: test-1 criteria: Goal - input: "Query" + vars: + prompt: "Query" `, ); const result = await validateEvalFile(filePath); - expect(result.valid).toBe(false); - expect( - result.errors.some( - (e) => - e.severity === 'error' && - e.message.includes('workspace.repos[].clone has been removed'), - ), - ).toBe(true); + expect(result.valid).toBe(true); + expect(result.errors.filter((e) => e.severity === 'error')).toHaveLength(0); }); - it('errors when removed repo acquisition fields are set', async () => { - const filePath = path.join(tempDir, 'workspace-removed-acquisition-fields-error.yaml'); + it('accepts Docker environment setup args without repo identity', async () => { + const filePath = path.join(tempDir, 'environment-docker.yaml'); await writeFile( filePath, - `workspace: - repos: - - path: ./repo - repo: https://github.com/org/repo.git - type: git - resolve: custom - resolver: custom + `prompts: + - "{{ prompt }}" +environment: + type: docker + image: swebench/sweb.eval.django__django:latest + workdir: /testbed + setup: + command: ./setup.sh + args: + commit: abc123 tests: - id: test-1 criteria: Goal - input: "Query" + vars: + prompt: "Query" `, ); const result = await validateEvalFile(filePath); - expect(result.valid).toBe(false); - expect( - result.errors.some( - (e) => - e.severity === 'error' && e.message.includes('workspace.repos[].type has been removed'), - ), - ).toBe(true); - expect( - result.errors.some( - (e) => - e.severity === 'error' && - e.message.includes('workspace.repos[].resolve has been removed'), - ), - ).toBe(true); - expect( - result.errors.some( - (e) => - e.severity === 'error' && - e.message.includes('workspace.repos[].resolver has been removed'), - ), - ).toBe(true); + expect(result.valid).toBe(true); + expect(result.errors.filter((e) => e.severity === 'error')).toHaveLength(0); }); - it('errors when non-Docker repo omits repo identity', async () => { - const filePath = path.join(tempDir, 'workspace-missing-repo-error.yaml'); + it('errors when Docker environment omits both image and context', async () => { + const filePath = path.join(tempDir, 'environment-docker-missing-source.yaml'); await writeFile( filePath, - `workspace: - repos: - - path: ./repo - commit: main + `environment: + type: docker + workdir: /testbed tests: - id: test-1 criteria: Goal @@ -2351,53 +2334,27 @@ tests: expect( result.errors.some( (e) => - e.severity === 'error' && e.message.includes('repos[].repo is required for non-Docker'), + e.severity === 'error' && + e.location === 'environment' && + e.message.includes("docker recipes must define either 'image' or 'context'"), ), ).toBe(true); }); - it('allows Docker repo hints without repo identity', async () => { - const filePath = path.join(tempDir, 'workspace-docker-repo-hint.yaml'); - await writeFile( - filePath, - `prompts: - - "{{ prompt }}" -workspace: - docker: - image: swebench/sweb.eval.django__django:latest - repos: - - path: /testbed - commit: abc123 -tests: - - id: test-1 - criteria: Goal - vars: - prompt: "Query" -`, - ); - - const result = await validateEvalFile(filePath); - - expect(result.valid).toBe(true); - expect(result.errors.filter((e) => e.severity === 'error')).toHaveLength(0); - }); - - it('errors when an external workspace file uses legacy source', async () => { - const workspaceFile = path.join(tempDir, 'external-workspace.yaml'); + it('errors when an external environment file wraps the recipe', async () => { + const environmentFile = path.join(tempDir, 'external-environment.yaml'); await writeFile( - workspaceFile, - `repos: - - path: ./repo - source: - type: git - url: https://github.com/org/repo.git + environmentFile, + `environment: + type: host + workdir: ./repo `, ); - const filePath = path.join(tempDir, 'workspace-legacy-source-external-error.yaml'); + const filePath = path.join(tempDir, 'environment-wrapped-external-error.yaml'); await writeFile( filePath, - `workspace: ./external-workspace.yaml + `environment: file://external-environment.yaml tests: - id: test-1 criteria: Goal @@ -2411,18 +2368,17 @@ tests: expect( result.errors.some( (e) => - e.filePath === workspaceFile && e.severity === 'error' && - e.message.includes('workspace.repos[].source has been removed'), + e.message.includes('must contain the environment recipe directly'), ), ).toBe(true); }); - it('rejects a missing external workspace file', async () => { - const filePath = path.join(tempDir, 'workspace-missing-external.yaml'); + it('rejects a missing external environment file', async () => { + const filePath = path.join(tempDir, 'environment-missing-external.yaml'); await writeFile( filePath, - `workspace: ./does-not-exist.yaml + `environment: file://does-not-exist.yaml tests: - id: test-1 criteria: Goal @@ -2436,7 +2392,8 @@ tests: expect( result.errors.some( (e) => - e.severity === 'error' && e.message.includes('Failed to load external workspace file'), + e.severity === 'error' && + e.message.includes('environment recipe file not found or unreadable'), ), ).toBe(true); }); diff --git a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts index e164c5987..81cba92c0 100644 --- a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts +++ b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts @@ -11,7 +11,7 @@ const minimalEvalPrefix = `tests: input: hello `; -describe('validateWorkspacePaths', () => { +describe('validateWorkspacePaths legacy internal workspace checks', () => { let tempDir: string; beforeAll(async () => { @@ -30,7 +30,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('returns no errors when workspace file reference exists (with no internal paths)', async () => { + it('returns no errors when a legacy internal workspace file reference exists', async () => { const wsFilePath = path.join(tempDir, 'workspace.yaml'); await writeFile(wsFilePath, 'template: ~\n'); @@ -41,7 +41,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('returns no errors when workspace file is missing (eval-validator owns that check)', async () => { + it('returns no errors when legacy workspace file is missing (eval-validator owns that check)', async () => { const evalFilePath = path.join(tempDir, 'eval-ws-ref-missing.yaml'); await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: missing-workspace.yaml\n`); @@ -50,7 +50,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('errors when workspace.template does not exist (inline workspace)', async () => { + it('errors when legacy internal workspace.template does not exist', async () => { const evalFilePath = path.join(tempDir, 'eval-inline-template.yaml'); await writeFile( evalFilePath, @@ -64,7 +64,7 @@ describe('validateWorkspacePaths', () => { expect(errors[0]?.message).toContain('Template path not found'); }); - it('returns no errors when workspace.template exists', async () => { + it('returns no errors when legacy internal workspace.template exists', async () => { const templateDir = path.join(tempDir, 'my-template'); await mkdir(templateDir, { recursive: true }); @@ -75,7 +75,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('errors when hook before_all command has a missing relative script', async () => { + it('errors when legacy internal workspace hook command has a missing script', async () => { const evalFilePath = path.join(tempDir, 'eval-hook-missing.yaml'); await writeFile( evalFilePath, @@ -95,7 +95,7 @@ describe('validateWorkspacePaths', () => { expect(errors[0]?.message).toContain('setup.mjs'); }); - it('returns no errors when hook script exists at resolved path', async () => { + it('returns no errors when legacy internal workspace hook script exists', async () => { const scriptsDir = path.join(tempDir, 'scripts'); await mkdir(scriptsDir, { recursive: true }); const setupScript = path.join(scriptsDir, 'setup.mjs'); @@ -135,7 +135,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('checks hooks inside external workspace file', async () => { + it('checks hooks inside a legacy external workspace file', async () => { const wsFilePath = path.join(tempDir, 'ws-with-hooks.yaml'); await writeFile( wsFilePath, @@ -155,7 +155,7 @@ describe('validateWorkspacePaths', () => { expect(errors[0]?.message).toContain('missing-setup.mjs'); }); - it('checks template inside external workspace file', async () => { + it('checks template inside a legacy external workspace file', async () => { const wsFilePath = path.join(tempDir, 'ws-with-template.yaml'); await writeFile(wsFilePath, 'template: ./missing-template\n'); diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index 72f0c1704..fea972ced 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -234,7 +234,7 @@ tests: expect(cases[0].workspace?.hooks?.before_all?.cwd).toBe(path.join(testDir, 'scripts')); }); - it('should parse workspace template path', async () => { + it('rejects public workspace template authoring', async () => { const evalFile = path.join(testDir, 'workspace-template.yaml'); await writeFile( evalFile, @@ -250,13 +250,13 @@ tests: `, ); - const cases = await loadTests(evalFile, testDir); - expect(cases).toHaveLength(1); - expect(cases[0].workspace?.template).toBe(path.join(testDir, 'workspace-template')); + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /workspace\.template has been removed from public eval YAML/, + ); }); - it('should parse Docker repos without source (prebuilt image)', async () => { - const evalFile = path.join(testDir, 'workspace-docker-no-source.yaml'); + it('parses Docker environment recipes without repo identity', async () => { + const evalFile = path.join(testDir, 'environment-docker-no-source.yaml'); await writeFile( evalFile, `prompts: @@ -264,11 +264,13 @@ tests: tests: - id: docker-no-source criteria: Should work - workspace: - docker: - image: swebench/sweb.eval.django__django:latest - repos: - - path: /testbed + environment: + type: docker + image: swebench/sweb.eval.django__django:latest + workdir: /testbed + setup: + command: ./setup.sh + args: commit: abc123def vars: input: Do something @@ -277,17 +279,19 @@ tests: const cases = await loadTests(evalFile, testDir); expect(cases).toHaveLength(1); - expect(cases[0].workspace?.docker).toEqual({ + expect(cases[0].environment).toMatchObject({ + type: 'docker', image: 'swebench/sweb.eval.django__django:latest', + workdir: '/testbed', + setup: { + command: './setup.sh', + args: { commit: 'abc123def' }, + }, }); - expect(cases[0].workspace?.repos).toHaveLength(1); - expect(cases[0].workspace?.repos?.[0].path).toBe('/testbed'); - expect(cases[0].workspace?.repos?.[0].repo).toBeUndefined(); - expect(cases[0].workspace?.repos?.[0].commit).toBe('abc123def'); }); - it('should parse Docker repos with path + commit but no repo', async () => { - const evalFile = path.join(testDir, 'workspace-repo-path-checkout-only.yaml'); + it('parses Docker environment setup args with path + commit metadata', async () => { + const evalFile = path.join(testDir, 'environment-path-checkout-only.yaml'); await writeFile( evalFile, `prompts: @@ -295,11 +299,13 @@ tests: tests: - id: path-checkout-only criteria: Should work - workspace: - docker: - image: myimage:latest - repos: - - path: /workspace/project + environment: + type: docker + image: myimage:latest + workdir: /workspace/project + setup: + command: ./setup.sh + args: commit: v2.0.0 vars: input: Do something @@ -308,12 +314,15 @@ tests: const cases = await loadTests(evalFile, testDir); expect(cases).toHaveLength(1); - expect(cases[0].workspace?.repos?.[0].path).toBe('/workspace/project'); - expect(cases[0].workspace?.repos?.[0].repo).toBeUndefined(); - expect(cases[0].workspace?.repos?.[0].commit).toBe('v2.0.0'); + expect(cases[0].environment).toMatchObject({ + type: 'docker', + image: 'myimage:latest', + workdir: '/workspace/project', + setup: { args: { commit: 'v2.0.0' } }, + }); }); - it('parses workspace repos from YAML', async () => { + it('rejects public workspace repos authoring', async () => { const evalFile = path.join(testDir, 'workspace-repos.yaml'); await writeFile( evalFile, @@ -336,14 +345,9 @@ tests: `, ); - const cases = await loadTests(evalFile, testDir); - const workspace = cases[0].workspace; - expect(workspace?.repos).toHaveLength(1); - expect(workspace?.repos?.[0].path).toBe('./repo-a'); - expect(workspace?.repos?.[0].repo).toBe('https://github.com/org/repo.git'); - expect(workspace?.repos?.[0].commit).toBe('main'); - expect(workspace?.repos?.[0].ancestor).toBe(1); - expect(workspace?.repos?.[0].sparse).toEqual(['src/**']); + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /workspace\.repos has been removed from public eval YAML/, + ); }); it('rejects removed workspace repo resolver field', async () => { @@ -367,11 +371,11 @@ tests: ); await expect(loadTests(evalFile, testDir)).rejects.toThrow( - 'workspace.repos[].resolver has been removed', + /workspace\.repos has been removed from public eval YAML/, ); }); - it('parses workspace hooks after_each reset config', async () => { + it('parses legacy internal workspace hooks after_each reset config', async () => { const evalFile = path.join(testDir, 'workspace-reset.yaml'); await writeFile( evalFile, @@ -394,16 +398,13 @@ tests: expect(cases[0].workspace?.hooks?.after_each?.reset).toBe('fast'); }); - it('parses workspace scope field', async () => { + it('rejects public workspace scope authoring', async () => { const evalFile = path.join(testDir, 'workspace-scope.yaml'); await writeFile( evalFile, `description: test workspace: scope: attempt - repos: - - path: ./repo-a - repo: https://github.com/org/repo.git prompts: - "{{ input }}" tests: @@ -414,8 +415,9 @@ tests: `, ); - const cases = await loadTests(evalFile, testDir); - expect(cases[0].workspace?.scope).toBe('attempt'); + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /workspace\.scope has been removed from public eval YAML/, + ); }); it('rejects removed workspace isolation field', async () => { @@ -574,11 +576,6 @@ tests: await writeFile( workspaceFile, ` -template: ./workspace-template -repos: - - path: ./my-repo - repo: https://github.com/org/repo.git - commit: main hooks: after_each: reset: fast @@ -609,11 +606,6 @@ tests: // Both cases inherit the external workspace config for (const c of cases) { expect(c.workspace).toBeDefined(); - // template resolved relative to workspace file's directory - expect(c.workspace?.template).toBe(path.join(wsDir, 'workspace-template')); - expect(c.workspace?.repos).toHaveLength(1); - expect(c.workspace?.repos?.[0].repo).toBe('https://github.com/org/repo.git'); - expect(c.workspace?.repos?.[0].commit).toBe('main'); expect(c.workspace?.hooks?.after_each?.reset).toBe('fast'); } }); @@ -626,7 +618,6 @@ tests: await writeFile( workspaceFile, ` -template: ./my-template hooks: before_all: command: ["node", "setup.mjs"] @@ -650,8 +641,6 @@ tests: const cases = await loadTests(evalFile, testDir); expect(cases).toHaveLength(1); - // template resolved relative to workspace file dir (nested/config/) - expect(cases[0].workspace?.template).toBe(path.join(wsDir, 'my-template')); // cwd resolved relative to workspace file dir expect(cases[0].workspace?.hooks?.before_all?.cwd).toBe(path.join(wsDir, 'scripts')); // workspaceFileDir is set to the workspace file's directory @@ -736,7 +725,7 @@ tests: ); }); - it('should throw a clear error when external workspace file wraps config under workspace', async () => { + it('should throw a clear error when legacy external workspace file wraps config under workspace', async () => { const wsDir = path.join(testDir, 'wrapped-workspace'); await mkdir(wsDir, { recursive: true }); @@ -778,7 +767,6 @@ tests: await writeFile( workspaceFile, ` -template: ./base-template hooks: before_all: command: ["node", "base-setup.mjs"] @@ -820,16 +808,14 @@ tests: 'node', 'base-setup.mjs', ]); - expect(defaultCase?.workspace?.template).toBe(path.join(wsDir, 'base-template')); expect(defaultCase?.workspace?.hooks?.after_each?.reset).toBe('fast'); - // override-case: before_all replaced, template and after_each inherited + // override-case: before_all replaced, after_each inherited const overrideCase = cases.find((c) => c.id === 'override-case'); expect(overrideCase?.workspace?.hooks?.before_all?.command).toEqual([ 'node', 'custom-setup.mjs', ]); - expect(overrideCase?.workspace?.template).toBe(path.join(wsDir, 'base-template')); expect(overrideCase?.workspace?.hooks?.after_each?.reset).toBe('fast'); }); }); diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts index 61c425f35..f3487b79d 100644 --- a/packages/core/test/evaluation/workspace/deps-scanner.test.ts +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -5,7 +5,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { scanRepoDeps } from '../../../src/evaluation/workspace/deps-scanner.js'; -describe('scanRepoDeps', () => { +describe('scanRepoDeps legacy internal workspace compatibility', () => { let tempDir: string; beforeAll(async () => { @@ -23,7 +23,7 @@ describe('scanRepoDeps', () => { return filePath; } - it('extracts repos from suite-level workspace', async () => { + it('extracts repos from legacy suite-level workspace config', async () => { const file = await writeYaml( 'suite-level.eval.yaml', ` @@ -52,7 +52,7 @@ tests: expect(result.repos[0].usedBy).toEqual([file]); }); - it('extracts repos from per-case workspace', async () => { + it('extracts repos from legacy per-case workspace config', async () => { const file = await writeYaml( 'per-test.eval.yaml', ` @@ -173,7 +173,7 @@ tests: expect(urls).toContain('https://github.com/org/repo.git@develop'); }); - it('resolves external workspace file references', async () => { + it('resolves legacy external workspace file references', async () => { await writeYaml( 'shared/workspace.yaml', ` diff --git a/packages/sdk/src/eval.ts b/packages/sdk/src/eval.ts index 762ff42bb..35cbe9c43 100644 --- a/packages/sdk/src/eval.ts +++ b/packages/sdk/src/eval.ts @@ -32,6 +32,7 @@ const KNOWN_SNAKE_CASE_KEYS = { onDependencyFailure: 'on_dependency_failure', onTurnFailure: 'on_turn_failure', outputPath: 'output_path', + readOnly: 'read_only', reasoningEffort: 'reasoning_effort', scoreRange: 'score_range', scoreRanges: 'score_ranges', @@ -115,6 +116,49 @@ export interface EvalWorkspace { readonly docker?: EvalDockerWorkspace; } +export interface EvalEnvironmentSetup { + readonly command: string | readonly string[]; + readonly args?: Readonly>; + readonly env?: Readonly>; + readonly timeoutSeconds?: number; +} + +export interface EvalDockerEnvironmentMount { + readonly source: string; + readonly target: string; + readonly access?: 'ro' | 'rw'; + readonly readOnly?: boolean; +} + +export interface EvalDockerEnvironmentResources { + readonly cpus?: number; + readonly memory?: string; + readonly disk?: string; + readonly gpu?: boolean | string; +} + +export interface EvalHostEnvironment { + readonly type: 'host'; + readonly workdir: string; + readonly setup?: EvalEnvironmentSetup; + readonly env?: Readonly>; +} + +export interface EvalDockerEnvironment { + readonly type: 'docker'; + readonly workdir: string; + readonly context?: string; + readonly dockerfile?: string; + readonly image?: string; + readonly setup?: EvalEnvironmentSetup; + readonly env?: Readonly>; + readonly resources?: EvalDockerEnvironmentResources; + readonly mounts?: readonly EvalDockerEnvironmentMount[]; + readonly secrets?: Readonly>; +} + +export type EvalEnvironment = EvalHostEnvironment | EvalDockerEnvironment; + export interface EvalTargetRef { readonly label: string; readonly id?: string; @@ -174,6 +218,7 @@ export interface EvalTest { readonly expectedOutput?: string | Readonly> | readonly EvalMessage[]; readonly assert?: readonly EvalAssertionConfig[]; readonly execution?: EvalExecution; + readonly environment?: EvalEnvironment | string; readonly workspace?: EvalWorkspace; readonly metadata?: Readonly>; readonly conversationId?: string; @@ -223,6 +268,7 @@ export interface EvalDefinition { readonly threshold?: number; readonly budgetUsd?: number; readonly assert?: readonly EvalAssertionConfig[]; + readonly environment?: EvalEnvironment | string; readonly workspace?: EvalWorkspace | string; } diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 446d63468..351bf4dde 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -146,8 +146,14 @@ export { type DefinedEvalSuite, type EvalAssertionConfig, type EvalDefinition, + type EvalDockerEnvironment, + type EvalDockerEnvironmentMount, + type EvalDockerEnvironmentResources, type EvalDockerWorkspace, + type EvalEnvironment, + type EvalEnvironmentSetup, type EvalExecution, + type EvalHostEnvironment, type EvalMessage, type EvalMessageContent, type EvalRequires, diff --git a/packages/sdk/test/eval-authoring.test.ts b/packages/sdk/test/eval-authoring.test.ts index 6fceee2a8..e4cd3c776 100644 --- a/packages/sdk/test/eval-authoring.test.ts +++ b/packages/sdk/test/eval-authoring.test.ts @@ -47,18 +47,12 @@ describe('YAML-aligned eval authoring helpers', () => { vars: { input: 'Say hello.' }, inputFiles: ['fixtures/prompt.md'], expectedOutput: 'Hello there', - workspace: { - hooks: { - beforeEach: { - command: 'git reset --hard', - timeoutMs: 5_000, - }, - afterEach: { - command: ['git', 'status'], - }, - afterAll: { - command: ['echo', 'done'], - }, + environment: { + type: 'host', + workdir: 'fixtures/workspace', + setup: { + command: ['bun', 'scripts/setup.ts'], + timeoutSeconds: 5, }, }, mode: 'conversation', @@ -139,18 +133,12 @@ describe('YAML-aligned eval authoring helpers', () => { vars: { input: 'Say hello.' }, input_files: ['fixtures/prompt.md'], expected_output: 'Hello there', - workspace: { - hooks: { - before_each: { - command: 'git reset --hard', - timeout_ms: 5_000, - }, - after_each: { - command: ['git', 'status'], - }, - after_all: { - command: ['echo', 'done'], - }, + environment: { + type: 'host', + workdir: 'fixtures/workspace', + setup: { + command: ['bun', 'scripts/setup.ts'], + timeout_seconds: 5, }, }, mode: 'conversation',