diff --git a/packages/core/src/evaluation/graders/llm-grader.ts b/packages/core/src/evaluation/graders/llm-grader.ts index 246aa3c59..198c40a5e 100644 --- a/packages/core/src/evaluation/graders/llm-grader.ts +++ b/packages/core/src/evaluation/graders/llm-grader.ts @@ -119,6 +119,29 @@ const freeformEvaluationSchema = z.object({ .optional(), }); +const promptfooCheckSchema = z + .object({ + id: z.string().optional(), + text: z.string().optional(), + pass: z.union([z.boolean(), z.string(), z.number()]).optional(), + passed: z.union([z.boolean(), z.string(), z.number()]).optional(), + satisfied: z.union([z.boolean(), z.string(), z.number()]).optional(), + score: z.union([z.number(), z.string(), z.boolean()]).optional(), + reason: z.string().optional(), + evidence: z.string().optional(), + reasoning: z.string().optional(), + }) + .passthrough(); + +const promptfooGradingResultSchema = z + .object({ + reason: z.string().optional(), + pass: z.union([z.boolean(), z.string(), z.number()]).optional(), + score: z.union([z.number(), z.string(), z.boolean()]).optional(), + checks: z.array(promptfooCheckSchema).optional(), + }) + .passthrough(); + const rubricCheckResultSchema = z.object({ id: z.string().describe('The ID of the rubric item being checked'), satisfied: z.boolean().describe('Whether this rubric requirement is met'), @@ -153,6 +176,13 @@ interface StructuredGenerationResult { readonly tokenUsage?: TokenUsage; } +interface NormalizedPromptfooGradingResult { + readonly score: number; + readonly verdict: import('../types.js').EvaluationVerdict; + readonly assertions: readonly AssertionEntry[]; + readonly details: JsonObject; +} + function stringifyPretty(value: unknown): string { return value === undefined ? '' : JSON.stringify(value, null, 2); } @@ -215,6 +245,174 @@ function resolveContentBasePath(context: EvaluationContext): string | undefined return undefined; } +function isJsonRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function hasPromptfooAggregateFields(value: unknown): boolean { + if (!isJsonRecord(value)) { + return false; + } + return ( + 'pass' in value || + 'reason' in value || + ('checks' in value && !('overall_reasoning' in value) && !('assertions' in value)) + ); +} + +function normalizePromptfooPass(value: unknown): boolean { + if (typeof value === 'boolean') { + return value; + } + if (value === undefined) { + return true; + } + return /^(true|yes|pass|y)$/i.test(String(value)); +} + +function normalizePromptfooScore(value: unknown, pass: boolean): number { + if (typeof value === 'number') { + return clampScore(value); + } + const numeric = Number(value); + return clampScore(Number.isFinite(numeric) ? numeric : Number(pass)); +} + +function promptfooThreshold(config: GraderConfig | undefined): number | undefined { + const maybeConfigThreshold = + config && 'config' in config && isJsonRecord(config.config) + ? config.config.threshold + : undefined; + if (typeof maybeConfigThreshold === 'number' && Number.isFinite(maybeConfigThreshold)) { + return maybeConfigThreshold; + } + return config && 'min_score' in config && typeof config.min_score === 'number' + ? config.min_score + : undefined; +} + +function promptfooFallbackText(config: GraderConfig | undefined, fallbackText: string): string { + if (config?.type === 'llm-rubric' && typeof config.value === 'string') { + return config.value; + } + return fallbackText; +} + +function promptfooFallbackReason( + pass: boolean, + score: number, + threshold: number | undefined, +): string { + if (pass) { + return 'Grading passed'; + } + if (threshold !== undefined && score < threshold) { + return `Score ${score} below threshold ${threshold}`; + } + return 'Grading failed'; +} + +function promptfooCheckPass(check: z.infer): boolean { + const pass = check.pass ?? check.passed ?? check.satisfied; + if (pass !== undefined) { + return normalizePromptfooPass(pass); + } + if (check.score !== undefined) { + return normalizePromptfooScore(check.score, false) >= 0.8; + } + return false; +} + +function promptfooCheckText( + check: z.infer, + rubricMap: ReadonlyMap, +): string { + const rubric = check.id ? rubricMap.get(check.id) : undefined; + if (rubric) { + return `[${rubric.id}] ${rubric.outcome}`; + } + if (isNonEmptyString(check.text)) { + return check.text.trim(); + } + if (isNonEmptyString(check.id)) { + return `[${check.id}]`; + } + return 'llm-rubric check'; +} + +function normalizePromptfooGradingResult(options: { + readonly raw: unknown; + readonly config: GraderConfig | undefined; + readonly fallbackText: string; + readonly rubrics?: readonly RubricItem[]; +}): NormalizedPromptfooGradingResult { + const data = promptfooGradingResultSchema.parse(options.raw); + const rubricMap = new Map((options.rubrics ?? []).map((rubric) => [rubric.id, rubric])); + const normalizedChecks = data.checks?.map((check) => { + const checkPass = promptfooCheckPass(check); + return { + ...(check.id ? { id: check.id } : {}), + text: promptfooCheckText(check, rubricMap), + pass: checkPass, + ...(check.score !== undefined + ? { score: normalizePromptfooScore(check.score, checkPass) } + : {}), + reason: check.reason ?? check.evidence ?? check.reasoning ?? '', + ...(check.evidence && check.reason && check.evidence !== check.reason + ? { evidence: check.evidence } + : {}), + }; + }); + const threshold = promptfooThreshold(options.config); + const rawPass = + data.pass !== undefined + ? normalizePromptfooPass(data.pass) + : normalizedChecks && normalizedChecks.length > 0 + ? normalizedChecks.every((check) => check.pass) + : true; + const score = + data.score !== undefined + ? normalizePromptfooScore(data.score, rawPass) + : normalizedChecks && normalizedChecks.length > 0 + ? normalizedChecks.reduce((total, check) => { + if (typeof check.score === 'number') { + return total + check.score; + } + return total + Number(check.pass); + }, 0) / normalizedChecks.length + : normalizePromptfooScore(undefined, rawPass); + const pass = threshold !== undefined ? rawPass && score >= threshold : rawPass; + const reason = data.reason?.trim() || promptfooFallbackReason(pass, score, threshold); + + const assertions: AssertionEntry[] = + normalizedChecks && normalizedChecks.length > 0 + ? normalizedChecks.map((check) => ({ + text: check.text, + passed: check.pass, + evidence: check.reason || check.evidence, + })) + : [ + { + text: options.fallbackText.trim() || 'llm-rubric result', + passed: pass, + evidence: reason, + }, + ]; + + return { + score, + verdict: pass ? 'pass' : 'fail', + assertions, + details: { + pass, + score, + reason, + ...(threshold !== undefined ? { threshold } : {}), + ...(normalizedChecks && normalizedChecks.length > 0 ? { checks: normalizedChecks } : {}), + }, + }; +} + export class LlmGrader implements Grader { readonly kind = 'llm-grader'; @@ -307,7 +505,10 @@ export class LlmGrader implements Grader { const variables = buildTemplateVariables(context); // Build system prompt (only the mandatory output schema) - const systemPrompt = buildOutputSchema(); + const systemPrompt = + context.evaluator?.type === 'llm-rubric' + ? buildPromptfooRubricOutputSchema() + : buildOutputSchema(); // Build user prompt based on custom template or default template const graderTemplate = @@ -338,19 +539,34 @@ export class LlmGrader implements Grader { systemPrompt, userPrompt, schema: freeformEvaluationSchema, + parseResponse: (raw) => { + if (hasPromptfooAggregateFields(raw)) { + return normalizePromptfooGradingResult({ + raw, + config: context.evaluator, + fallbackText: promptfooFallbackText(context.evaluator, context.evalCase.criteria), + }); + } + const data = freeformEvaluationSchema.parse(raw); + const score = clampScore(data.score); + const assertions: AssertionEntry[] = Array.isArray(data.assertions) + ? data.assertions.slice(0, 8) + : []; + return { + score, + verdict: scoreToVerdict(score), + assertions, + details: data.details as JsonObject | undefined, + }; + }, images, }); - const score = clampScore(data.score); - const assertions: AssertionEntry[] = Array.isArray(data.assertions) - ? data.assertions.slice(0, 8) - : []; - return { - score, - verdict: scoreToVerdict(score), - assertions, - expectedAspectCount: Math.max(assertions.length, 1), + score: data.score, + verdict: data.verdict, + assertions: data.assertions, + expectedAspectCount: Math.max(data.assertions.length, 1), graderRawRequest, graderTarget: graderProvider.targetName, details: data.details as JsonObject | undefined, @@ -415,18 +631,29 @@ export class LlmGrader implements Grader { systemPrompt, userPrompt: prompt, schema: rubricEvaluationSchema, + parseResponse: (raw) => { + if (hasPromptfooAggregateFields(raw)) { + return normalizePromptfooGradingResult({ + raw, + config: context.evaluator, + fallbackText: promptfooFallbackText(context.evaluator, context.evalCase.criteria), + rubrics, + }); + } + const data = rubricEvaluationSchema.parse(raw); + return calculateRubricScore(data, rubrics); + }, images, }); - const { score, verdict, assertions } = calculateRubricScore(data, rubrics); - return { - score, - verdict, - assertions, + score: data.score, + verdict: data.verdict, + assertions: data.assertions, expectedAspectCount: rubrics.length, graderRawRequest, graderTarget: graderProvider.targetName, + details: 'details' in data ? data.details : undefined, tokenUsage, }; } catch (e: unknown) { @@ -570,6 +797,8 @@ export class LlmGrader implements Grader { graderRawRequest, details, graderProvider.targetName, + context.evaluator, + promptfooFallbackText(context.evaluator, context.evalCase.criteria), ); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -665,6 +894,8 @@ export class LlmGrader implements Grader { graderRawRequest, details, provider.targetName, + context.evaluator, + promptfooFallbackText(context.evaluator, context.evalCase.criteria), ); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -868,11 +1099,31 @@ export class LlmGrader implements Grader { graderRawRequest: JsonObject, details: JsonObject, graderTarget?: string, + config?: GraderConfig, + fallbackText = 'llm-rubric result', ): EvaluationScore { try { const parsed = parseJsonFromText(text); if (rubrics && rubrics.length > 0) { + if (hasPromptfooAggregateFields(parsed)) { + const normalized = normalizePromptfooGradingResult({ + raw: parsed, + config, + fallbackText, + rubrics, + }); + return { + score: normalized.score, + verdict: normalized.verdict, + assertions: normalized.assertions, + expectedAspectCount: rubrics.length, + graderRawRequest, + graderTarget, + details: { ...details, ...normalized.details }, + }; + } + const data = rubricEvaluationSchema.parse(parsed); const { score, verdict, assertions } = calculateRubricScore(data, rubrics); return { @@ -886,6 +1137,23 @@ export class LlmGrader implements Grader { }; } + if (hasPromptfooAggregateFields(parsed)) { + const normalized = normalizePromptfooGradingResult({ + raw: parsed, + config, + fallbackText, + }); + return { + score: normalized.score, + verdict: normalized.verdict, + assertions: normalized.assertions, + expectedAspectCount: Math.max(normalized.assertions.length, 1), + graderRawRequest, + graderTarget, + details: { ...details, ...normalized.details }, + }; + } + const data = freeformEvaluationSchema.parse(parsed); const score = clampScore(data.score); const assertions: AssertionEntry[] = Array.isArray(data.assertions) @@ -1073,15 +1341,17 @@ export class LlmGrader implements Grader { // LLM mode retry logic // --------------------------------------------------------------------------- - private async runWithRetry(options: { + private async runWithRetry(options: { readonly context: EvaluationContext; readonly graderProvider: Provider; readonly systemPrompt: string; readonly userPrompt: string; - readonly schema: z.ZodSchema; + readonly schema: z.ZodSchema; + readonly parseResponse?: (raw: unknown) => TData; readonly images?: readonly ContentImage[]; - }): Promise<{ data: T; providerResponse?: ProviderResponse; tokenUsage?: TokenUsage }> { - const { context, graderProvider, systemPrompt, userPrompt, schema, images } = options; + }): Promise<{ data: TData; providerResponse?: ProviderResponse; tokenUsage?: TokenUsage }> { + const { context, graderProvider, systemPrompt, userPrompt, schema, parseResponse, images } = + options; let lastError: Error | undefined; let lastInvalidResponse: StructuredGenerationResult | undefined; @@ -1098,9 +1368,10 @@ export class LlmGrader implements Grader { }); const canRepairResponse = result.text.trim().length > 0; lastInvalidResponse = canRepairResponse ? result : undefined; - let data: T; + let data: TData; try { - data = schema.parse(parseJsonFromText(result.text)); + const raw = parseJsonFromText(result.text); + data = parseResponse ? parseResponse(raw) : (schema.parse(raw) as unknown as TData); } catch (e: unknown) { lastError = e instanceof Error ? e : new Error(String(e)); shouldAttemptStructureFix = canRepairResponse; @@ -1127,7 +1398,8 @@ export class LlmGrader implements Grader { invalidResponse: lastInvalidResponse.text, }), }); - const data = schema.parse(parseJsonFromText(repaired.text)); + const raw = parseJsonFromText(repaired.text); + const data = parseResponse ? parseResponse(raw) : (schema.parse(raw) as unknown as TData); return { data, providerResponse: repaired.providerResponse, @@ -1196,6 +1468,30 @@ export function buildOutputSchema(): string { ].join('\n'); } +export function buildPromptfooRubricOutputSchema(): string { + return [ + 'You must respond with a single JSON object matching this schema:', + '', + '{', + ' "reason": "",', + ' "pass": ,', + ' "score": ,', + ' "checks": [', + ' {', + ' "id": "",', + ' "text": "",', + ' "pass": ,', + ' "score": ,', + ' "reason": "",', + ' "evidence": ""', + ' }', + ' ]', + '}', + '', + 'The "checks" array is optional. Return only JSON.', + ].join('\n'); +} + function buildStructureRepairPrompt(options: { readonly validationError: string; readonly invalidResponse: string; diff --git a/packages/core/src/evaluation/registry/builtin-graders.ts b/packages/core/src/evaluation/registry/builtin-graders.ts index f8d24a0d5..fe70f0da8 100644 --- a/packages/core/src/evaluation/registry/builtin-graders.ts +++ b/packages/core/src/evaluation/registry/builtin-graders.ts @@ -159,7 +159,7 @@ export const llmGraderFactory: GraderFactoryFn = (config, context) => { let graderTemplateOverride: string | undefined; let evalCase = evalContext.evalCase; - if (c.type === 'llm-rubric' && c.value !== undefined && !customPrompt) { + if (c.type === 'llm-rubric' && c.value !== undefined) { evalCase = { ...evalCase, criteria: formatRubricValue(c.value) }; } if (customPrompt) { diff --git a/packages/core/test/evaluation/graders.test.ts b/packages/core/test/evaluation/graders.test.ts index 3f5688137..b6b5a5330 100644 --- a/packages/core/test/evaluation/graders.test.ts +++ b/packages/core/test/evaluation/graders.test.ts @@ -175,6 +175,109 @@ describe('LlmGrader (llm-grader)', () => { expect(result.assertions.filter((a) => !a.passed)).toHaveLength(1); }); + it('accepts Promptfoo-style llm-rubric responses from custom prompts', async () => { + const graderProvider = new CapturingProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + reason: 'The answer captures the reference intent with minor omissions.', + pass: true, + score: 0.72, + }), + }, + ], + }); + + const evaluator = llmGraderFactory( + { + name: 'reference-match', + type: 'llm-rubric', + value: 'Matches the reference answer: {{ expected_output }}', + prompt: + 'Grade {{ output }} against {{ rubric }} and return { "reason": string, "pass": boolean, "score": number }', + }, + { + graderProvider, + llmGrader: new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }), + registry: {} as never, + }, + ); + + const result = await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-rubric' }, + candidate: 'Answer', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + }); + + expect(result.score).toBeCloseTo(0.72); + expect(result.verdict).toBe('pass'); + expect(result.assertions).toEqual([ + { + text: 'Matches the reference answer: {{ expected_output }}', + passed: true, + evidence: 'The answer captures the reference intent with minor omissions.', + }, + ]); + expect(graderProvider.lastRequest?.question).toContain( + 'Matches the reference answer: {{ expected_output }}', + ); + expect(graderProvider.lastRequest?.systemPrompt).toContain('"pass"'); + expect(graderProvider.lastRequest?.systemPrompt).not.toContain('"assertions"'); + }); + + it('applies explicit Promptfoo-style threshold to llm-rubric pass results', async () => { + const graderProvider = new StubProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + reason: 'Partially correct but below the configured bar.', + pass: true, + score: 0.6, + }), + }, + ], + }); + + const evaluator = new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }); + + const result = await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-rubric' }, + candidate: 'Answer', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + evaluator: { + name: 'rubric', + type: 'llm-rubric', + value: 'Answer meets the rubric', + config: { threshold: 0.75 }, + }, + }); + + expect(result.score).toBeCloseTo(0.6); + expect(result.verdict).toBe('fail'); + expect(result.assertions).toEqual([ + { + text: 'Answer meets the rubric', + passed: false, + evidence: 'Partially correct but below the configured bar.', + }, + ]); + expect(result.details?.threshold).toBe(0.75); + }); + it('recovers when a freeform assertion uses passed: mixed', async () => { const graderProvider = new StubProvider({ output: [ @@ -608,6 +711,153 @@ describe('LlmGrader (llm-grader)', () => { ).toContain('[r2]'); }); + it('maps Promptfoo-style rubric checks into internal assertion entries', async () => { + const graderProvider = new StubProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + reason: 'The answer covers logging but omits tests.', + pass: false, + score: 0.45, + checks: [ + { + id: 'r1', + text: 'Mentions logging', + pass: true, + score: 1, + reason: 'Structured logging is described.', + }, + { + id: 'r2', + text: 'Mentions tests', + pass: false, + score: 0, + reason: 'No test strategy is mentioned.', + }, + ], + }), + }, + ], + }); + + const evaluator = new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }); + + const result = await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-rubric' }, + candidate: 'Answer', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + evaluator: { + name: 'rubric', + type: 'llm-rubric', + rubrics: [ + { id: 'r1', outcome: 'Mentions logging', weight: 1.0, required: true }, + { id: 'r2', outcome: 'Mentions tests', weight: 1.0, required: false }, + ], + }, + }); + + expect(result.score).toBeCloseTo(0.45); + expect(result.verdict).toBe('fail'); + expect(result.assertions).toEqual([ + { + text: '[r1] Mentions logging', + passed: true, + evidence: 'Structured logging is described.', + }, + { + text: '[r2] Mentions tests', + passed: false, + evidence: 'No test strategy is mentioned.', + }, + ]); + expect(result.details?.reason).toBe('The answer covers logging but omits tests.'); + }); + + it('derives Promptfoo-style aggregate score from checks when omitted', async () => { + const graderProvider = new StubProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + checks: [ + { id: 'r1', pass: true, reason: 'Present.' }, + { id: 'r2', pass: false, reason: 'Missing.' }, + ], + }), + }, + ], + }); + + const evaluator = new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }); + + const result = await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-rubric' }, + candidate: 'Answer', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + evaluator: { + name: 'rubric', + type: 'llm-rubric', + rubrics: [ + { id: 'r1', outcome: 'Mentions logging', weight: 1.0, required: true }, + { id: 'r2', outcome: 'Mentions tests', weight: 1.0, required: false }, + ], + }, + }); + + expect(result.score).toBeCloseTo(0.5); + expect(result.verdict).toBe('fail'); + expect(result.assertions.map((assertion) => assertion.passed)).toEqual([true, false]); + }); + + it('synthesizes a Promptfoo-style reason when the judge omits one', async () => { + const graderProvider = new StubProvider({ + output: [ + { + role: 'assistant', + content: JSON.stringify({ + pass: false, + score: 0.2, + }), + }, + ], + }); + + const evaluator = new LlmGrader({ + resolveGraderProvider: async () => graderProvider, + }); + + const result = await evaluator.evaluate({ + evalCase: { ...baseTestCase, evaluator: 'llm-rubric' }, + candidate: 'Answer', + target: baseTarget, + provider: graderProvider, + attempt: 0, + promptInputs: { question: '' }, + now: new Date(), + evaluator: { + name: 'rubric', + type: 'llm-rubric', + value: 'Answer meets the rubric', + }, + }); + + expect(result.verdict).toBe('fail'); + expect(result.assertions[0]?.evidence).toBe('Grading failed'); + }); + it('preserves correctness and contradiction operators in rubric prompts', async () => { const graderProvider = new CapturingProvider({ output: [