refactor(eval): unify outline-language and whiteboard-layout harness (#453)
* feat(eval): add resolveEvalModel helper with fail-fast Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(eval): add createRunDir helper with path sanitization Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(eval): add markdown report helpers * refactor(eval): move language test cases under eval/outline-language * feat(eval): add outline-language types * feat(eval): add outline-language LLM judge * feat(eval): add outline-language reporter * feat(eval): add outline-language runner entry * chore(eval): add eval:outline-language pnpm script * refactor(eval): adopt shared createRunDir and drop gpt-4o fallbacks in whiteboard runner * refactor(eval): drop redundant non-null assertions after narrowing Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * refactor(eval): remove SCORER_MODEL_DEFAULT hardcoded gpt-4o fallback * chore(eval): delete tests/generation and clean up vitest/gitignore config * docs(eval): explain why outline-language runner pre-validates env vars Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(eval): escape pipe chars in markdown summary table cells LLM judge output may contain | which breaks GFM table rendering. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: 杨慎 <117187635+cosarah@users.noreply.github.com>
This commit is contained in:
parent
ac97c7772e
commit
7d47b0ad3a
3
.gitignore
vendored
3
.gitignore
vendored
@ -71,7 +71,6 @@ next-env.d.ts
|
||||
|
||||
# docs
|
||||
/docs
|
||||
tests/generation/*.eval.result.md
|
||||
|
||||
# Eval results
|
||||
eval/whiteboard-layout/results/
|
||||
eval/outline-language/results/
|
||||
|
||||
48
eval/outline-language/judge.ts
Normal file
48
eval/outline-language/judge.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import { generateText, type LanguageModel } from 'ai';
|
||||
import type { JudgeResult } from './types';
|
||||
|
||||
const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior.
|
||||
|
||||
You will be given:
|
||||
1. The original user requirement
|
||||
2. The generated language directive
|
||||
3. The ground truth description of expected behavior
|
||||
|
||||
Evaluation criteria — the directive should:
|
||||
- Use the correct primary teaching language
|
||||
- Handle terminology in a reasonable way for the subject and audience
|
||||
- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages
|
||||
|
||||
Be LENIENT in your evaluation:
|
||||
- The directive does NOT need to match the ground truth word-for-word
|
||||
- Different but equally valid approaches should PASS
|
||||
- If the teaching language is correct and the overall approach is reasonable, it should PASS
|
||||
- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation)
|
||||
|
||||
Respond with ONLY a JSON object:
|
||||
{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`;
|
||||
|
||||
/**
|
||||
* Ask an LLM-as-judge whether `directive` is a reasonable language directive
|
||||
* for `requirement` given `groundTruth`. Lenient rubric — see system prompt.
|
||||
*/
|
||||
export async function judgeDirective(
|
||||
judgeModel: LanguageModel,
|
||||
requirement: string,
|
||||
directive: string,
|
||||
groundTruth: string,
|
||||
): Promise<JudgeResult> {
|
||||
const result = await generateText({
|
||||
model: judgeModel,
|
||||
system: JUDGE_SYSTEM_PROMPT,
|
||||
prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`,
|
||||
temperature: 0,
|
||||
});
|
||||
|
||||
try {
|
||||
const text = result.text.replace(/```json\s*|\s*```/g, '').trim();
|
||||
return JSON.parse(text) as JudgeResult;
|
||||
} catch {
|
||||
return { pass: false, reason: `Failed to parse judge response: ${result.text}` };
|
||||
}
|
||||
}
|
||||
70
eval/outline-language/reporter.ts
Normal file
70
eval/outline-language/reporter.ts
Normal file
@ -0,0 +1,70 @@
|
||||
import { writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { renderHeader, renderSummaryTable } from '../shared/markdown-report';
|
||||
import type { EvalResult } from './types';
|
||||
|
||||
export interface ReportContext {
|
||||
inferenceModel: string;
|
||||
judgeModel: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write `report.md` into `runDir`. Returns the absolute path of the written file.
|
||||
*
|
||||
* Structure mirrors the old `outline-language.eval.result.md`:
|
||||
* 1. Header (date, models, pass count)
|
||||
* 2. One detail block per case (PASS / **FAIL**)
|
||||
* 3. Summary table of all cases
|
||||
*/
|
||||
export function writeReport(runDir: string, results: EvalResult[], ctx: ReportContext): string {
|
||||
const passed = results.filter((r) => r.judgePassed).length;
|
||||
const total = results.length;
|
||||
const pct = total === 0 ? 0 : Math.round((passed / total) * 100);
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(
|
||||
...renderHeader({
|
||||
title: 'Outline Language Inference Eval Results',
|
||||
timestamp: new Date().toISOString(),
|
||||
model: ctx.inferenceModel,
|
||||
judgeModel: ctx.judgeModel,
|
||||
extra: {
|
||||
Passed: `${passed}/${total} (${pct}%)`,
|
||||
Method: 'real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
lines.push(`## Detail`, ``);
|
||||
for (const r of results) {
|
||||
const icon = r.judgePassed ? 'PASS' : '**FAIL**';
|
||||
lines.push(`### ${icon} ${r.case_id}`, ``);
|
||||
lines.push(`- **Category**: ${r.category}`);
|
||||
lines.push(`- **Input**: \`${r.requirement}\``);
|
||||
if (r.pdfTextSample) {
|
||||
lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``);
|
||||
}
|
||||
lines.push(`- **Ground truth**: ${r.groundTruth}`);
|
||||
lines.push(`- **Directive**: ${r.directive}`);
|
||||
lines.push(`- **Outlines generated**: ${r.outlinesCount}`);
|
||||
lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`);
|
||||
lines.push(``);
|
||||
}
|
||||
|
||||
lines.push(`## Summary`, ``);
|
||||
const rows: string[][] = results.map((r, i) => [
|
||||
String(i + 1),
|
||||
r.case_id,
|
||||
r.category,
|
||||
String(r.outlinesCount),
|
||||
r.judgePassed ? 'PASS' : 'FAIL',
|
||||
r.judgeReason,
|
||||
]);
|
||||
lines.push(
|
||||
...renderSummaryTable(['#', 'Case', 'Category', 'Outlines', 'Result', 'Judge reason'], rows),
|
||||
);
|
||||
|
||||
const outPath = join(runDir, 'report.md');
|
||||
writeFileSync(outPath, lines.join('\n'), 'utf-8');
|
||||
return outPath;
|
||||
}
|
||||
174
eval/outline-language/runner.ts
Normal file
174
eval/outline-language/runner.ts
Normal file
@ -0,0 +1,174 @@
|
||||
/**
|
||||
* Outline Language Inference — Real LLM Evaluation Runner
|
||||
*
|
||||
* Calls generateSceneOutlinesFromRequirements for each test case, then uses
|
||||
* an LLM-as-judge to score the inferred languageDirective against ground truth.
|
||||
*
|
||||
* Required env:
|
||||
* EVAL_INFERENCE_MODEL Model for outline generation (or DEFAULT_MODEL)
|
||||
* EVAL_JUDGE_MODEL Model for LLM-as-judge
|
||||
*
|
||||
* Usage:
|
||||
* EVAL_INFERENCE_MODEL=<provider:model> EVAL_JUDGE_MODEL=<provider:model> \
|
||||
* pnpm eval:outline-language
|
||||
*
|
||||
* Output: eval/outline-language/results/<inference-model>/<timestamp>/report.md
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator';
|
||||
import { callLLM } from '@/lib/ai/llm';
|
||||
import type { AICallFn } from '@/lib/generation/pipeline-types';
|
||||
import { resolveEvalModel } from '../shared/resolve-model';
|
||||
import { createRunDir } from '../shared/run-dir';
|
||||
import { judgeDirective } from './judge';
|
||||
import { writeReport } from './reporter';
|
||||
import type { LanguageTestCase, EvalResult } from './types';
|
||||
|
||||
const OUTPUT_DIR = 'eval/outline-language/results';
|
||||
|
||||
function getCurrentDir(): string {
|
||||
return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url));
|
||||
}
|
||||
|
||||
function loadScenarios(): LanguageTestCase[] {
|
||||
const path = join(getCurrentDir(), 'scenarios/language-test-cases.json');
|
||||
return JSON.parse(readFileSync(path, 'utf-8')) as LanguageTestCase[];
|
||||
}
|
||||
|
||||
// Pre-validate env with tailored messages (including example model strings).
|
||||
// resolveEvalModel() also throws on missing vars, but with a shorter message;
|
||||
// surfacing the example before any async work makes misconfiguration obvious.
|
||||
function requireModelEnv(): { inferenceModelStr: string; judgeModelStr: string } {
|
||||
const inferenceModelStr = process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL;
|
||||
const judgeModelStr = process.env.EVAL_JUDGE_MODEL;
|
||||
if (!inferenceModelStr) {
|
||||
console.error(
|
||||
'Error: EVAL_INFERENCE_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_INFERENCE_MODEL=openai:gpt-4.1',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!judgeModelStr) {
|
||||
console.error(
|
||||
'Error: EVAL_JUDGE_MODEL must be set. Example: EVAL_JUDGE_MODEL=anthropic:claude-haiku-4-5',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
return { inferenceModelStr, judgeModelStr };
|
||||
}
|
||||
|
||||
async function runCase(
|
||||
tc: LanguageTestCase,
|
||||
aiCall: AICallFn,
|
||||
judgeModel: Awaited<ReturnType<typeof resolveEvalModel>>['model'],
|
||||
): Promise<EvalResult> {
|
||||
try {
|
||||
const result = await generateSceneOutlinesFromRequirements(
|
||||
{ requirement: tc.requirement },
|
||||
tc.pdfTextSample || undefined,
|
||||
undefined,
|
||||
aiCall,
|
||||
undefined,
|
||||
);
|
||||
|
||||
if (!result.success || !result.data) {
|
||||
return {
|
||||
case_id: tc.case_id,
|
||||
category: tc.category,
|
||||
requirement: tc.requirement,
|
||||
pdfTextSample: tc.pdfTextSample,
|
||||
groundTruth: tc.ground_truth,
|
||||
directive: '',
|
||||
outlinesCount: 0,
|
||||
judgePassed: false,
|
||||
judgeReason: `Outline generation failed: ${result.error || 'unknown error'}`,
|
||||
};
|
||||
}
|
||||
|
||||
const { languageDirective, outlines } = result.data;
|
||||
const judge = await judgeDirective(
|
||||
judgeModel,
|
||||
tc.requirement,
|
||||
languageDirective,
|
||||
tc.ground_truth,
|
||||
);
|
||||
|
||||
return {
|
||||
case_id: tc.case_id,
|
||||
category: tc.category,
|
||||
requirement: tc.requirement,
|
||||
pdfTextSample: tc.pdfTextSample,
|
||||
groundTruth: tc.ground_truth,
|
||||
directive: languageDirective,
|
||||
outlinesCount: outlines.length,
|
||||
judgePassed: judge.pass,
|
||||
judgeReason: judge.reason,
|
||||
};
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
case_id: tc.case_id,
|
||||
category: tc.category,
|
||||
requirement: tc.requirement,
|
||||
pdfTextSample: tc.pdfTextSample,
|
||||
groundTruth: tc.ground_truth,
|
||||
directive: '',
|
||||
outlinesCount: 0,
|
||||
judgePassed: false,
|
||||
judgeReason: `Exception: ${msg}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { inferenceModelStr, judgeModelStr } = requireModelEnv();
|
||||
|
||||
console.log('=== Outline Language Inference Eval ===');
|
||||
console.log(`Inference: ${inferenceModelStr} | Judge: ${judgeModelStr}`);
|
||||
|
||||
const { model: inferenceModel, modelInfo } = await resolveEvalModel(
|
||||
'EVAL_INFERENCE_MODEL',
|
||||
process.env.DEFAULT_MODEL,
|
||||
);
|
||||
const { model: judgeModel } = await resolveEvalModel('EVAL_JUDGE_MODEL');
|
||||
|
||||
const aiCall: AICallFn = async (systemPrompt, userPrompt, _images) => {
|
||||
const result = await callLLM(
|
||||
{
|
||||
model: inferenceModel,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
maxOutputTokens: modelInfo?.outputWindow,
|
||||
},
|
||||
'eval-outline-language',
|
||||
);
|
||||
return result.text;
|
||||
};
|
||||
|
||||
const cases = loadScenarios();
|
||||
console.log(`Loaded ${cases.length} test case(s)`);
|
||||
|
||||
const runDir = createRunDir(OUTPUT_DIR, inferenceModelStr);
|
||||
console.log(`Output: ${runDir}`);
|
||||
|
||||
const results = await Promise.all(cases.map((tc) => runCase(tc, aiCall, judgeModel)));
|
||||
|
||||
const reportPath = writeReport(runDir, results, {
|
||||
inferenceModel: inferenceModelStr,
|
||||
judgeModel: judgeModelStr,
|
||||
});
|
||||
const passed = results.filter((r) => r.judgePassed).length;
|
||||
console.log(`\nReport: ${reportPath}`);
|
||||
console.log(`Passed: ${passed}/${results.length}`);
|
||||
|
||||
process.exit(passed === results.length ? 0 : 1);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
24
eval/outline-language/types.ts
Normal file
24
eval/outline-language/types.ts
Normal file
@ -0,0 +1,24 @@
|
||||
export interface LanguageTestCase {
|
||||
case_id: string;
|
||||
category: string;
|
||||
requirement: string;
|
||||
ground_truth: string;
|
||||
pdfTextSample?: string;
|
||||
}
|
||||
|
||||
export interface JudgeResult {
|
||||
pass: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface EvalResult {
|
||||
case_id: string;
|
||||
category: string;
|
||||
requirement: string;
|
||||
pdfTextSample?: string;
|
||||
groundTruth: string;
|
||||
directive: string;
|
||||
outlinesCount: number;
|
||||
judgePassed: boolean;
|
||||
judgeReason: string;
|
||||
}
|
||||
35
eval/shared/markdown-report.ts
Normal file
35
eval/shared/markdown-report.ts
Normal file
@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Thin markdown helpers shared across eval reporters. Each returns `string[]`
|
||||
* so callers can push lines directly into their own buffer:
|
||||
*
|
||||
* const lines: string[] = [];
|
||||
* lines.push(...renderHeader({ title: 'Foo', ... }));
|
||||
* lines.push(...renderSummaryTable(['A', 'B'], rows));
|
||||
* writeFileSync(path, lines.join('\n'));
|
||||
*/
|
||||
|
||||
export interface ReportHeader {
|
||||
title: string;
|
||||
timestamp: string;
|
||||
model: string;
|
||||
judgeModel?: string;
|
||||
extra?: Record<string, string | number>;
|
||||
}
|
||||
|
||||
export function renderHeader(h: ReportHeader): string[] {
|
||||
const lines = [`# ${h.title}`, ``, `- **Date**: ${h.timestamp}`, `- **Model**: ${h.model}`];
|
||||
if (h.judgeModel) lines.push(`- **Judge model**: ${h.judgeModel}`);
|
||||
for (const [k, v] of Object.entries(h.extra || {})) {
|
||||
lines.push(`- **${k}**: ${v}`);
|
||||
}
|
||||
lines.push(``);
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function renderSummaryTable(headers: string[], rows: string[][]): string[] {
|
||||
const sep = `|${headers.map(() => '---').join('|')}|`;
|
||||
const lines = [`| ${headers.join(' | ')} |`, sep];
|
||||
for (const r of rows) lines.push(`| ${r.map((c) => c.replace(/\|/g, '\\|')).join(' | ')} |`);
|
||||
lines.push(``);
|
||||
return lines;
|
||||
}
|
||||
18
eval/shared/resolve-model.ts
Normal file
18
eval/shared/resolve-model.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { resolveModel } from '@/lib/server/resolve-model';
|
||||
|
||||
/**
|
||||
* Resolve a model for an eval runner. Reads `process.env[envVar]`, falls back
|
||||
* to `fallback` if provided, and throws a clear error if neither is set.
|
||||
*
|
||||
* Never introduces a hardcoded default model string — evals must be explicit
|
||||
* about what they measure.
|
||||
*/
|
||||
export async function resolveEvalModel(envVar: string, fallback?: string) {
|
||||
const modelString = process.env[envVar] || fallback;
|
||||
if (!modelString) {
|
||||
throw new Error(
|
||||
`Eval model not configured: set ${envVar} in the environment (or pass an explicit fallback).`,
|
||||
);
|
||||
}
|
||||
return resolveModel({ modelString });
|
||||
}
|
||||
16
eval/shared/run-dir.ts
Normal file
16
eval/shared/run-dir.ts
Normal file
@ -0,0 +1,16 @@
|
||||
import { mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Build and create a run directory under `<baseDir>/<sanitized-model>/<timestamp>/`.
|
||||
* The model string is sanitized by replacing `:` and `/` with `-` so it is
|
||||
* safe to use as a directory name. Timestamp is ISO-8601 with colons and dots
|
||||
* replaced by dashes, truncated to second precision.
|
||||
*/
|
||||
export function createRunDir(baseDir: string, model: string): string {
|
||||
const sanitizedModel = model.replace(/[:/]/g, '-');
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||||
const runDir = join(baseDir, sanitizedModel, timestamp);
|
||||
mkdirSync(runDir, { recursive: true });
|
||||
return runDir;
|
||||
}
|
||||
@ -9,16 +9,17 @@ import { EvalStateManager } from './state-manager';
|
||||
import { initCapture, captureWhiteboard, closeCapture } from './capture';
|
||||
import { scoreScreenshot } from './scorer';
|
||||
import { generateReport } from './reporter';
|
||||
import { createRunDir } from '../shared/run-dir';
|
||||
|
||||
// ==================== CLI Args ====================
|
||||
//
|
||||
// Model configuration follows the same pattern as the outline-language eval:
|
||||
// EVAL_CHAT_MODEL Model for chat generation (default: DEFAULT_MODEL or gpt-4o-mini)
|
||||
// EVAL_SCORER_MODEL Model for VLM scoring (default: openai:gpt-4o)
|
||||
// Required env:
|
||||
// EVAL_CHAT_MODEL (or DEFAULT_MODEL) Model for chat generation
|
||||
// EVAL_SCORER_MODEL Model for VLM scoring
|
||||
//
|
||||
// Usage:
|
||||
// EVAL_CHAT_MODEL=google:gemini-3.1-pro-preview \
|
||||
// EVAL_SCORER_MODEL=google:gemini-2.0-flash \
|
||||
// EVAL_CHAT_MODEL=<provider:model> \
|
||||
// EVAL_SCORER_MODEL=<provider:model> \
|
||||
// pnpm eval:whiteboard --scenario physics-force-decomposition
|
||||
|
||||
const { values: args } = parseArgs({
|
||||
@ -32,8 +33,22 @@ const { values: args } = parseArgs({
|
||||
});
|
||||
|
||||
const BASE_URL = args['base-url']!;
|
||||
const CHAT_MODEL = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL || 'openai:gpt-4o-mini';
|
||||
const SCORER_MODEL = process.env.EVAL_SCORER_MODEL || 'openai:gpt-4o';
|
||||
const CHAT_MODEL_RAW = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL;
|
||||
const SCORER_MODEL_RAW = process.env.EVAL_SCORER_MODEL;
|
||||
if (!CHAT_MODEL_RAW) {
|
||||
console.error(
|
||||
'Error: EVAL_CHAT_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_CHAT_MODEL=openai:gpt-4.1',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!SCORER_MODEL_RAW) {
|
||||
console.error(
|
||||
'Error: EVAL_SCORER_MODEL must be set. Example: EVAL_SCORER_MODEL=google:gemini-2.5-flash',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const CHAT_MODEL: string = CHAT_MODEL_RAW;
|
||||
const SCORER_MODEL: string = SCORER_MODEL_RAW;
|
||||
const REPEAT = parseInt(args.repeat || '1', 10);
|
||||
const OUTPUT_DIR = args['output-dir']!;
|
||||
const SCENARIO_FILTER = args.scenario;
|
||||
@ -325,11 +340,7 @@ async function main() {
|
||||
}
|
||||
console.log(`Loaded ${scenarios.length} scenario(s)`);
|
||||
|
||||
// Create run directory: results/<model>/<timestamp>/
|
||||
const sanitizedModel = CHAT_MODEL.replace(/[:/]/g, '-');
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
||||
const runDir = join(OUTPUT_DIR, sanitizedModel, timestamp);
|
||||
mkdirSync(runDir, { recursive: true });
|
||||
const runDir = createRunDir(OUTPUT_DIR, CHAT_MODEL);
|
||||
console.log(`Output: ${runDir}`);
|
||||
|
||||
await initCapture(BASE_URL);
|
||||
|
||||
@ -5,7 +5,8 @@
|
||||
* so model configuration follows the same `provider:model` convention as the rest
|
||||
* of the codebase. Supports all providers (OpenAI, Google, Anthropic, etc.).
|
||||
*
|
||||
* Environment variable: EVAL_SCORER_MODEL (default: openai:gpt-4o)
|
||||
* The caller supplies the model string explicitly (typically from EVAL_SCORER_MODEL);
|
||||
* this function no longer has a hardcoded default.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
@ -13,8 +14,6 @@ import { generateText } from 'ai';
|
||||
import { resolveModel } from '@/lib/server/resolve-model';
|
||||
import type { VlmScore } from './types';
|
||||
|
||||
const SCORER_MODEL_DEFAULT = 'openai:gpt-4o';
|
||||
|
||||
const RUBRIC_PROMPT = `You are evaluating a classroom whiteboard screenshot from an AI teaching assistant. Score like a teacher reviewing their own board work for a student's benefit.
|
||||
|
||||
Context: This is a real-time teaching whiteboard, NOT a poster or infographic.
|
||||
@ -62,18 +61,16 @@ Output ONLY a JSON object with this exact structure (no markdown, no code fences
|
||||
/**
|
||||
* Score a whiteboard screenshot using a VLM.
|
||||
*
|
||||
* Model is resolved via EVAL_SCORER_MODEL env var or the provided modelString,
|
||||
* using the same resolveModel() infrastructure as the rest of the project.
|
||||
* The caller must provide the model string explicitly (typically from EVAL_SCORER_MODEL);
|
||||
* this function no longer has a hardcoded default.
|
||||
*/
|
||||
export async function scoreScreenshot(
|
||||
screenshotPath: string,
|
||||
modelString?: string,
|
||||
modelString: string,
|
||||
): Promise<VlmScore> {
|
||||
const imageBuffer = readFileSync(screenshotPath);
|
||||
|
||||
const { model } = await resolveModel({
|
||||
modelString: modelString || process.env.EVAL_SCORER_MODEL || SCORER_MODEL_DEFAULT,
|
||||
});
|
||||
const { model } = await resolveModel({ modelString });
|
||||
|
||||
const result = await generateText({
|
||||
model,
|
||||
|
||||
@ -18,7 +18,8 @@
|
||||
"test": "vitest run",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:ui": "playwright test --ui",
|
||||
"eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts"
|
||||
"eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts",
|
||||
"eval:outline-language": "tsx eval/outline-language/runner.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.23",
|
||||
|
||||
72
tests/eval/outline-language/reporter.test.ts
Normal file
72
tests/eval/outline-language/reporter.test.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { writeReport } from '@/eval/outline-language/reporter';
|
||||
import type { EvalResult } from '@/eval/outline-language/types';
|
||||
|
||||
describe('writeReport', () => {
|
||||
let runDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
runDir = mkdtempSync(join(tmpdir(), 'reporter-test-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(runDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const sample: EvalResult[] = [
|
||||
{
|
||||
case_id: 'en-001',
|
||||
category: 'english-teaching',
|
||||
requirement: 'teach English to beginners',
|
||||
groundTruth: 'English as primary',
|
||||
directive: 'Use English as the teaching language',
|
||||
outlinesCount: 3,
|
||||
judgePassed: true,
|
||||
judgeReason: 'correct language',
|
||||
},
|
||||
{
|
||||
case_id: 'zh-002',
|
||||
category: 'chinese-teaching',
|
||||
requirement: '用中文讲授数学',
|
||||
groundTruth: '以中文为主',
|
||||
directive: 'Use Chinese throughout',
|
||||
outlinesCount: 2,
|
||||
judgePassed: false,
|
||||
judgeReason: 'wrong phrasing',
|
||||
},
|
||||
];
|
||||
|
||||
it('writes a report.md file with header, per-case detail, and summary table', () => {
|
||||
const path = writeReport(runDir, sample, {
|
||||
inferenceModel: 'openai:gpt-4.1',
|
||||
judgeModel: 'anthropic:claude-haiku-4-5',
|
||||
});
|
||||
expect(existsSync(path)).toBe(true);
|
||||
expect(path).toBe(join(runDir, 'report.md'));
|
||||
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
expect(content).toContain('# Outline Language Inference Eval Results');
|
||||
expect(content).toContain('**Model**: openai:gpt-4.1');
|
||||
expect(content).toContain('**Judge model**: anthropic:claude-haiku-4-5');
|
||||
expect(content).toContain('**Passed**: 1/2');
|
||||
expect(content).toContain('### PASS en-001');
|
||||
expect(content).toContain('### **FAIL** zh-002');
|
||||
expect(content).toContain('## Summary');
|
||||
expect(content).toContain('| # | Case | Category | Outlines | Result | Judge reason |');
|
||||
expect(content).toContain('| 1 | en-001 | english-teaching | 3 | PASS | correct language |');
|
||||
expect(content).toContain('| 2 | zh-002 | chinese-teaching | 2 | FAIL | wrong phrasing |');
|
||||
});
|
||||
|
||||
it('returns a successfully written path even when all cases pass', () => {
|
||||
const allPass: EvalResult[] = [{ ...sample[0] }];
|
||||
const path = writeReport(runDir, allPass, {
|
||||
inferenceModel: 'openai:gpt-4.1',
|
||||
judgeModel: 'openai:gpt-4.1',
|
||||
});
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
expect(content).toContain('**Passed**: 1/1 (100%)');
|
||||
});
|
||||
});
|
||||
51
tests/eval/shared/resolve-model.test.ts
Normal file
51
tests/eval/shared/resolve-model.test.ts
Normal file
@ -0,0 +1,51 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
|
||||
describe('resolveEvalModel', () => {
|
||||
const ORIG_ENV = process.env;
|
||||
|
||||
beforeEach(() => {
|
||||
process.env = { ...ORIG_ENV };
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.env = ORIG_ENV;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('throws a helpful error when env var is unset and no fallback is given', async () => {
|
||||
delete process.env.EVAL_FOO_MODEL;
|
||||
vi.doMock('@/lib/server/resolve-model', () => ({
|
||||
resolveModel: vi.fn(),
|
||||
}));
|
||||
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
|
||||
await expect(resolveEvalModel('EVAL_FOO_MODEL')).rejects.toThrow(/EVAL_FOO_MODEL/);
|
||||
});
|
||||
|
||||
it('uses the env var when set', async () => {
|
||||
process.env.EVAL_FOO_MODEL = 'openai:gpt-4.1';
|
||||
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
|
||||
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
|
||||
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
|
||||
await resolveEvalModel('EVAL_FOO_MODEL');
|
||||
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'openai:gpt-4.1' });
|
||||
});
|
||||
|
||||
it('uses the explicit fallback when env var is unset', async () => {
|
||||
delete process.env.EVAL_FOO_MODEL;
|
||||
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
|
||||
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
|
||||
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
|
||||
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
|
||||
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'google:gemini-2.5-flash' });
|
||||
});
|
||||
|
||||
it('env var takes precedence over fallback', async () => {
|
||||
process.env.EVAL_FOO_MODEL = 'anthropic:claude-haiku-4-5';
|
||||
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
|
||||
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
|
||||
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
|
||||
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
|
||||
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'anthropic:claude-haiku-4-5' });
|
||||
});
|
||||
});
|
||||
38
tests/eval/shared/run-dir.test.ts
Normal file
38
tests/eval/shared/run-dir.test.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { existsSync, rmSync, mkdtempSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { createRunDir } from '@/eval/shared/run-dir';
|
||||
|
||||
describe('createRunDir', () => {
|
||||
let tempRoot: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempRoot = mkdtempSync(join(tmpdir(), 'run-dir-test-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tempRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('creates <base>/<sanitized-model>/<timestamp>/ and returns the path', () => {
|
||||
const runDir = createRunDir(tempRoot, 'openai:gpt-4.1');
|
||||
expect(runDir.startsWith(join(tempRoot, 'openai-gpt-4.1'))).toBe(true);
|
||||
expect(existsSync(runDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('sanitizes both : and / from the model string', () => {
|
||||
const runDir = createRunDir(tempRoot, 'google:gemini-2.5-flash/latest');
|
||||
expect(runDir).toContain('google-gemini-2.5-flash-latest');
|
||||
expect(runDir).not.toMatch(/[:/]gemini/);
|
||||
});
|
||||
|
||||
it('timestamp segment has no colons or dots', () => {
|
||||
const runDir = createRunDir(tempRoot, 'x');
|
||||
const segments = runDir.split('/');
|
||||
const timestamp = segments[segments.length - 1];
|
||||
expect(timestamp).not.toContain(':');
|
||||
expect(timestamp).not.toContain('.');
|
||||
expect(timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}$/);
|
||||
});
|
||||
});
|
||||
@ -1,272 +0,0 @@
|
||||
/**
|
||||
* Outline Language Inference — Real LLM Evaluation Tests
|
||||
*
|
||||
* Tests the languageDirective inferred during REAL outline generation.
|
||||
* Calls the actual generateSceneOutlinesFromRequirements function,
|
||||
* then uses an LLM-as-judge to evaluate the directive against ground truth.
|
||||
*
|
||||
* Calls real LLM APIs — meant to be run locally, NOT in CI/CD.
|
||||
*
|
||||
* Environment variables:
|
||||
* EVAL_INFERENCE_MODEL Model for outline generation (default: DEFAULT_MODEL or gpt-4o-mini)
|
||||
* EVAL_JUDGE_MODEL Model for LLM-as-judge (default: openai:gpt-4o-mini)
|
||||
*
|
||||
* Usage:
|
||||
* EVAL_INFERENCE_MODEL=google:gemini-3-flash-preview \
|
||||
* EVAL_JUDGE_MODEL=openai:gpt-4o-mini \
|
||||
* pnpm vitest run tests/generation/outline-language.eval.test.ts
|
||||
*
|
||||
* Results are written to tests/generation/outline-language.eval.result.md
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
|
||||
import { writeFileSync, readFileSync } from 'fs';
|
||||
import { resolve } from 'path';
|
||||
import { generateText } from 'ai';
|
||||
import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator';
|
||||
import { resolveModel } from '@/lib/server/resolve-model';
|
||||
import type { AICallFn } from '@/lib/generation/pipeline-types';
|
||||
import { callLLM } from '@/lib/ai/llm';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test case definition
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface LanguageTestCase {
|
||||
case_id: string;
|
||||
category: string;
|
||||
requirement: string;
|
||||
ground_truth: string;
|
||||
pdfTextSample?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Load test cases from JSON
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TEST_CASES: LanguageTestCase[] = JSON.parse(
|
||||
readFileSync(resolve(__dirname, 'language-test-cases.json'), 'utf-8'),
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Model resolution
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const JUDGE_MODEL_DEFAULT = 'openai:gpt-4o-mini';
|
||||
|
||||
/** Resolve the inference model (EVAL_INFERENCE_MODEL → DEFAULT_MODEL → gpt-4o-mini) */
|
||||
function getInferenceModel() {
|
||||
return resolveModel({
|
||||
modelString: process.env.EVAL_INFERENCE_MODEL || undefined,
|
||||
});
|
||||
}
|
||||
|
||||
/** Resolve the judge model (EVAL_JUDGE_MODEL → gpt-4o-mini) */
|
||||
function getJudgeModel() {
|
||||
return resolveModel({
|
||||
modelString: process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build AICallFn from the inference model
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildAICallFn(): Promise<AICallFn> {
|
||||
const { model: languageModel, modelInfo } = await getInferenceModel();
|
||||
return async (systemPrompt, userPrompt, _images) => {
|
||||
const result = await callLLM(
|
||||
{
|
||||
model: languageModel,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
maxOutputTokens: modelInfo?.outputWindow,
|
||||
},
|
||||
'eval-outline-language',
|
||||
);
|
||||
return result.text;
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LLM Judge
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior.
|
||||
|
||||
You will be given:
|
||||
1. The original user requirement
|
||||
2. The generated language directive
|
||||
3. The ground truth description of expected behavior
|
||||
|
||||
Evaluation criteria — the directive should:
|
||||
- Use the correct primary teaching language
|
||||
- Handle terminology in a reasonable way for the subject and audience
|
||||
- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages
|
||||
|
||||
Be LENIENT in your evaluation:
|
||||
- The directive does NOT need to match the ground truth word-for-word
|
||||
- Different but equally valid approaches should PASS
|
||||
- If the teaching language is correct and the overall approach is reasonable, it should PASS
|
||||
- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation)
|
||||
|
||||
Respond with ONLY a JSON object:
|
||||
{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`;
|
||||
|
||||
interface JudgeResult {
|
||||
pass: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
async function judgeDirective(
|
||||
requirement: string,
|
||||
directive: string,
|
||||
groundTruth: string,
|
||||
): Promise<JudgeResult> {
|
||||
const { model } = await getJudgeModel();
|
||||
const result = await generateText({
|
||||
model,
|
||||
system: JUDGE_SYSTEM_PROMPT,
|
||||
prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`,
|
||||
temperature: 0,
|
||||
});
|
||||
|
||||
try {
|
||||
const text = result.text.replace(/```json\s*|\s*```/g, '').trim();
|
||||
return JSON.parse(text) as JudgeResult;
|
||||
} catch {
|
||||
return { pass: false, reason: `Failed to parse judge response: ${result.text}` };
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result collector
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface EvalResult {
|
||||
case_id: string;
|
||||
category: string;
|
||||
requirement: string;
|
||||
pdfTextSample?: string;
|
||||
groundTruth: string;
|
||||
directive: string;
|
||||
outlinesCount: number;
|
||||
judgePassed: boolean;
|
||||
judgeReason: string;
|
||||
}
|
||||
|
||||
const results: EvalResult[] = [];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test runner
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('Outline Language Inference Evaluation', () => {
|
||||
let aiCall: AICallFn;
|
||||
|
||||
beforeAll(async () => {
|
||||
aiCall = await buildAICallFn();
|
||||
});
|
||||
|
||||
for (const tc of TEST_CASES) {
|
||||
it.concurrent(
|
||||
`${tc.case_id}: ${tc.requirement.slice(0, 50)}`,
|
||||
{ timeout: 120_000 },
|
||||
async () => {
|
||||
// Call the REAL outline generation function
|
||||
const result = await generateSceneOutlinesFromRequirements(
|
||||
{ requirement: tc.requirement },
|
||||
tc.pdfTextSample || undefined, // pass PDF text if available
|
||||
undefined, // no PDF images
|
||||
aiCall,
|
||||
undefined, // no callbacks
|
||||
);
|
||||
|
||||
expect(result.success, `Outline generation failed: ${result.error}`).toBe(true);
|
||||
expect(result.data).toBeDefined();
|
||||
|
||||
const { languageDirective, outlines } = result.data!;
|
||||
expect(languageDirective, 'languageDirective should not be empty').toBeTruthy();
|
||||
expect(outlines.length, 'should generate at least 1 outline').toBeGreaterThan(0);
|
||||
|
||||
// LLM-as-judge
|
||||
const judge = await judgeDirective(tc.requirement, languageDirective, tc.ground_truth);
|
||||
|
||||
results.push({
|
||||
case_id: tc.case_id,
|
||||
category: tc.category,
|
||||
requirement: tc.requirement,
|
||||
pdfTextSample: tc.pdfTextSample,
|
||||
groundTruth: tc.ground_truth,
|
||||
directive: languageDirective,
|
||||
outlinesCount: outlines.length,
|
||||
judgePassed: judge.pass,
|
||||
judgeReason: judge.reason,
|
||||
});
|
||||
|
||||
expect(judge.pass, `Judge failed: ${judge.reason}`).toBe(true);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Write results file after all tests
|
||||
afterAll(() => {
|
||||
if (results.length === 0) return;
|
||||
|
||||
const passed = results.filter((r) => r.judgePassed).length;
|
||||
const total = results.length;
|
||||
const inferenceModelStr =
|
||||
process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL || '(default: gpt-4o-mini)';
|
||||
const judgeModelStr = process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT;
|
||||
|
||||
const lines: string[] = [
|
||||
`# Outline Language Inference Eval Results`,
|
||||
``,
|
||||
`- **Date**: ${new Date().toISOString()}`,
|
||||
`- **Passed**: ${passed}/${total} (${((passed / total) * 100).toFixed(0)}%)`,
|
||||
`- **Inference model**: ${inferenceModelStr}`,
|
||||
`- **Judge model**: ${judgeModelStr}`,
|
||||
`- **Method**: real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge`,
|
||||
`- **Test cases**: curated from production data`,
|
||||
``,
|
||||
`## Detail`,
|
||||
``,
|
||||
];
|
||||
|
||||
for (const r of results) {
|
||||
const icon = r.judgePassed ? 'PASS' : '**FAIL**';
|
||||
|
||||
lines.push(`### ${icon} ${r.case_id}`);
|
||||
lines.push(``);
|
||||
lines.push(`- **Category**: ${r.category}`);
|
||||
lines.push(`- **Input**: \`${r.requirement}\``);
|
||||
if (r.pdfTextSample) {
|
||||
lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``);
|
||||
}
|
||||
lines.push(`- **Ground truth**: ${r.groundTruth}`);
|
||||
lines.push(`- **Directive**: ${r.directive}`);
|
||||
lines.push(`- **Outlines generated**: ${r.outlinesCount}`);
|
||||
lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`);
|
||||
lines.push(``);
|
||||
}
|
||||
|
||||
lines.push(`## Summary`);
|
||||
lines.push(``);
|
||||
lines.push(`| # | Case | Category | Outlines | Result | Judge reason |`);
|
||||
lines.push(`|---|------|----------|----------|--------|--------------|`);
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const r = results[i];
|
||||
const icon = r.judgePassed ? 'PASS' : 'FAIL';
|
||||
lines.push(
|
||||
`| ${i + 1} | ${r.case_id} | ${r.category} | ${r.outlinesCount} | ${icon} | ${r.judgeReason} |`,
|
||||
);
|
||||
}
|
||||
lines.push(``);
|
||||
|
||||
const outPath = resolve(__dirname, 'outline-language.eval.result.md');
|
||||
writeFileSync(outPath, lines.join('\n'), 'utf-8');
|
||||
console.log(`\nResults written to: ${outPath}`);
|
||||
});
|
||||
});
|
||||
@ -9,7 +9,6 @@ export default defineConfig({
|
||||
},
|
||||
test: {
|
||||
include: ['tests/**/*.test.ts'],
|
||||
exclude: ['tests/**/*.eval.test.ts'],
|
||||
setupFiles: ['tests/setup-env.ts'],
|
||||
},
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user