refactor(eval): unify outline-language and whiteboard-layout harness (#453)

* feat(eval): add resolveEvalModel helper with fail-fast

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* feat(eval): add createRunDir helper with path sanitization

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* feat(eval): add markdown report helpers

* refactor(eval): move language test cases under eval/outline-language

* feat(eval): add outline-language types

* feat(eval): add outline-language LLM judge

* feat(eval): add outline-language reporter

* feat(eval): add outline-language runner entry

* chore(eval): add eval:outline-language pnpm script

* refactor(eval): adopt shared createRunDir and drop gpt-4o fallbacks in whiteboard runner

* refactor(eval): drop redundant non-null assertions after narrowing

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* refactor(eval): remove SCORER_MODEL_DEFAULT hardcoded gpt-4o fallback

* chore(eval): delete tests/generation and clean up vitest/gitignore config

* docs(eval): explain why outline-language runner pre-validates env vars

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(eval): escape pipe chars in markdown summary table cells

LLM judge output may contain | which breaks GFM table rendering.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: 杨慎 <117187635+cosarah@users.noreply.github.com>
This commit is contained in:
wyuc 2026-04-19 21:29:18 +08:00 committed by GitHub
parent ac97c7772e
commit 7d47b0ad3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 578 additions and 297 deletions

3
.gitignore vendored
View File

@ -71,7 +71,6 @@ next-env.d.ts
# docs
/docs
tests/generation/*.eval.result.md
# Eval results
eval/whiteboard-layout/results/
eval/outline-language/results/

View File

@ -0,0 +1,48 @@
import { generateText, type LanguageModel } from 'ai';
import type { JudgeResult } from './types';
const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior.
You will be given:
1. The original user requirement
2. The generated language directive
3. The ground truth description of expected behavior
Evaluation criteria — the directive should:
- Use the correct primary teaching language
- Handle terminology in a reasonable way for the subject and audience
- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages
Be LENIENT in your evaluation:
- The directive does NOT need to match the ground truth word-for-word
- Different but equally valid approaches should PASS
- If the teaching language is correct and the overall approach is reasonable, it should PASS
- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation)
Respond with ONLY a JSON object:
{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`;
/**
* Ask an LLM-as-judge whether `directive` is a reasonable language directive
* for `requirement` given `groundTruth`. Lenient rubric — see system prompt.
*/
export async function judgeDirective(
judgeModel: LanguageModel,
requirement: string,
directive: string,
groundTruth: string,
): Promise<JudgeResult> {
const result = await generateText({
model: judgeModel,
system: JUDGE_SYSTEM_PROMPT,
prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`,
temperature: 0,
});
try {
const text = result.text.replace(/```json\s*|\s*```/g, '').trim();
return JSON.parse(text) as JudgeResult;
} catch {
return { pass: false, reason: `Failed to parse judge response: ${result.text}` };
}
}

View File

@ -0,0 +1,70 @@
import { writeFileSync } from 'fs';
import { join } from 'path';
import { renderHeader, renderSummaryTable } from '../shared/markdown-report';
import type { EvalResult } from './types';
export interface ReportContext {
inferenceModel: string;
judgeModel: string;
}
/**
* Write `report.md` into `runDir`. Returns the absolute path of the written file.
*
* Structure mirrors the old `outline-language.eval.result.md`:
* 1. Header (date, models, pass count)
* 2. One detail block per case (PASS / **FAIL**)
* 3. Summary table of all cases
*/
export function writeReport(runDir: string, results: EvalResult[], ctx: ReportContext): string {
const passed = results.filter((r) => r.judgePassed).length;
const total = results.length;
const pct = total === 0 ? 0 : Math.round((passed / total) * 100);
const lines: string[] = [];
lines.push(
...renderHeader({
title: 'Outline Language Inference Eval Results',
timestamp: new Date().toISOString(),
model: ctx.inferenceModel,
judgeModel: ctx.judgeModel,
extra: {
Passed: `${passed}/${total} (${pct}%)`,
Method: 'real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge',
},
}),
);
lines.push(`## Detail`, ``);
for (const r of results) {
const icon = r.judgePassed ? 'PASS' : '**FAIL**';
lines.push(`### ${icon} ${r.case_id}`, ``);
lines.push(`- **Category**: ${r.category}`);
lines.push(`- **Input**: \`${r.requirement}\``);
if (r.pdfTextSample) {
lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``);
}
lines.push(`- **Ground truth**: ${r.groundTruth}`);
lines.push(`- **Directive**: ${r.directive}`);
lines.push(`- **Outlines generated**: ${r.outlinesCount}`);
lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`);
lines.push(``);
}
lines.push(`## Summary`, ``);
const rows: string[][] = results.map((r, i) => [
String(i + 1),
r.case_id,
r.category,
String(r.outlinesCount),
r.judgePassed ? 'PASS' : 'FAIL',
r.judgeReason,
]);
lines.push(
...renderSummaryTable(['#', 'Case', 'Category', 'Outlines', 'Result', 'Judge reason'], rows),
);
const outPath = join(runDir, 'report.md');
writeFileSync(outPath, lines.join('\n'), 'utf-8');
return outPath;
}

View File

@ -0,0 +1,174 @@
/**
* Outline Language Inference — Real LLM Evaluation Runner
*
* Calls generateSceneOutlinesFromRequirements for each test case, then uses
* an LLM-as-judge to score the inferred languageDirective against ground truth.
*
* Required env:
* EVAL_INFERENCE_MODEL Model for outline generation (or DEFAULT_MODEL)
* EVAL_JUDGE_MODEL Model for LLM-as-judge
*
* Usage:
* EVAL_INFERENCE_MODEL=<provider:model> EVAL_JUDGE_MODEL=<provider:model> \
* pnpm eval:outline-language
*
* Output: eval/outline-language/results/<inference-model>/<timestamp>/report.md
*/
import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator';
import { callLLM } from '@/lib/ai/llm';
import type { AICallFn } from '@/lib/generation/pipeline-types';
import { resolveEvalModel } from '../shared/resolve-model';
import { createRunDir } from '../shared/run-dir';
import { judgeDirective } from './judge';
import { writeReport } from './reporter';
import type { LanguageTestCase, EvalResult } from './types';
const OUTPUT_DIR = 'eval/outline-language/results';
function getCurrentDir(): string {
return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url));
}
function loadScenarios(): LanguageTestCase[] {
const path = join(getCurrentDir(), 'scenarios/language-test-cases.json');
return JSON.parse(readFileSync(path, 'utf-8')) as LanguageTestCase[];
}
// Pre-validate env with tailored messages (including example model strings).
// resolveEvalModel() also throws on missing vars, but with a shorter message;
// surfacing the example before any async work makes misconfiguration obvious.
function requireModelEnv(): { inferenceModelStr: string; judgeModelStr: string } {
const inferenceModelStr = process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL;
const judgeModelStr = process.env.EVAL_JUDGE_MODEL;
if (!inferenceModelStr) {
console.error(
'Error: EVAL_INFERENCE_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_INFERENCE_MODEL=openai:gpt-4.1',
);
process.exit(1);
}
if (!judgeModelStr) {
console.error(
'Error: EVAL_JUDGE_MODEL must be set. Example: EVAL_JUDGE_MODEL=anthropic:claude-haiku-4-5',
);
process.exit(1);
}
return { inferenceModelStr, judgeModelStr };
}
async function runCase(
tc: LanguageTestCase,
aiCall: AICallFn,
judgeModel: Awaited<ReturnType<typeof resolveEvalModel>>['model'],
): Promise<EvalResult> {
try {
const result = await generateSceneOutlinesFromRequirements(
{ requirement: tc.requirement },
tc.pdfTextSample || undefined,
undefined,
aiCall,
undefined,
);
if (!result.success || !result.data) {
return {
case_id: tc.case_id,
category: tc.category,
requirement: tc.requirement,
pdfTextSample: tc.pdfTextSample,
groundTruth: tc.ground_truth,
directive: '',
outlinesCount: 0,
judgePassed: false,
judgeReason: `Outline generation failed: ${result.error || 'unknown error'}`,
};
}
const { languageDirective, outlines } = result.data;
const judge = await judgeDirective(
judgeModel,
tc.requirement,
languageDirective,
tc.ground_truth,
);
return {
case_id: tc.case_id,
category: tc.category,
requirement: tc.requirement,
pdfTextSample: tc.pdfTextSample,
groundTruth: tc.ground_truth,
directive: languageDirective,
outlinesCount: outlines.length,
judgePassed: judge.pass,
judgeReason: judge.reason,
};
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return {
case_id: tc.case_id,
category: tc.category,
requirement: tc.requirement,
pdfTextSample: tc.pdfTextSample,
groundTruth: tc.ground_truth,
directive: '',
outlinesCount: 0,
judgePassed: false,
judgeReason: `Exception: ${msg}`,
};
}
}
async function main() {
const { inferenceModelStr, judgeModelStr } = requireModelEnv();
console.log('=== Outline Language Inference Eval ===');
console.log(`Inference: ${inferenceModelStr} | Judge: ${judgeModelStr}`);
const { model: inferenceModel, modelInfo } = await resolveEvalModel(
'EVAL_INFERENCE_MODEL',
process.env.DEFAULT_MODEL,
);
const { model: judgeModel } = await resolveEvalModel('EVAL_JUDGE_MODEL');
const aiCall: AICallFn = async (systemPrompt, userPrompt, _images) => {
const result = await callLLM(
{
model: inferenceModel,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: userPrompt },
],
maxOutputTokens: modelInfo?.outputWindow,
},
'eval-outline-language',
);
return result.text;
};
const cases = loadScenarios();
console.log(`Loaded ${cases.length} test case(s)`);
const runDir = createRunDir(OUTPUT_DIR, inferenceModelStr);
console.log(`Output: ${runDir}`);
const results = await Promise.all(cases.map((tc) => runCase(tc, aiCall, judgeModel)));
const reportPath = writeReport(runDir, results, {
inferenceModel: inferenceModelStr,
judgeModel: judgeModelStr,
});
const passed = results.filter((r) => r.judgePassed).length;
console.log(`\nReport: ${reportPath}`);
console.log(`Passed: ${passed}/${results.length}`);
process.exit(passed === results.length ? 0 : 1);
}
main().catch((err) => {
console.error('Fatal error:', err);
process.exit(1);
});

View File

@ -0,0 +1,24 @@
export interface LanguageTestCase {
case_id: string;
category: string;
requirement: string;
ground_truth: string;
pdfTextSample?: string;
}
export interface JudgeResult {
pass: boolean;
reason: string;
}
export interface EvalResult {
case_id: string;
category: string;
requirement: string;
pdfTextSample?: string;
groundTruth: string;
directive: string;
outlinesCount: number;
judgePassed: boolean;
judgeReason: string;
}

View File

@ -0,0 +1,35 @@
/**
* Thin markdown helpers shared across eval reporters. Each returns `string[]`
* so callers can push lines directly into their own buffer:
*
* const lines: string[] = [];
* lines.push(...renderHeader({ title: 'Foo', ... }));
* lines.push(...renderSummaryTable(['A', 'B'], rows));
* writeFileSync(path, lines.join('\n'));
*/
export interface ReportHeader {
title: string;
timestamp: string;
model: string;
judgeModel?: string;
extra?: Record<string, string | number>;
}
export function renderHeader(h: ReportHeader): string[] {
const lines = [`# ${h.title}`, ``, `- **Date**: ${h.timestamp}`, `- **Model**: ${h.model}`];
if (h.judgeModel) lines.push(`- **Judge model**: ${h.judgeModel}`);
for (const [k, v] of Object.entries(h.extra || {})) {
lines.push(`- **${k}**: ${v}`);
}
lines.push(``);
return lines;
}
export function renderSummaryTable(headers: string[], rows: string[][]): string[] {
const sep = `|${headers.map(() => '---').join('|')}|`;
const lines = [`| ${headers.join(' | ')} |`, sep];
for (const r of rows) lines.push(`| ${r.map((c) => c.replace(/\|/g, '\\|')).join(' | ')} |`);
lines.push(``);
return lines;
}

View File

@ -0,0 +1,18 @@
import { resolveModel } from '@/lib/server/resolve-model';
/**
* Resolve a model for an eval runner. Reads `process.env[envVar]`, falls back
* to `fallback` if provided, and throws a clear error if neither is set.
*
* Never introduces a hardcoded default model string — evals must be explicit
* about what they measure.
*/
export async function resolveEvalModel(envVar: string, fallback?: string) {
const modelString = process.env[envVar] || fallback;
if (!modelString) {
throw new Error(
`Eval model not configured: set ${envVar} in the environment (or pass an explicit fallback).`,
);
}
return resolveModel({ modelString });
}

16
eval/shared/run-dir.ts Normal file
View File

@ -0,0 +1,16 @@
import { mkdirSync } from 'fs';
import { join } from 'path';
/**
* Build and create a run directory under `<baseDir>/<sanitized-model>/<timestamp>/`.
* The model string is sanitized by replacing `:` and `/` with `-` so it is
* safe to use as a directory name. Timestamp is ISO-8601 with colons and dots
* replaced by dashes, truncated to second precision.
*/
export function createRunDir(baseDir: string, model: string): string {
const sanitizedModel = model.replace(/[:/]/g, '-');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const runDir = join(baseDir, sanitizedModel, timestamp);
mkdirSync(runDir, { recursive: true });
return runDir;
}

View File

@ -9,16 +9,17 @@ import { EvalStateManager } from './state-manager';
import { initCapture, captureWhiteboard, closeCapture } from './capture';
import { scoreScreenshot } from './scorer';
import { generateReport } from './reporter';
import { createRunDir } from '../shared/run-dir';
// ==================== CLI Args ====================
//
// Model configuration follows the same pattern as the outline-language eval:
// EVAL_CHAT_MODEL Model for chat generation (default: DEFAULT_MODEL or gpt-4o-mini)
// EVAL_SCORER_MODEL Model for VLM scoring (default: openai:gpt-4o)
// Required env:
// EVAL_CHAT_MODEL (or DEFAULT_MODEL) Model for chat generation
// EVAL_SCORER_MODEL Model for VLM scoring
//
// Usage:
// EVAL_CHAT_MODEL=google:gemini-3.1-pro-preview \
// EVAL_SCORER_MODEL=google:gemini-2.0-flash \
// EVAL_CHAT_MODEL=<provider:model> \
// EVAL_SCORER_MODEL=<provider:model> \
// pnpm eval:whiteboard --scenario physics-force-decomposition
const { values: args } = parseArgs({
@ -32,8 +33,22 @@ const { values: args } = parseArgs({
});
const BASE_URL = args['base-url']!;
const CHAT_MODEL = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL || 'openai:gpt-4o-mini';
const SCORER_MODEL = process.env.EVAL_SCORER_MODEL || 'openai:gpt-4o';
const CHAT_MODEL_RAW = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL;
const SCORER_MODEL_RAW = process.env.EVAL_SCORER_MODEL;
if (!CHAT_MODEL_RAW) {
console.error(
'Error: EVAL_CHAT_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_CHAT_MODEL=openai:gpt-4.1',
);
process.exit(1);
}
if (!SCORER_MODEL_RAW) {
console.error(
'Error: EVAL_SCORER_MODEL must be set. Example: EVAL_SCORER_MODEL=google:gemini-2.5-flash',
);
process.exit(1);
}
const CHAT_MODEL: string = CHAT_MODEL_RAW;
const SCORER_MODEL: string = SCORER_MODEL_RAW;
const REPEAT = parseInt(args.repeat || '1', 10);
const OUTPUT_DIR = args['output-dir']!;
const SCENARIO_FILTER = args.scenario;
@ -325,11 +340,7 @@ async function main() {
}
console.log(`Loaded ${scenarios.length} scenario(s)`);
// Create run directory: results/<model>/<timestamp>/
const sanitizedModel = CHAT_MODEL.replace(/[:/]/g, '-');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const runDir = join(OUTPUT_DIR, sanitizedModel, timestamp);
mkdirSync(runDir, { recursive: true });
const runDir = createRunDir(OUTPUT_DIR, CHAT_MODEL);
console.log(`Output: ${runDir}`);
await initCapture(BASE_URL);

View File

@ -5,7 +5,8 @@
* so model configuration follows the same `provider:model` convention as the rest
* of the codebase. Supports all providers (OpenAI, Google, Anthropic, etc.).
*
* Environment variable: EVAL_SCORER_MODEL (default: openai:gpt-4o)
* The caller supplies the model string explicitly (typically from EVAL_SCORER_MODEL);
* this function no longer has a hardcoded default.
*/
import { readFileSync } from 'fs';
@ -13,8 +14,6 @@ import { generateText } from 'ai';
import { resolveModel } from '@/lib/server/resolve-model';
import type { VlmScore } from './types';
const SCORER_MODEL_DEFAULT = 'openai:gpt-4o';
const RUBRIC_PROMPT = `You are evaluating a classroom whiteboard screenshot from an AI teaching assistant. Score like a teacher reviewing their own board work for a student's benefit.
Context: This is a real-time teaching whiteboard, NOT a poster or infographic.
@ -62,18 +61,16 @@ Output ONLY a JSON object with this exact structure (no markdown, no code fences
/**
* Score a whiteboard screenshot using a VLM.
*
* Model is resolved via EVAL_SCORER_MODEL env var or the provided modelString,
* using the same resolveModel() infrastructure as the rest of the project.
* The caller must provide the model string explicitly (typically from EVAL_SCORER_MODEL);
* this function no longer has a hardcoded default.
*/
export async function scoreScreenshot(
screenshotPath: string,
modelString?: string,
modelString: string,
): Promise<VlmScore> {
const imageBuffer = readFileSync(screenshotPath);
const { model } = await resolveModel({
modelString: modelString || process.env.EVAL_SCORER_MODEL || SCORER_MODEL_DEFAULT,
});
const { model } = await resolveModel({ modelString });
const result = await generateText({
model,

View File

@ -18,7 +18,8 @@
"test": "vitest run",
"test:e2e": "playwright test",
"test:e2e:ui": "playwright test --ui",
"eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts"
"eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts",
"eval:outline-language": "tsx eval/outline-language/runner.ts"
},
"dependencies": {
"@ai-sdk/anthropic": "^3.0.23",

View File

@ -0,0 +1,72 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { writeReport } from '@/eval/outline-language/reporter';
import type { EvalResult } from '@/eval/outline-language/types';
describe('writeReport', () => {
let runDir: string;
beforeEach(() => {
runDir = mkdtempSync(join(tmpdir(), 'reporter-test-'));
});
afterEach(() => {
rmSync(runDir, { recursive: true, force: true });
});
const sample: EvalResult[] = [
{
case_id: 'en-001',
category: 'english-teaching',
requirement: 'teach English to beginners',
groundTruth: 'English as primary',
directive: 'Use English as the teaching language',
outlinesCount: 3,
judgePassed: true,
judgeReason: 'correct language',
},
{
case_id: 'zh-002',
category: 'chinese-teaching',
requirement: '用中文讲授数学',
groundTruth: '以中文为主',
directive: 'Use Chinese throughout',
outlinesCount: 2,
judgePassed: false,
judgeReason: 'wrong phrasing',
},
];
it('writes a report.md file with header, per-case detail, and summary table', () => {
const path = writeReport(runDir, sample, {
inferenceModel: 'openai:gpt-4.1',
judgeModel: 'anthropic:claude-haiku-4-5',
});
expect(existsSync(path)).toBe(true);
expect(path).toBe(join(runDir, 'report.md'));
const content = readFileSync(path, 'utf-8');
expect(content).toContain('# Outline Language Inference Eval Results');
expect(content).toContain('**Model**: openai:gpt-4.1');
expect(content).toContain('**Judge model**: anthropic:claude-haiku-4-5');
expect(content).toContain('**Passed**: 1/2');
expect(content).toContain('### PASS en-001');
expect(content).toContain('### **FAIL** zh-002');
expect(content).toContain('## Summary');
expect(content).toContain('| # | Case | Category | Outlines | Result | Judge reason |');
expect(content).toContain('| 1 | en-001 | english-teaching | 3 | PASS | correct language |');
expect(content).toContain('| 2 | zh-002 | chinese-teaching | 2 | FAIL | wrong phrasing |');
});
it('returns a successfully written path even when all cases pass', () => {
const allPass: EvalResult[] = [{ ...sample[0] }];
const path = writeReport(runDir, allPass, {
inferenceModel: 'openai:gpt-4.1',
judgeModel: 'openai:gpt-4.1',
});
const content = readFileSync(path, 'utf-8');
expect(content).toContain('**Passed**: 1/1 (100%)');
});
});

View File

@ -0,0 +1,51 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
describe('resolveEvalModel', () => {
const ORIG_ENV = process.env;
beforeEach(() => {
process.env = { ...ORIG_ENV };
vi.resetModules();
});
afterEach(() => {
process.env = ORIG_ENV;
vi.restoreAllMocks();
});
it('throws a helpful error when env var is unset and no fallback is given', async () => {
delete process.env.EVAL_FOO_MODEL;
vi.doMock('@/lib/server/resolve-model', () => ({
resolveModel: vi.fn(),
}));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await expect(resolveEvalModel('EVAL_FOO_MODEL')).rejects.toThrow(/EVAL_FOO_MODEL/);
});
it('uses the env var when set', async () => {
process.env.EVAL_FOO_MODEL = 'openai:gpt-4.1';
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'openai:gpt-4.1' });
});
it('uses the explicit fallback when env var is unset', async () => {
delete process.env.EVAL_FOO_MODEL;
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'google:gemini-2.5-flash' });
});
it('env var takes precedence over fallback', async () => {
process.env.EVAL_FOO_MODEL = 'anthropic:claude-haiku-4-5';
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'anthropic:claude-haiku-4-5' });
});
});

View File

@ -0,0 +1,38 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { existsSync, rmSync, mkdtempSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { createRunDir } from '@/eval/shared/run-dir';
describe('createRunDir', () => {
let tempRoot: string;
beforeEach(() => {
tempRoot = mkdtempSync(join(tmpdir(), 'run-dir-test-'));
});
afterEach(() => {
rmSync(tempRoot, { recursive: true, force: true });
});
it('creates <base>/<sanitized-model>/<timestamp>/ and returns the path', () => {
const runDir = createRunDir(tempRoot, 'openai:gpt-4.1');
expect(runDir.startsWith(join(tempRoot, 'openai-gpt-4.1'))).toBe(true);
expect(existsSync(runDir)).toBe(true);
});
it('sanitizes both : and / from the model string', () => {
const runDir = createRunDir(tempRoot, 'google:gemini-2.5-flash/latest');
expect(runDir).toContain('google-gemini-2.5-flash-latest');
expect(runDir).not.toMatch(/[:/]gemini/);
});
it('timestamp segment has no colons or dots', () => {
const runDir = createRunDir(tempRoot, 'x');
const segments = runDir.split('/');
const timestamp = segments[segments.length - 1];
expect(timestamp).not.toContain(':');
expect(timestamp).not.toContain('.');
expect(timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}$/);
});
});

View File

@ -1,272 +0,0 @@
/**
* Outline Language Inference — Real LLM Evaluation Tests
*
* Tests the languageDirective inferred during REAL outline generation.
* Calls the actual generateSceneOutlinesFromRequirements function,
* then uses an LLM-as-judge to evaluate the directive against ground truth.
*
* Calls real LLM APIs — meant to be run locally, NOT in CI/CD.
*
* Environment variables:
* EVAL_INFERENCE_MODEL Model for outline generation (default: DEFAULT_MODEL or gpt-4o-mini)
* EVAL_JUDGE_MODEL Model for LLM-as-judge (default: openai:gpt-4o-mini)
*
* Usage:
* EVAL_INFERENCE_MODEL=google:gemini-3-flash-preview \
* EVAL_JUDGE_MODEL=openai:gpt-4o-mini \
* pnpm vitest run tests/generation/outline-language.eval.test.ts
*
* Results are written to tests/generation/outline-language.eval.result.md
*/
import { describe, it, expect, afterAll, beforeAll } from 'vitest';
import { writeFileSync, readFileSync } from 'fs';
import { resolve } from 'path';
import { generateText } from 'ai';
import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator';
import { resolveModel } from '@/lib/server/resolve-model';
import type { AICallFn } from '@/lib/generation/pipeline-types';
import { callLLM } from '@/lib/ai/llm';
// ---------------------------------------------------------------------------
// Test case definition
// ---------------------------------------------------------------------------
interface LanguageTestCase {
case_id: string;
category: string;
requirement: string;
ground_truth: string;
pdfTextSample?: string;
}
// ---------------------------------------------------------------------------
// Load test cases from JSON
// ---------------------------------------------------------------------------
const TEST_CASES: LanguageTestCase[] = JSON.parse(
readFileSync(resolve(__dirname, 'language-test-cases.json'), 'utf-8'),
);
// ---------------------------------------------------------------------------
// Model resolution
// ---------------------------------------------------------------------------
const JUDGE_MODEL_DEFAULT = 'openai:gpt-4o-mini';
/** Resolve the inference model (EVAL_INFERENCE_MODEL → DEFAULT_MODEL → gpt-4o-mini) */
function getInferenceModel() {
return resolveModel({
modelString: process.env.EVAL_INFERENCE_MODEL || undefined,
});
}
/** Resolve the judge model (EVAL_JUDGE_MODEL → gpt-4o-mini) */
function getJudgeModel() {
return resolveModel({
modelString: process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT,
});
}
// ---------------------------------------------------------------------------
// Build AICallFn from the inference model
// ---------------------------------------------------------------------------
async function buildAICallFn(): Promise<AICallFn> {
const { model: languageModel, modelInfo } = await getInferenceModel();
return async (systemPrompt, userPrompt, _images) => {
const result = await callLLM(
{
model: languageModel,
messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: userPrompt },
],
maxOutputTokens: modelInfo?.outputWindow,
},
'eval-outline-language',
);
return result.text;
};
}
// ---------------------------------------------------------------------------
// LLM Judge
// ---------------------------------------------------------------------------
const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior.
You will be given:
1. The original user requirement
2. The generated language directive
3. The ground truth description of expected behavior
Evaluation criteria — the directive should:
- Use the correct primary teaching language
- Handle terminology in a reasonable way for the subject and audience
- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages
Be LENIENT in your evaluation:
- The directive does NOT need to match the ground truth word-for-word
- Different but equally valid approaches should PASS
- If the teaching language is correct and the overall approach is reasonable, it should PASS
- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation)
Respond with ONLY a JSON object:
{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`;
interface JudgeResult {
pass: boolean;
reason: string;
}
async function judgeDirective(
requirement: string,
directive: string,
groundTruth: string,
): Promise<JudgeResult> {
const { model } = await getJudgeModel();
const result = await generateText({
model,
system: JUDGE_SYSTEM_PROMPT,
prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`,
temperature: 0,
});
try {
const text = result.text.replace(/```json\s*|\s*```/g, '').trim();
return JSON.parse(text) as JudgeResult;
} catch {
return { pass: false, reason: `Failed to parse judge response: ${result.text}` };
}
}
// ---------------------------------------------------------------------------
// Result collector
// ---------------------------------------------------------------------------
interface EvalResult {
case_id: string;
category: string;
requirement: string;
pdfTextSample?: string;
groundTruth: string;
directive: string;
outlinesCount: number;
judgePassed: boolean;
judgeReason: string;
}
const results: EvalResult[] = [];
// ---------------------------------------------------------------------------
// Test runner
// ---------------------------------------------------------------------------
describe('Outline Language Inference Evaluation', () => {
let aiCall: AICallFn;
beforeAll(async () => {
aiCall = await buildAICallFn();
});
for (const tc of TEST_CASES) {
it.concurrent(
`${tc.case_id}: ${tc.requirement.slice(0, 50)}`,
{ timeout: 120_000 },
async () => {
// Call the REAL outline generation function
const result = await generateSceneOutlinesFromRequirements(
{ requirement: tc.requirement },
tc.pdfTextSample || undefined, // pass PDF text if available
undefined, // no PDF images
aiCall,
undefined, // no callbacks
);
expect(result.success, `Outline generation failed: ${result.error}`).toBe(true);
expect(result.data).toBeDefined();
const { languageDirective, outlines } = result.data!;
expect(languageDirective, 'languageDirective should not be empty').toBeTruthy();
expect(outlines.length, 'should generate at least 1 outline').toBeGreaterThan(0);
// LLM-as-judge
const judge = await judgeDirective(tc.requirement, languageDirective, tc.ground_truth);
results.push({
case_id: tc.case_id,
category: tc.category,
requirement: tc.requirement,
pdfTextSample: tc.pdfTextSample,
groundTruth: tc.ground_truth,
directive: languageDirective,
outlinesCount: outlines.length,
judgePassed: judge.pass,
judgeReason: judge.reason,
});
expect(judge.pass, `Judge failed: ${judge.reason}`).toBe(true);
},
);
}
// Write results file after all tests
afterAll(() => {
if (results.length === 0) return;
const passed = results.filter((r) => r.judgePassed).length;
const total = results.length;
const inferenceModelStr =
process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL || '(default: gpt-4o-mini)';
const judgeModelStr = process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT;
const lines: string[] = [
`# Outline Language Inference Eval Results`,
``,
`- **Date**: ${new Date().toISOString()}`,
`- **Passed**: ${passed}/${total} (${((passed / total) * 100).toFixed(0)}%)`,
`- **Inference model**: ${inferenceModelStr}`,
`- **Judge model**: ${judgeModelStr}`,
`- **Method**: real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge`,
`- **Test cases**: curated from production data`,
``,
`## Detail`,
``,
];
for (const r of results) {
const icon = r.judgePassed ? 'PASS' : '**FAIL**';
lines.push(`### ${icon} ${r.case_id}`);
lines.push(``);
lines.push(`- **Category**: ${r.category}`);
lines.push(`- **Input**: \`${r.requirement}\``);
if (r.pdfTextSample) {
lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``);
}
lines.push(`- **Ground truth**: ${r.groundTruth}`);
lines.push(`- **Directive**: ${r.directive}`);
lines.push(`- **Outlines generated**: ${r.outlinesCount}`);
lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`);
lines.push(``);
}
lines.push(`## Summary`);
lines.push(``);
lines.push(`| # | Case | Category | Outlines | Result | Judge reason |`);
lines.push(`|---|------|----------|----------|--------|--------------|`);
for (let i = 0; i < results.length; i++) {
const r = results[i];
const icon = r.judgePassed ? 'PASS' : 'FAIL';
lines.push(
`| ${i + 1} | ${r.case_id} | ${r.category} | ${r.outlinesCount} | ${icon} | ${r.judgeReason} |`,
);
}
lines.push(``);
const outPath = resolve(__dirname, 'outline-language.eval.result.md');
writeFileSync(outPath, lines.join('\n'), 'utf-8');
console.log(`\nResults written to: ${outPath}`);
});
});

View File

@ -9,7 +9,6 @@ export default defineConfig({
},
test: {
include: ['tests/**/*.test.ts'],
exclude: ['tests/**/*.eval.test.ts'],
setupFiles: ['tests/setup-env.ts'],
},
});