diff --git a/.gitignore b/.gitignore index 5423a7e..e1984b9 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,6 @@ next-env.d.ts # docs /docs -tests/generation/*.eval.result.md - # Eval results eval/whiteboard-layout/results/ +eval/outline-language/results/ diff --git a/eval/outline-language/judge.ts b/eval/outline-language/judge.ts new file mode 100644 index 0000000..ec55f95 --- /dev/null +++ b/eval/outline-language/judge.ts @@ -0,0 +1,48 @@ +import { generateText, type LanguageModel } from 'ai'; +import type { JudgeResult } from './types'; + +const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior. + +You will be given: +1. The original user requirement +2. The generated language directive +3. The ground truth description of expected behavior + +Evaluation criteria — the directive should: +- Use the correct primary teaching language +- Handle terminology in a reasonable way for the subject and audience +- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages + +Be LENIENT in your evaluation: +- The directive does NOT need to match the ground truth word-for-word +- Different but equally valid approaches should PASS +- If the teaching language is correct and the overall approach is reasonable, it should PASS +- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation) + +Respond with ONLY a JSON object: +{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`; + +/** + * Ask an LLM-as-judge whether `directive` is a reasonable language directive + * for `requirement` given `groundTruth`. Lenient rubric — see system prompt. + */ +export async function judgeDirective( + judgeModel: LanguageModel, + requirement: string, + directive: string, + groundTruth: string, +): Promise { + const result = await generateText({ + model: judgeModel, + system: JUDGE_SYSTEM_PROMPT, + prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`, + temperature: 0, + }); + + try { + const text = result.text.replace(/```json\s*|\s*```/g, '').trim(); + return JSON.parse(text) as JudgeResult; + } catch { + return { pass: false, reason: `Failed to parse judge response: ${result.text}` }; + } +} diff --git a/eval/outline-language/reporter.ts b/eval/outline-language/reporter.ts new file mode 100644 index 0000000..924f058 --- /dev/null +++ b/eval/outline-language/reporter.ts @@ -0,0 +1,70 @@ +import { writeFileSync } from 'fs'; +import { join } from 'path'; +import { renderHeader, renderSummaryTable } from '../shared/markdown-report'; +import type { EvalResult } from './types'; + +export interface ReportContext { + inferenceModel: string; + judgeModel: string; +} + +/** + * Write `report.md` into `runDir`. Returns the absolute path of the written file. + * + * Structure mirrors the old `outline-language.eval.result.md`: + * 1. Header (date, models, pass count) + * 2. One detail block per case (PASS / **FAIL**) + * 3. Summary table of all cases + */ +export function writeReport(runDir: string, results: EvalResult[], ctx: ReportContext): string { + const passed = results.filter((r) => r.judgePassed).length; + const total = results.length; + const pct = total === 0 ? 0 : Math.round((passed / total) * 100); + + const lines: string[] = []; + lines.push( + ...renderHeader({ + title: 'Outline Language Inference Eval Results', + timestamp: new Date().toISOString(), + model: ctx.inferenceModel, + judgeModel: ctx.judgeModel, + extra: { + Passed: `${passed}/${total} (${pct}%)`, + Method: 'real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge', + }, + }), + ); + + lines.push(`## Detail`, ``); + for (const r of results) { + const icon = r.judgePassed ? 'PASS' : '**FAIL**'; + lines.push(`### ${icon} ${r.case_id}`, ``); + lines.push(`- **Category**: ${r.category}`); + lines.push(`- **Input**: \`${r.requirement}\``); + if (r.pdfTextSample) { + lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``); + } + lines.push(`- **Ground truth**: ${r.groundTruth}`); + lines.push(`- **Directive**: ${r.directive}`); + lines.push(`- **Outlines generated**: ${r.outlinesCount}`); + lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`); + lines.push(``); + } + + lines.push(`## Summary`, ``); + const rows: string[][] = results.map((r, i) => [ + String(i + 1), + r.case_id, + r.category, + String(r.outlinesCount), + r.judgePassed ? 'PASS' : 'FAIL', + r.judgeReason, + ]); + lines.push( + ...renderSummaryTable(['#', 'Case', 'Category', 'Outlines', 'Result', 'Judge reason'], rows), + ); + + const outPath = join(runDir, 'report.md'); + writeFileSync(outPath, lines.join('\n'), 'utf-8'); + return outPath; +} diff --git a/eval/outline-language/runner.ts b/eval/outline-language/runner.ts new file mode 100644 index 0000000..b39b6f1 --- /dev/null +++ b/eval/outline-language/runner.ts @@ -0,0 +1,174 @@ +/** + * Outline Language Inference — Real LLM Evaluation Runner + * + * Calls generateSceneOutlinesFromRequirements for each test case, then uses + * an LLM-as-judge to score the inferred languageDirective against ground truth. + * + * Required env: + * EVAL_INFERENCE_MODEL Model for outline generation (or DEFAULT_MODEL) + * EVAL_JUDGE_MODEL Model for LLM-as-judge + * + * Usage: + * EVAL_INFERENCE_MODEL= EVAL_JUDGE_MODEL= \ + * pnpm eval:outline-language + * + * Output: eval/outline-language/results///report.md + */ + +import { readFileSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator'; +import { callLLM } from '@/lib/ai/llm'; +import type { AICallFn } from '@/lib/generation/pipeline-types'; +import { resolveEvalModel } from '../shared/resolve-model'; +import { createRunDir } from '../shared/run-dir'; +import { judgeDirective } from './judge'; +import { writeReport } from './reporter'; +import type { LanguageTestCase, EvalResult } from './types'; + +const OUTPUT_DIR = 'eval/outline-language/results'; + +function getCurrentDir(): string { + return typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url)); +} + +function loadScenarios(): LanguageTestCase[] { + const path = join(getCurrentDir(), 'scenarios/language-test-cases.json'); + return JSON.parse(readFileSync(path, 'utf-8')) as LanguageTestCase[]; +} + +// Pre-validate env with tailored messages (including example model strings). +// resolveEvalModel() also throws on missing vars, but with a shorter message; +// surfacing the example before any async work makes misconfiguration obvious. +function requireModelEnv(): { inferenceModelStr: string; judgeModelStr: string } { + const inferenceModelStr = process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL; + const judgeModelStr = process.env.EVAL_JUDGE_MODEL; + if (!inferenceModelStr) { + console.error( + 'Error: EVAL_INFERENCE_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_INFERENCE_MODEL=openai:gpt-4.1', + ); + process.exit(1); + } + if (!judgeModelStr) { + console.error( + 'Error: EVAL_JUDGE_MODEL must be set. Example: EVAL_JUDGE_MODEL=anthropic:claude-haiku-4-5', + ); + process.exit(1); + } + return { inferenceModelStr, judgeModelStr }; +} + +async function runCase( + tc: LanguageTestCase, + aiCall: AICallFn, + judgeModel: Awaited>['model'], +): Promise { + try { + const result = await generateSceneOutlinesFromRequirements( + { requirement: tc.requirement }, + tc.pdfTextSample || undefined, + undefined, + aiCall, + undefined, + ); + + if (!result.success || !result.data) { + return { + case_id: tc.case_id, + category: tc.category, + requirement: tc.requirement, + pdfTextSample: tc.pdfTextSample, + groundTruth: tc.ground_truth, + directive: '', + outlinesCount: 0, + judgePassed: false, + judgeReason: `Outline generation failed: ${result.error || 'unknown error'}`, + }; + } + + const { languageDirective, outlines } = result.data; + const judge = await judgeDirective( + judgeModel, + tc.requirement, + languageDirective, + tc.ground_truth, + ); + + return { + case_id: tc.case_id, + category: tc.category, + requirement: tc.requirement, + pdfTextSample: tc.pdfTextSample, + groundTruth: tc.ground_truth, + directive: languageDirective, + outlinesCount: outlines.length, + judgePassed: judge.pass, + judgeReason: judge.reason, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { + case_id: tc.case_id, + category: tc.category, + requirement: tc.requirement, + pdfTextSample: tc.pdfTextSample, + groundTruth: tc.ground_truth, + directive: '', + outlinesCount: 0, + judgePassed: false, + judgeReason: `Exception: ${msg}`, + }; + } +} + +async function main() { + const { inferenceModelStr, judgeModelStr } = requireModelEnv(); + + console.log('=== Outline Language Inference Eval ==='); + console.log(`Inference: ${inferenceModelStr} | Judge: ${judgeModelStr}`); + + const { model: inferenceModel, modelInfo } = await resolveEvalModel( + 'EVAL_INFERENCE_MODEL', + process.env.DEFAULT_MODEL, + ); + const { model: judgeModel } = await resolveEvalModel('EVAL_JUDGE_MODEL'); + + const aiCall: AICallFn = async (systemPrompt, userPrompt, _images) => { + const result = await callLLM( + { + model: inferenceModel, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userPrompt }, + ], + maxOutputTokens: modelInfo?.outputWindow, + }, + 'eval-outline-language', + ); + return result.text; + }; + + const cases = loadScenarios(); + console.log(`Loaded ${cases.length} test case(s)`); + + const runDir = createRunDir(OUTPUT_DIR, inferenceModelStr); + console.log(`Output: ${runDir}`); + + const results = await Promise.all(cases.map((tc) => runCase(tc, aiCall, judgeModel))); + + const reportPath = writeReport(runDir, results, { + inferenceModel: inferenceModelStr, + judgeModel: judgeModelStr, + }); + const passed = results.filter((r) => r.judgePassed).length; + console.log(`\nReport: ${reportPath}`); + console.log(`Passed: ${passed}/${results.length}`); + + process.exit(passed === results.length ? 0 : 1); +} + +main().catch((err) => { + console.error('Fatal error:', err); + process.exit(1); +}); diff --git a/tests/generation/language-test-cases.json b/eval/outline-language/scenarios/language-test-cases.json similarity index 100% rename from tests/generation/language-test-cases.json rename to eval/outline-language/scenarios/language-test-cases.json diff --git a/eval/outline-language/types.ts b/eval/outline-language/types.ts new file mode 100644 index 0000000..a040719 --- /dev/null +++ b/eval/outline-language/types.ts @@ -0,0 +1,24 @@ +export interface LanguageTestCase { + case_id: string; + category: string; + requirement: string; + ground_truth: string; + pdfTextSample?: string; +} + +export interface JudgeResult { + pass: boolean; + reason: string; +} + +export interface EvalResult { + case_id: string; + category: string; + requirement: string; + pdfTextSample?: string; + groundTruth: string; + directive: string; + outlinesCount: number; + judgePassed: boolean; + judgeReason: string; +} diff --git a/eval/shared/markdown-report.ts b/eval/shared/markdown-report.ts new file mode 100644 index 0000000..79411f1 --- /dev/null +++ b/eval/shared/markdown-report.ts @@ -0,0 +1,35 @@ +/** + * Thin markdown helpers shared across eval reporters. Each returns `string[]` + * so callers can push lines directly into their own buffer: + * + * const lines: string[] = []; + * lines.push(...renderHeader({ title: 'Foo', ... })); + * lines.push(...renderSummaryTable(['A', 'B'], rows)); + * writeFileSync(path, lines.join('\n')); + */ + +export interface ReportHeader { + title: string; + timestamp: string; + model: string; + judgeModel?: string; + extra?: Record; +} + +export function renderHeader(h: ReportHeader): string[] { + const lines = [`# ${h.title}`, ``, `- **Date**: ${h.timestamp}`, `- **Model**: ${h.model}`]; + if (h.judgeModel) lines.push(`- **Judge model**: ${h.judgeModel}`); + for (const [k, v] of Object.entries(h.extra || {})) { + lines.push(`- **${k}**: ${v}`); + } + lines.push(``); + return lines; +} + +export function renderSummaryTable(headers: string[], rows: string[][]): string[] { + const sep = `|${headers.map(() => '---').join('|')}|`; + const lines = [`| ${headers.join(' | ')} |`, sep]; + for (const r of rows) lines.push(`| ${r.map((c) => c.replace(/\|/g, '\\|')).join(' | ')} |`); + lines.push(``); + return lines; +} diff --git a/eval/shared/resolve-model.ts b/eval/shared/resolve-model.ts new file mode 100644 index 0000000..9e4b870 --- /dev/null +++ b/eval/shared/resolve-model.ts @@ -0,0 +1,18 @@ +import { resolveModel } from '@/lib/server/resolve-model'; + +/** + * Resolve a model for an eval runner. Reads `process.env[envVar]`, falls back + * to `fallback` if provided, and throws a clear error if neither is set. + * + * Never introduces a hardcoded default model string — evals must be explicit + * about what they measure. + */ +export async function resolveEvalModel(envVar: string, fallback?: string) { + const modelString = process.env[envVar] || fallback; + if (!modelString) { + throw new Error( + `Eval model not configured: set ${envVar} in the environment (or pass an explicit fallback).`, + ); + } + return resolveModel({ modelString }); +} diff --git a/eval/shared/run-dir.ts b/eval/shared/run-dir.ts new file mode 100644 index 0000000..7e35252 --- /dev/null +++ b/eval/shared/run-dir.ts @@ -0,0 +1,16 @@ +import { mkdirSync } from 'fs'; +import { join } from 'path'; + +/** + * Build and create a run directory under `///`. + * The model string is sanitized by replacing `:` and `/` with `-` so it is + * safe to use as a directory name. Timestamp is ISO-8601 with colons and dots + * replaced by dashes, truncated to second precision. + */ +export function createRunDir(baseDir: string, model: string): string { + const sanitizedModel = model.replace(/[:/]/g, '-'); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); + const runDir = join(baseDir, sanitizedModel, timestamp); + mkdirSync(runDir, { recursive: true }); + return runDir; +} diff --git a/eval/whiteboard-layout/runner.ts b/eval/whiteboard-layout/runner.ts index 0d25ac1..64e74ec 100644 --- a/eval/whiteboard-layout/runner.ts +++ b/eval/whiteboard-layout/runner.ts @@ -9,16 +9,17 @@ import { EvalStateManager } from './state-manager'; import { initCapture, captureWhiteboard, closeCapture } from './capture'; import { scoreScreenshot } from './scorer'; import { generateReport } from './reporter'; +import { createRunDir } from '../shared/run-dir'; // ==================== CLI Args ==================== // -// Model configuration follows the same pattern as the outline-language eval: -// EVAL_CHAT_MODEL Model for chat generation (default: DEFAULT_MODEL or gpt-4o-mini) -// EVAL_SCORER_MODEL Model for VLM scoring (default: openai:gpt-4o) +// Required env: +// EVAL_CHAT_MODEL (or DEFAULT_MODEL) Model for chat generation +// EVAL_SCORER_MODEL Model for VLM scoring // // Usage: -// EVAL_CHAT_MODEL=google:gemini-3.1-pro-preview \ -// EVAL_SCORER_MODEL=google:gemini-2.0-flash \ +// EVAL_CHAT_MODEL= \ +// EVAL_SCORER_MODEL= \ // pnpm eval:whiteboard --scenario physics-force-decomposition const { values: args } = parseArgs({ @@ -32,8 +33,22 @@ const { values: args } = parseArgs({ }); const BASE_URL = args['base-url']!; -const CHAT_MODEL = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL || 'openai:gpt-4o-mini'; -const SCORER_MODEL = process.env.EVAL_SCORER_MODEL || 'openai:gpt-4o'; +const CHAT_MODEL_RAW = process.env.EVAL_CHAT_MODEL || process.env.DEFAULT_MODEL; +const SCORER_MODEL_RAW = process.env.EVAL_SCORER_MODEL; +if (!CHAT_MODEL_RAW) { + console.error( + 'Error: EVAL_CHAT_MODEL (or DEFAULT_MODEL) must be set. Example: EVAL_CHAT_MODEL=openai:gpt-4.1', + ); + process.exit(1); +} +if (!SCORER_MODEL_RAW) { + console.error( + 'Error: EVAL_SCORER_MODEL must be set. Example: EVAL_SCORER_MODEL=google:gemini-2.5-flash', + ); + process.exit(1); +} +const CHAT_MODEL: string = CHAT_MODEL_RAW; +const SCORER_MODEL: string = SCORER_MODEL_RAW; const REPEAT = parseInt(args.repeat || '1', 10); const OUTPUT_DIR = args['output-dir']!; const SCENARIO_FILTER = args.scenario; @@ -325,11 +340,7 @@ async function main() { } console.log(`Loaded ${scenarios.length} scenario(s)`); - // Create run directory: results/// - const sanitizedModel = CHAT_MODEL.replace(/[:/]/g, '-'); - const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); - const runDir = join(OUTPUT_DIR, sanitizedModel, timestamp); - mkdirSync(runDir, { recursive: true }); + const runDir = createRunDir(OUTPUT_DIR, CHAT_MODEL); console.log(`Output: ${runDir}`); await initCapture(BASE_URL); diff --git a/eval/whiteboard-layout/scorer.ts b/eval/whiteboard-layout/scorer.ts index ca2c1f4..9be528a 100644 --- a/eval/whiteboard-layout/scorer.ts +++ b/eval/whiteboard-layout/scorer.ts @@ -5,7 +5,8 @@ * so model configuration follows the same `provider:model` convention as the rest * of the codebase. Supports all providers (OpenAI, Google, Anthropic, etc.). * - * Environment variable: EVAL_SCORER_MODEL (default: openai:gpt-4o) + * The caller supplies the model string explicitly (typically from EVAL_SCORER_MODEL); + * this function no longer has a hardcoded default. */ import { readFileSync } from 'fs'; @@ -13,8 +14,6 @@ import { generateText } from 'ai'; import { resolveModel } from '@/lib/server/resolve-model'; import type { VlmScore } from './types'; -const SCORER_MODEL_DEFAULT = 'openai:gpt-4o'; - const RUBRIC_PROMPT = `You are evaluating a classroom whiteboard screenshot from an AI teaching assistant. Score like a teacher reviewing their own board work for a student's benefit. Context: This is a real-time teaching whiteboard, NOT a poster or infographic. @@ -62,18 +61,16 @@ Output ONLY a JSON object with this exact structure (no markdown, no code fences /** * Score a whiteboard screenshot using a VLM. * - * Model is resolved via EVAL_SCORER_MODEL env var or the provided modelString, - * using the same resolveModel() infrastructure as the rest of the project. + * The caller must provide the model string explicitly (typically from EVAL_SCORER_MODEL); + * this function no longer has a hardcoded default. */ export async function scoreScreenshot( screenshotPath: string, - modelString?: string, + modelString: string, ): Promise { const imageBuffer = readFileSync(screenshotPath); - const { model } = await resolveModel({ - modelString: modelString || process.env.EVAL_SCORER_MODEL || SCORER_MODEL_DEFAULT, - }); + const { model } = await resolveModel({ modelString }); const result = await generateText({ model, diff --git a/package.json b/package.json index 3225170..85f0a7e 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "test": "vitest run", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", - "eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts" + "eval:whiteboard": "tsx eval/whiteboard-layout/runner.ts", + "eval:outline-language": "tsx eval/outline-language/runner.ts" }, "dependencies": { "@ai-sdk/anthropic": "^3.0.23", diff --git a/tests/eval/outline-language/reporter.test.ts b/tests/eval/outline-language/reporter.test.ts new file mode 100644 index 0000000..c6d1d33 --- /dev/null +++ b/tests/eval/outline-language/reporter.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { writeReport } from '@/eval/outline-language/reporter'; +import type { EvalResult } from '@/eval/outline-language/types'; + +describe('writeReport', () => { + let runDir: string; + + beforeEach(() => { + runDir = mkdtempSync(join(tmpdir(), 'reporter-test-')); + }); + + afterEach(() => { + rmSync(runDir, { recursive: true, force: true }); + }); + + const sample: EvalResult[] = [ + { + case_id: 'en-001', + category: 'english-teaching', + requirement: 'teach English to beginners', + groundTruth: 'English as primary', + directive: 'Use English as the teaching language', + outlinesCount: 3, + judgePassed: true, + judgeReason: 'correct language', + }, + { + case_id: 'zh-002', + category: 'chinese-teaching', + requirement: '用中文讲授数学', + groundTruth: '以中文为主', + directive: 'Use Chinese throughout', + outlinesCount: 2, + judgePassed: false, + judgeReason: 'wrong phrasing', + }, + ]; + + it('writes a report.md file with header, per-case detail, and summary table', () => { + const path = writeReport(runDir, sample, { + inferenceModel: 'openai:gpt-4.1', + judgeModel: 'anthropic:claude-haiku-4-5', + }); + expect(existsSync(path)).toBe(true); + expect(path).toBe(join(runDir, 'report.md')); + + const content = readFileSync(path, 'utf-8'); + expect(content).toContain('# Outline Language Inference Eval Results'); + expect(content).toContain('**Model**: openai:gpt-4.1'); + expect(content).toContain('**Judge model**: anthropic:claude-haiku-4-5'); + expect(content).toContain('**Passed**: 1/2'); + expect(content).toContain('### PASS en-001'); + expect(content).toContain('### **FAIL** zh-002'); + expect(content).toContain('## Summary'); + expect(content).toContain('| # | Case | Category | Outlines | Result | Judge reason |'); + expect(content).toContain('| 1 | en-001 | english-teaching | 3 | PASS | correct language |'); + expect(content).toContain('| 2 | zh-002 | chinese-teaching | 2 | FAIL | wrong phrasing |'); + }); + + it('returns a successfully written path even when all cases pass', () => { + const allPass: EvalResult[] = [{ ...sample[0] }]; + const path = writeReport(runDir, allPass, { + inferenceModel: 'openai:gpt-4.1', + judgeModel: 'openai:gpt-4.1', + }); + const content = readFileSync(path, 'utf-8'); + expect(content).toContain('**Passed**: 1/1 (100%)'); + }); +}); diff --git a/tests/eval/shared/resolve-model.test.ts b/tests/eval/shared/resolve-model.test.ts new file mode 100644 index 0000000..40768f6 --- /dev/null +++ b/tests/eval/shared/resolve-model.test.ts @@ -0,0 +1,51 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + +describe('resolveEvalModel', () => { + const ORIG_ENV = process.env; + + beforeEach(() => { + process.env = { ...ORIG_ENV }; + vi.resetModules(); + }); + + afterEach(() => { + process.env = ORIG_ENV; + vi.restoreAllMocks(); + }); + + it('throws a helpful error when env var is unset and no fallback is given', async () => { + delete process.env.EVAL_FOO_MODEL; + vi.doMock('@/lib/server/resolve-model', () => ({ + resolveModel: vi.fn(), + })); + const { resolveEvalModel } = await import('@/eval/shared/resolve-model'); + await expect(resolveEvalModel('EVAL_FOO_MODEL')).rejects.toThrow(/EVAL_FOO_MODEL/); + }); + + it('uses the env var when set', async () => { + process.env.EVAL_FOO_MODEL = 'openai:gpt-4.1'; + const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} }); + vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel })); + const { resolveEvalModel } = await import('@/eval/shared/resolve-model'); + await resolveEvalModel('EVAL_FOO_MODEL'); + expect(resolveModel).toHaveBeenCalledWith({ modelString: 'openai:gpt-4.1' }); + }); + + it('uses the explicit fallback when env var is unset', async () => { + delete process.env.EVAL_FOO_MODEL; + const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} }); + vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel })); + const { resolveEvalModel } = await import('@/eval/shared/resolve-model'); + await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash'); + expect(resolveModel).toHaveBeenCalledWith({ modelString: 'google:gemini-2.5-flash' }); + }); + + it('env var takes precedence over fallback', async () => { + process.env.EVAL_FOO_MODEL = 'anthropic:claude-haiku-4-5'; + const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} }); + vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel })); + const { resolveEvalModel } = await import('@/eval/shared/resolve-model'); + await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash'); + expect(resolveModel).toHaveBeenCalledWith({ modelString: 'anthropic:claude-haiku-4-5' }); + }); +}); diff --git a/tests/eval/shared/run-dir.test.ts b/tests/eval/shared/run-dir.test.ts new file mode 100644 index 0000000..71ef561 --- /dev/null +++ b/tests/eval/shared/run-dir.test.ts @@ -0,0 +1,38 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { existsSync, rmSync, mkdtempSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { createRunDir } from '@/eval/shared/run-dir'; + +describe('createRunDir', () => { + let tempRoot: string; + + beforeEach(() => { + tempRoot = mkdtempSync(join(tmpdir(), 'run-dir-test-')); + }); + + afterEach(() => { + rmSync(tempRoot, { recursive: true, force: true }); + }); + + it('creates /// and returns the path', () => { + const runDir = createRunDir(tempRoot, 'openai:gpt-4.1'); + expect(runDir.startsWith(join(tempRoot, 'openai-gpt-4.1'))).toBe(true); + expect(existsSync(runDir)).toBe(true); + }); + + it('sanitizes both : and / from the model string', () => { + const runDir = createRunDir(tempRoot, 'google:gemini-2.5-flash/latest'); + expect(runDir).toContain('google-gemini-2.5-flash-latest'); + expect(runDir).not.toMatch(/[:/]gemini/); + }); + + it('timestamp segment has no colons or dots', () => { + const runDir = createRunDir(tempRoot, 'x'); + const segments = runDir.split('/'); + const timestamp = segments[segments.length - 1]; + expect(timestamp).not.toContain(':'); + expect(timestamp).not.toContain('.'); + expect(timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}$/); + }); +}); diff --git a/tests/generation/outline-language.eval.test.ts b/tests/generation/outline-language.eval.test.ts deleted file mode 100644 index d929c85..0000000 --- a/tests/generation/outline-language.eval.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -/** - * Outline Language Inference — Real LLM Evaluation Tests - * - * Tests the languageDirective inferred during REAL outline generation. - * Calls the actual generateSceneOutlinesFromRequirements function, - * then uses an LLM-as-judge to evaluate the directive against ground truth. - * - * Calls real LLM APIs — meant to be run locally, NOT in CI/CD. - * - * Environment variables: - * EVAL_INFERENCE_MODEL Model for outline generation (default: DEFAULT_MODEL or gpt-4o-mini) - * EVAL_JUDGE_MODEL Model for LLM-as-judge (default: openai:gpt-4o-mini) - * - * Usage: - * EVAL_INFERENCE_MODEL=google:gemini-3-flash-preview \ - * EVAL_JUDGE_MODEL=openai:gpt-4o-mini \ - * pnpm vitest run tests/generation/outline-language.eval.test.ts - * - * Results are written to tests/generation/outline-language.eval.result.md - */ - -import { describe, it, expect, afterAll, beforeAll } from 'vitest'; -import { writeFileSync, readFileSync } from 'fs'; -import { resolve } from 'path'; -import { generateText } from 'ai'; -import { generateSceneOutlinesFromRequirements } from '@/lib/generation/outline-generator'; -import { resolveModel } from '@/lib/server/resolve-model'; -import type { AICallFn } from '@/lib/generation/pipeline-types'; -import { callLLM } from '@/lib/ai/llm'; - -// --------------------------------------------------------------------------- -// Test case definition -// --------------------------------------------------------------------------- - -interface LanguageTestCase { - case_id: string; - category: string; - requirement: string; - ground_truth: string; - pdfTextSample?: string; -} - -// --------------------------------------------------------------------------- -// Load test cases from JSON -// --------------------------------------------------------------------------- - -const TEST_CASES: LanguageTestCase[] = JSON.parse( - readFileSync(resolve(__dirname, 'language-test-cases.json'), 'utf-8'), -); - -// --------------------------------------------------------------------------- -// Model resolution -// --------------------------------------------------------------------------- - -const JUDGE_MODEL_DEFAULT = 'openai:gpt-4o-mini'; - -/** Resolve the inference model (EVAL_INFERENCE_MODEL → DEFAULT_MODEL → gpt-4o-mini) */ -function getInferenceModel() { - return resolveModel({ - modelString: process.env.EVAL_INFERENCE_MODEL || undefined, - }); -} - -/** Resolve the judge model (EVAL_JUDGE_MODEL → gpt-4o-mini) */ -function getJudgeModel() { - return resolveModel({ - modelString: process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT, - }); -} - -// --------------------------------------------------------------------------- -// Build AICallFn from the inference model -// --------------------------------------------------------------------------- - -async function buildAICallFn(): Promise { - const { model: languageModel, modelInfo } = await getInferenceModel(); - return async (systemPrompt, userPrompt, _images) => { - const result = await callLLM( - { - model: languageModel, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userPrompt }, - ], - maxOutputTokens: modelInfo?.outputWindow, - }, - 'eval-outline-language', - ); - return result.text; - }; -} - -// --------------------------------------------------------------------------- -// LLM Judge -// --------------------------------------------------------------------------- - -const JUDGE_SYSTEM_PROMPT = `You are evaluating whether a language directive for an AI course generation system is reasonable given the expected behavior. - -You will be given: -1. The original user requirement -2. The generated language directive -3. The ground truth description of expected behavior - -Evaluation criteria — the directive should: -- Use the correct primary teaching language -- Handle terminology in a reasonable way for the subject and audience -- For cross-language scenarios (foreign language learning, cross-language PDF), acknowledge both languages - -Be LENIENT in your evaluation: -- The directive does NOT need to match the ground truth word-for-word -- Different but equally valid approaches should PASS -- If the teaching language is correct and the overall approach is reasonable, it should PASS -- Only FAIL if the directive is clearly WRONG (e.g., wrong teaching language, completely ignoring a cross-language situation) - -Respond with ONLY a JSON object: -{"pass": true/false, "reason": "brief explanation (1-2 sentences)"}`; - -interface JudgeResult { - pass: boolean; - reason: string; -} - -async function judgeDirective( - requirement: string, - directive: string, - groundTruth: string, -): Promise { - const { model } = await getJudgeModel(); - const result = await generateText({ - model, - system: JUDGE_SYSTEM_PROMPT, - prompt: `Requirement: "${requirement}"\n\nGenerated directive: "${directive}"\n\nGround truth: "${groundTruth}"`, - temperature: 0, - }); - - try { - const text = result.text.replace(/```json\s*|\s*```/g, '').trim(); - return JSON.parse(text) as JudgeResult; - } catch { - return { pass: false, reason: `Failed to parse judge response: ${result.text}` }; - } -} - -// --------------------------------------------------------------------------- -// Result collector -// --------------------------------------------------------------------------- - -interface EvalResult { - case_id: string; - category: string; - requirement: string; - pdfTextSample?: string; - groundTruth: string; - directive: string; - outlinesCount: number; - judgePassed: boolean; - judgeReason: string; -} - -const results: EvalResult[] = []; - -// --------------------------------------------------------------------------- -// Test runner -// --------------------------------------------------------------------------- - -describe('Outline Language Inference Evaluation', () => { - let aiCall: AICallFn; - - beforeAll(async () => { - aiCall = await buildAICallFn(); - }); - - for (const tc of TEST_CASES) { - it.concurrent( - `${tc.case_id}: ${tc.requirement.slice(0, 50)}`, - { timeout: 120_000 }, - async () => { - // Call the REAL outline generation function - const result = await generateSceneOutlinesFromRequirements( - { requirement: tc.requirement }, - tc.pdfTextSample || undefined, // pass PDF text if available - undefined, // no PDF images - aiCall, - undefined, // no callbacks - ); - - expect(result.success, `Outline generation failed: ${result.error}`).toBe(true); - expect(result.data).toBeDefined(); - - const { languageDirective, outlines } = result.data!; - expect(languageDirective, 'languageDirective should not be empty').toBeTruthy(); - expect(outlines.length, 'should generate at least 1 outline').toBeGreaterThan(0); - - // LLM-as-judge - const judge = await judgeDirective(tc.requirement, languageDirective, tc.ground_truth); - - results.push({ - case_id: tc.case_id, - category: tc.category, - requirement: tc.requirement, - pdfTextSample: tc.pdfTextSample, - groundTruth: tc.ground_truth, - directive: languageDirective, - outlinesCount: outlines.length, - judgePassed: judge.pass, - judgeReason: judge.reason, - }); - - expect(judge.pass, `Judge failed: ${judge.reason}`).toBe(true); - }, - ); - } - - // Write results file after all tests - afterAll(() => { - if (results.length === 0) return; - - const passed = results.filter((r) => r.judgePassed).length; - const total = results.length; - const inferenceModelStr = - process.env.EVAL_INFERENCE_MODEL || process.env.DEFAULT_MODEL || '(default: gpt-4o-mini)'; - const judgeModelStr = process.env.EVAL_JUDGE_MODEL || JUDGE_MODEL_DEFAULT; - - const lines: string[] = [ - `# Outline Language Inference Eval Results`, - ``, - `- **Date**: ${new Date().toISOString()}`, - `- **Passed**: ${passed}/${total} (${((passed / total) * 100).toFixed(0)}%)`, - `- **Inference model**: ${inferenceModelStr}`, - `- **Judge model**: ${judgeModelStr}`, - `- **Method**: real outline generation (generateSceneOutlinesFromRequirements) + LLM-as-judge`, - `- **Test cases**: curated from production data`, - ``, - `## Detail`, - ``, - ]; - - for (const r of results) { - const icon = r.judgePassed ? 'PASS' : '**FAIL**'; - - lines.push(`### ${icon} ${r.case_id}`); - lines.push(``); - lines.push(`- **Category**: ${r.category}`); - lines.push(`- **Input**: \`${r.requirement}\``); - if (r.pdfTextSample) { - lines.push(`- **PDF sample**: \`${r.pdfTextSample.slice(0, 80)}...\``); - } - lines.push(`- **Ground truth**: ${r.groundTruth}`); - lines.push(`- **Directive**: ${r.directive}`); - lines.push(`- **Outlines generated**: ${r.outlinesCount}`); - lines.push(`- **Judge**: ${r.judgePassed ? 'PASS' : 'FAIL'} — ${r.judgeReason}`); - lines.push(``); - } - - lines.push(`## Summary`); - lines.push(``); - lines.push(`| # | Case | Category | Outlines | Result | Judge reason |`); - lines.push(`|---|------|----------|----------|--------|--------------|`); - for (let i = 0; i < results.length; i++) { - const r = results[i]; - const icon = r.judgePassed ? 'PASS' : 'FAIL'; - lines.push( - `| ${i + 1} | ${r.case_id} | ${r.category} | ${r.outlinesCount} | ${icon} | ${r.judgeReason} |`, - ); - } - lines.push(``); - - const outPath = resolve(__dirname, 'outline-language.eval.result.md'); - writeFileSync(outPath, lines.join('\n'), 'utf-8'); - console.log(`\nResults written to: ${outPath}`); - }); -}); diff --git a/vitest.config.ts b/vitest.config.ts index b7b2b4f..29c4eb5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,7 +9,6 @@ export default defineConfig({ }, test: { include: ['tests/**/*.test.ts'], - exclude: ['tests/**/*.eval.test.ts'], setupFiles: ['tests/setup-env.ts'], }, });