* feat(eval): add resolveEvalModel helper with fail-fast Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(eval): add createRunDir helper with path sanitization Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(eval): add markdown report helpers * refactor(eval): move language test cases under eval/outline-language * feat(eval): add outline-language types * feat(eval): add outline-language LLM judge * feat(eval): add outline-language reporter * feat(eval): add outline-language runner entry * chore(eval): add eval:outline-language pnpm script * refactor(eval): adopt shared createRunDir and drop gpt-4o fallbacks in whiteboard runner * refactor(eval): drop redundant non-null assertions after narrowing Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * refactor(eval): remove SCORER_MODEL_DEFAULT hardcoded gpt-4o fallback * chore(eval): delete tests/generation and clean up vitest/gitignore config * docs(eval): explain why outline-language runner pre-validates env vars Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(eval): escape pipe chars in markdown summary table cells LLM judge output may contain | which breaks GFM table rendering. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: 杨慎 <117187635+cosarah@users.noreply.github.com>
36 lines
1.2 KiB
TypeScript
36 lines
1.2 KiB
TypeScript
/**
|
|
* Thin markdown helpers shared across eval reporters. Each returns `string[]`
|
|
* so callers can push lines directly into their own buffer:
|
|
*
|
|
* const lines: string[] = [];
|
|
* lines.push(...renderHeader({ title: 'Foo', ... }));
|
|
* lines.push(...renderSummaryTable(['A', 'B'], rows));
|
|
* writeFileSync(path, lines.join('\n'));
|
|
*/
|
|
|
|
export interface ReportHeader {
|
|
title: string;
|
|
timestamp: string;
|
|
model: string;
|
|
judgeModel?: string;
|
|
extra?: Record<string, string | number>;
|
|
}
|
|
|
|
export function renderHeader(h: ReportHeader): string[] {
|
|
const lines = [`# ${h.title}`, ``, `- **Date**: ${h.timestamp}`, `- **Model**: ${h.model}`];
|
|
if (h.judgeModel) lines.push(`- **Judge model**: ${h.judgeModel}`);
|
|
for (const [k, v] of Object.entries(h.extra || {})) {
|
|
lines.push(`- **${k}**: ${v}`);
|
|
}
|
|
lines.push(``);
|
|
return lines;
|
|
}
|
|
|
|
export function renderSummaryTable(headers: string[], rows: string[][]): string[] {
|
|
const sep = `|${headers.map(() => '---').join('|')}|`;
|
|
const lines = [`| ${headers.join(' | ')} |`, sep];
|
|
for (const r of rows) lines.push(`| ${r.map((c) => c.replace(/\|/g, '\\|')).join(' | ')} |`);
|
|
lines.push(``);
|
|
return lines;
|
|
}
|