openmaic-custom/tests/eval/shared/resolve-model.test.ts
wyuc 7d47b0ad3a
refactor(eval): unify outline-language and whiteboard-layout harness (#453)
* feat(eval): add resolveEvalModel helper with fail-fast

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* feat(eval): add createRunDir helper with path sanitization

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* feat(eval): add markdown report helpers

* refactor(eval): move language test cases under eval/outline-language

* feat(eval): add outline-language types

* feat(eval): add outline-language LLM judge

* feat(eval): add outline-language reporter

* feat(eval): add outline-language runner entry

* chore(eval): add eval:outline-language pnpm script

* refactor(eval): adopt shared createRunDir and drop gpt-4o fallbacks in whiteboard runner

* refactor(eval): drop redundant non-null assertions after narrowing

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* refactor(eval): remove SCORER_MODEL_DEFAULT hardcoded gpt-4o fallback

* chore(eval): delete tests/generation and clean up vitest/gitignore config

* docs(eval): explain why outline-language runner pre-validates env vars

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(eval): escape pipe chars in markdown summary table cells

LLM judge output may contain | which breaks GFM table rendering.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: 杨慎 <117187635+cosarah@users.noreply.github.com>
2026-04-19 21:29:18 +08:00

52 lines
2.2 KiB
TypeScript

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
describe('resolveEvalModel', () => {
const ORIG_ENV = process.env;
beforeEach(() => {
process.env = { ...ORIG_ENV };
vi.resetModules();
});
afterEach(() => {
process.env = ORIG_ENV;
vi.restoreAllMocks();
});
it('throws a helpful error when env var is unset and no fallback is given', async () => {
delete process.env.EVAL_FOO_MODEL;
vi.doMock('@/lib/server/resolve-model', () => ({
resolveModel: vi.fn(),
}));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await expect(resolveEvalModel('EVAL_FOO_MODEL')).rejects.toThrow(/EVAL_FOO_MODEL/);
});
it('uses the env var when set', async () => {
process.env.EVAL_FOO_MODEL = 'openai:gpt-4.1';
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'openai:gpt-4.1' });
});
it('uses the explicit fallback when env var is unset', async () => {
delete process.env.EVAL_FOO_MODEL;
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'google:gemini-2.5-flash' });
});
it('env var takes precedence over fallback', async () => {
process.env.EVAL_FOO_MODEL = 'anthropic:claude-haiku-4-5';
const resolveModel = vi.fn().mockResolvedValue({ model: 'resolved', modelInfo: {} });
vi.doMock('@/lib/server/resolve-model', () => ({ resolveModel }));
const { resolveEvalModel } = await import('@/eval/shared/resolve-model');
await resolveEvalModel('EVAL_FOO_MODEL', 'google:gemini-2.5-flash');
expect(resolveModel).toHaveBeenCalledWith({ modelString: 'anthropic:claude-haiku-4-5' });
});
});