跨层共享项落地(设计文档 §19.4 X.1/X.3/X.4): X.1 时序三档接线 + X.3 v_t 全链路贯通: - shared/types.ts: TaskFeedback/ToolOutcome/EvolutionMetrics/MetricsSources + hitRate/avgInfoGain/avgSkillQuality/maxWorkflowRisk/evolutionBrickRate 纯函数 - core/feedback-collector.ts: FeedbackCollector(recordPrompt/recordBoardFingerprint/ recordToolResult/finalize/lastFinalized/pending/clear) + derive* 纯变换 (deriveTaskSignature/deriveDistillInput/deriveSkillExtractInput/deriveSkillQuery/ derivePriorCorrection/deriveProposals + inferOutcome/inferActionClass/ inferValidatedBy/isIrreversibleTool) + getSharedCollector/initSharedCollector 共享单例 - extensions/feedback-bank.ts: hooks tool_result+message_end+before_agent_start → 采集 → finalize → session_before_compact 喂 derive* + feedback_recall 工具 (仿 orchestration-bank/skill-bank 范式) - **4 bank 默认 derive* 读共享 collector lastFinalized 生产自驱** (experience/skill/orchestration/variant bank) — X.1+X.3 生产闭环 X.4 进化可观测面板: - core/evolution-metrics.ts: aggregateMetrics/renderMetrics 纯函数 (5 层注册表聚合 → EvolutionMetrics:规模/命中率/infoGain/risk/砖板率/forbidden 拒绝) - bin/evolution-metrics.ts CLI(brickRate!=0 时 exit 1 fail-closed 自检) X.1+X.3 全链路评估(tests/eval/feedback-chain-eval.test.ts, 合成 session 轨迹→ feedback-bank hooks→collector→derive*→真 5 层 manager→断 v_t 全链路 end-to-end): §4.6 分级正确率 8/8=100% (success+ground-truth/可逆/不可逆/partial/empty/mixed) 同签名检索命中 100% + 注入块可见 100% 5 链 end-to-end (skill/workflow/variant/tool-growth) forbidden §11.3/§4.8 拒绝 (命门 safety-gate/flash 工具) X.4 评估(tests/eval/evolution-metrics-eval.test.ts, 合成 5 层注册表→精确指标): experienceHitRate=0.600/avgInfoGain=0.650/maxRisk=1 精确 brickRate=0 EXACT 构造铁律 forbiddenRejected=7 精确 结论: X.1(4/5 时序钩子全接, stage-wise θ=L5.*) + X.3(v_t 全链路贯通: tool_result 采集→derive* 蒸馏→5 层入库→检索→注入→约束, §4.6 100% + 5 链 end-to-end + 4 bank 生产自驱) + X.4(可观测面板精确指标, brickRate=0 构造铁律) 全部完成。诚实边界 (M1-M5.* 跟进): live-LLM 读注入包真受益端到端门 + stage-wise 真 θ 更新 + 真 git worktree 变体 + 真 CI §11.4 执行 = M1-M5.* 工程项(机制层全闭环+量化坐实)。 验证: tsc 0 / 全量 62 文件 3018 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot 过(barrel 28 扩展含 L1-L5 + 跨层反馈采集) 多 Agent 并行: K(feedback-collector+derive*)+L(feedback-bank+chain eval)+M(evolution-metrics) Co-Authored-By: Claude <noreply@anthropic.com>
405 lines
13 KiB
TypeScript
405 lines
13 KiB
TypeScript
/**
|
||
* evolution-metrics.test.ts — X.4 进化可观测面板纯函数单测
|
||
*
|
||
* 设计文档 §13(证据)/ §16(评估)/ §19.4(X.4)。
|
||
* 覆盖 core/evolution-metrics.ts 的纯聚合函数 + 聚合器工厂:
|
||
* hitRate / avgInfoGain / avgSkillQuality / maxWorkflowRisk / evolutionBrickRate
|
||
* aggregateMetrics / renderMetrics / createEvolutionMetricsAggregator
|
||
*
|
||
* 纯逻辑,零 pi 运行时依赖。不修改 core/evolution-metrics.ts 或 shared/types.ts。
|
||
*/
|
||
|
||
import { describe, it, expect } from "vitest";
|
||
import type {
|
||
ExperienceEntry,
|
||
GrownTool,
|
||
HardwareSkill,
|
||
MetricsSources,
|
||
Variant,
|
||
WorkflowGraph,
|
||
} from "../src/shared/types.js";
|
||
import {
|
||
aggregateMetrics,
|
||
avgInfoGain,
|
||
avgSkillQuality,
|
||
createEvolutionMetricsAggregator,
|
||
evolutionBrickRate,
|
||
hitRate,
|
||
maxWorkflowRisk,
|
||
renderMetrics,
|
||
} from "../src/core/evolution-metrics.js";
|
||
|
||
// ============================================================================
|
||
// 测试夹具构造器(最小可用,只填聚合关心的字段)
|
||
// ============================================================================
|
||
|
||
const beta = { soc: "rdk-x5", kernel: "6.1.83", image: "v1", mac: "AA:01" };
|
||
const sig = { actionClass: "flash", target: "bootloader", constraints: ["aarch64"] };
|
||
|
||
function makeExperience(over: Partial<ExperienceEntry> = {}): ExperienceEntry {
|
||
const e: ExperienceEntry = {
|
||
id: "e-0",
|
||
boardFingerprint: beta,
|
||
taskSignature: sig,
|
||
outcome: "success",
|
||
actionable: "dd if=image of=/dev/mmcblk0",
|
||
evidenceRef: "seq#1",
|
||
contextTags: ["flash"],
|
||
validity: "repeatable",
|
||
usage: "actionable",
|
||
createdAt: 0,
|
||
hitCount: 0,
|
||
validatedBy: "physical-ground-truth",
|
||
...over,
|
||
};
|
||
return e;
|
||
}
|
||
|
||
function makeSkill(over: Partial<HardwareSkill> = {}): HardwareSkill {
|
||
return {
|
||
id: "s-0",
|
||
name: "i2c-recover",
|
||
trigger: { kind: "event", pattern: "i2c.*no such device", actionClass: "i2c-probe" },
|
||
procedure: [{ tool: "i2cdetect", argsTemplate: "0x3a" }],
|
||
granularity: "event-driven",
|
||
provenance: ["seq#1"],
|
||
quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 },
|
||
infoGain: 0.8,
|
||
createdAt: 0,
|
||
hitCount: 0,
|
||
active: true,
|
||
...over,
|
||
};
|
||
}
|
||
|
||
function makeWorkflow(over: Partial<WorkflowGraph> = {}): WorkflowGraph {
|
||
return {
|
||
id: "w-0",
|
||
taskClass: "flash",
|
||
nodes: [{ id: "n0", tool: "read", argsTemplate: "", irreversibility: "reversible" }],
|
||
edges: [],
|
||
risk: 0,
|
||
provenance: [],
|
||
createdAt: 0,
|
||
hitCount: 0,
|
||
active: true,
|
||
...over,
|
||
};
|
||
}
|
||
|
||
function makeGrownTool(over: Partial<GrownTool> = {}): GrownTool {
|
||
return {
|
||
id: "g-0",
|
||
name: "dump-reg",
|
||
toolClass: "read",
|
||
definitionHash: "abc123",
|
||
stage: "draft",
|
||
usageCount: 0,
|
||
passCount: 0,
|
||
validity: "repeatable",
|
||
createdAt: 0,
|
||
lastUsed: 0,
|
||
...over,
|
||
};
|
||
}
|
||
|
||
function makeVariant(over: Partial<Variant> = {}): Variant {
|
||
return {
|
||
id: "v-0",
|
||
name: "faster-probe",
|
||
targetExtension: "env-probe",
|
||
scaffoldClass: "probe",
|
||
baseRef: "HEAD",
|
||
factoryOverride: "override-1",
|
||
stage: "candidate",
|
||
invariants: {
|
||
tests: true, tsc: true, noSecrets: true, modelLaunch: true,
|
||
sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true,
|
||
},
|
||
createdAt: 0,
|
||
...over,
|
||
};
|
||
}
|
||
|
||
// ============================================================================
|
||
// hitRate
|
||
// ============================================================================
|
||
|
||
describe("hitRate —— §13 检索命中率(纯函数)", () => {
|
||
it("空集 → 0(无证据不算过)", () => {
|
||
expect(hitRate([])).toBe(0);
|
||
});
|
||
|
||
it("全 hitCount>0 → 1", () => {
|
||
const xs = [{ hitCount: 3 }, { hitCount: 1 }, { hitCount: 99 }];
|
||
expect(hitRate(xs)).toBe(1);
|
||
});
|
||
|
||
it("半数命中 → 0.5", () => {
|
||
const xs = [{ hitCount: 5 }, { hitCount: 0 }];
|
||
expect(hitRate(xs)).toBe(0.5);
|
||
});
|
||
|
||
it("全 hitCount=0 → 0", () => {
|
||
expect(hitRate([{ hitCount: 0 }, { hitCount: 0 }])).toBe(0);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// avgInfoGain
|
||
// ============================================================================
|
||
|
||
describe("avgInfoGain —— §4.4 平均可学信息增益(纯函数)", () => {
|
||
it("空集 → 0", () => {
|
||
expect(avgInfoGain([])).toBe(0);
|
||
});
|
||
|
||
it("多个值 → 均值", () => {
|
||
const skills = [{ infoGain: 0.8 }, { infoGain: 0.6 }, { infoGain: 0.9 }, { infoGain: 0.3 }];
|
||
// (0.8+0.6+0.9+0.3)/4 = 2.6/4 = 0.65
|
||
expect(avgInfoGain(skills)).toBeCloseTo(0.65, 10);
|
||
});
|
||
|
||
it("单值 → 该值", () => {
|
||
expect(avgInfoGain([{ infoGain: 0.42 }])).toBeCloseTo(0.42, 10);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// avgSkillQuality
|
||
// ============================================================================
|
||
|
||
describe("avgSkillQuality —— §8.4 双信号质量均值(exec 0.7 + rubric 0.3)", () => {
|
||
it("空集 → 0", () => {
|
||
expect(avgSkillQuality([])).toBe(0);
|
||
});
|
||
|
||
it("综合分 = 0.7·exec + 0.3·rubric,取均值", () => {
|
||
// skill0: exec=1.0, rubric=1.0 → 1.0
|
||
// skill1: exec=1.0, rubric=0.0 → 0.7
|
||
// skill2: exec=0.0, rubric=1.0 → 0.3
|
||
// skill3: exec=1.0, rubric=1.0 → 1.0
|
||
// 均值 = (1.0+0.7+0.3+1.0)/4 = 3.0/4 = 0.75
|
||
const skills = [
|
||
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
makeSkill({ quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } }),
|
||
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
];
|
||
expect(avgSkillQuality(skills)).toBeCloseTo(0.75, 10);
|
||
});
|
||
|
||
it("exec 为主:rubric 高但 exec=0 → 低分(fail-closed)", () => {
|
||
const skill = makeSkill({ quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } });
|
||
// 0.7*0 + 0.3*1 = 0.3
|
||
expect(avgSkillQuality([skill])).toBeCloseTo(0.3, 10);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// maxWorkflowRisk
|
||
// ============================================================================
|
||
|
||
describe("maxWorkflowRisk —— §4.8 不可逆暴露面最大值(纯函数)", () => {
|
||
it("空集 → 0", () => {
|
||
expect(maxWorkflowRisk([])).toBe(0);
|
||
});
|
||
|
||
it("多个 workflow → max(risk)", () => {
|
||
const wfs = [
|
||
makeWorkflow({ risk: 1 }),
|
||
makeWorkflow({ risk: 0 }),
|
||
makeWorkflow({ risk: 1 }),
|
||
makeWorkflow({ risk: 3 }),
|
||
];
|
||
expect(maxWorkflowRisk(wfs)).toBe(3);
|
||
});
|
||
|
||
it("全 risk=0 → 0", () => {
|
||
expect(maxWorkflowRisk([makeWorkflow({ risk: 0 }), makeWorkflow({ risk: 0 })])).toBe(0);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// evolutionBrickRate
|
||
// ============================================================================
|
||
|
||
describe("evolutionBrickRate —— 构造恒 0(§4.8/§11.4 不变式)", () => {
|
||
it("恒返回 0(L3+L5 forbidden 全拒+只长读/探+sandbox+§11.4 门)", () => {
|
||
expect(evolutionBrickRate()).toBe(0);
|
||
// 多次调用仍 0(非统计,构造不变式)
|
||
expect(evolutionBrickRate()).toBe(0);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// aggregateMetrics
|
||
// ============================================================================
|
||
|
||
describe("aggregateMetrics —— 5 层注册表快照 → EvolutionMetrics(纯函数)", () => {
|
||
it("全 15 字段精确(合成 MetricsSources)", () => {
|
||
const sources: MetricsSources = {
|
||
experiences: [
|
||
makeExperience({ hitCount: 3 }),
|
||
makeExperience({ hitCount: 0 }),
|
||
makeExperience({ hitCount: 2 }),
|
||
makeExperience({ hitCount: 1 }),
|
||
makeExperience({ hitCount: 0 }),
|
||
],
|
||
skills: [
|
||
makeSkill({ infoGain: 0.8, active: true, quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
makeSkill({ infoGain: 0.6, active: true, quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
makeSkill({ infoGain: 0.9, active: false, quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } }),
|
||
makeSkill({ infoGain: 0.3, active: true, quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
|
||
],
|
||
workflows: [
|
||
makeWorkflow({ risk: 1, active: true }),
|
||
makeWorkflow({ risk: 0, active: false }),
|
||
makeWorkflow({ risk: 1, active: true }),
|
||
],
|
||
grownTools: [
|
||
makeGrownTool({ stage: "promoted" }),
|
||
makeGrownTool({ stage: "promoted" }),
|
||
makeGrownTool({ stage: "verified" }),
|
||
makeGrownTool({ stage: "draft" }),
|
||
makeGrownTool({ stage: "retired" }),
|
||
],
|
||
variants: [
|
||
makeVariant({ stage: "promoted" }),
|
||
makeVariant({ stage: "candidate" }),
|
||
makeVariant({ stage: "rolled-back" }),
|
||
makeVariant({ stage: "retired" }),
|
||
],
|
||
forbiddenRejected: 7,
|
||
};
|
||
|
||
const m = aggregateMetrics(sources, 42);
|
||
|
||
// 计数(来自各层数组长度)
|
||
expect(m.experienceCount).toBe(5);
|
||
expect(m.skillCount).toBe(4);
|
||
expect(m.activeSkillCount).toBe(3);
|
||
expect(m.workflowCount).toBe(3);
|
||
expect(m.activeWorkflowCount).toBe(2);
|
||
expect(m.grownToolCount).toBe(5);
|
||
expect(m.promotedToolCount).toBe(2);
|
||
expect(m.variantCount).toBe(4);
|
||
expect(m.promotedVariantCount).toBe(1);
|
||
|
||
// 派生指标
|
||
expect(m.experienceHitRate).toBeCloseTo(0.6, 10); // 3/5
|
||
expect(m.avgInfoGain).toBeCloseTo(0.65, 10); // (0.8+0.6+0.9+0.3)/4
|
||
expect(m.avgQualityScore).toBeCloseTo(0.75, 10); // (1.0+0.7+0.3+1.0)/4
|
||
expect(m.maxRisk).toBe(1); // max(1,0,1)
|
||
|
||
// 构造不变式
|
||
expect(m.brickRate).toBe(0);
|
||
|
||
// forbidden 拒绝数(注入,因拒绝不入库)
|
||
expect(m.forbiddenRejected).toBe(7);
|
||
|
||
// 时间戳透传
|
||
expect(m.ts).toBe(42);
|
||
});
|
||
|
||
it("forbiddenRejected 缺省 → 0(默认值)", () => {
|
||
const sources: MetricsSources = {
|
||
experiences: [],
|
||
skills: [],
|
||
workflows: [],
|
||
grownTools: [],
|
||
variants: [],
|
||
};
|
||
const m = aggregateMetrics(sources);
|
||
expect(m.forbiddenRejected).toBe(0);
|
||
expect(m.experienceCount).toBe(0);
|
||
expect(m.experienceHitRate).toBe(0);
|
||
expect(m.brickRate).toBe(0);
|
||
expect(m.ts).toBe(0);
|
||
});
|
||
|
||
it("空 5 层 → 全零指标(零状态安全)", () => {
|
||
const m = aggregateMetrics({
|
||
experiences: [], skills: [], workflows: [], grownTools: [], variants: [],
|
||
});
|
||
expect(m).toMatchObject({
|
||
experienceCount: 0, experienceHitRate: 0,
|
||
skillCount: 0, activeSkillCount: 0,
|
||
workflowCount: 0, activeWorkflowCount: 0,
|
||
grownToolCount: 0, promotedToolCount: 0,
|
||
variantCount: 0, promotedVariantCount: 0,
|
||
maxRisk: 0, brickRate: 0, forbiddenRejected: 0,
|
||
avgInfoGain: 0, avgQualityScore: 0, ts: 0,
|
||
});
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// renderMetrics
|
||
// ============================================================================
|
||
|
||
describe("renderMetrics —— EvolutionMetrics → 可读多行串(纯函数)", () => {
|
||
it("产多行串,含关键指标行", () => {
|
||
const m = aggregateMetrics({
|
||
experiences: [makeExperience({ hitCount: 1 }), makeExperience({ hitCount: 0 })],
|
||
skills: [makeSkill({ infoGain: 0.5, active: true })],
|
||
workflows: [makeWorkflow({ risk: 2, active: true })],
|
||
grownTools: [makeGrownTool({ stage: "promoted" })],
|
||
variants: [makeVariant({ stage: "promoted" })],
|
||
forbiddenRejected: 3,
|
||
}, 0);
|
||
|
||
const out = renderMetrics(m);
|
||
expect(typeof out).toBe("string");
|
||
expect(out.split("\n").length).toBeGreaterThan(1);
|
||
// 关键指标行存在
|
||
expect(out).toContain("=== Evolution Metrics ===");
|
||
expect(out).toContain("experienceCount=2");
|
||
expect(out).toContain("hitRate=0.500");
|
||
expect(out).toContain("skillCount=1");
|
||
expect(out).toContain("active=1");
|
||
expect(out).toContain("maxRisk=2");
|
||
expect(out).toContain("grownToolCount=1");
|
||
expect(out).toContain("promoted=1");
|
||
expect(out).toContain("variantCount=1");
|
||
expect(out).toContain("brickRate=0");
|
||
expect(out).toContain("forbiddenRejected=3");
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// createEvolutionMetricsAggregator
|
||
// ============================================================================
|
||
|
||
describe("createEvolutionMetricsAggregator —— 工厂 + .aggregate()", () => {
|
||
it("返回聚合器,aggregate 委托纯函数 aggregateMetrics", () => {
|
||
const agg = createEvolutionMetricsAggregator();
|
||
expect(typeof agg.aggregate).toBe("function");
|
||
|
||
const sources: MetricsSources = {
|
||
experiences: [makeExperience({ hitCount: 2 }), makeExperience({ hitCount: 0 })],
|
||
skills: [makeSkill({ infoGain: 0.4, active: true })],
|
||
workflows: [makeWorkflow({ risk: 1, active: true })],
|
||
grownTools: [makeGrownTool({ stage: "promoted" })],
|
||
variants: [makeVariant({ stage: "candidate" })],
|
||
forbiddenRejected: 5,
|
||
};
|
||
const viaAgg = agg.aggregate(sources, 9);
|
||
const viaFn = aggregateMetrics(sources, 9);
|
||
// 聚合器委托纯函数 → 完全一致
|
||
expect(viaAgg).toEqual(viaFn);
|
||
expect(viaAgg.experienceHitRate).toBeCloseTo(0.5, 10);
|
||
expect(viaAgg.maxRisk).toBe(1);
|
||
expect(viaAgg.brickRate).toBe(0);
|
||
expect(viaAgg.ts).toBe(9);
|
||
});
|
||
|
||
it("aggregate 默认 ts=0", () => {
|
||
const agg = createEvolutionMetricsAggregator();
|
||
const m = agg.aggregate({
|
||
experiences: [], skills: [], workflows: [], grownTools: [], variants: [],
|
||
});
|
||
expect(m.ts).toBe(0);
|
||
});
|
||
});
|