physical_byte_agent/tests/evolution-metrics.test.ts
d-robotics 74f4cd428e
Some checks are pending
CI / quality-gate (push) Waiting to run
CI / type-drift (push) Waiting to run
feat(跨层 X.1+X.3+X.4): 反馈采集全链路 + 可观测面板 + 量化坐实 (M1-M5+X 全闭环)
跨层共享项落地(设计文档 §19.4 X.1/X.3/X.4):

X.1 时序三档接线 + X.3 v_t 全链路贯通:
- shared/types.ts: TaskFeedback/ToolOutcome/EvolutionMetrics/MetricsSources
  + hitRate/avgInfoGain/avgSkillQuality/maxWorkflowRisk/evolutionBrickRate 纯函数
- core/feedback-collector.ts: FeedbackCollector(recordPrompt/recordBoardFingerprint/
  recordToolResult/finalize/lastFinalized/pending/clear) + derive* 纯变换
  (deriveTaskSignature/deriveDistillInput/deriveSkillExtractInput/deriveSkillQuery/
  derivePriorCorrection/deriveProposals + inferOutcome/inferActionClass/
  inferValidatedBy/isIrreversibleTool) + getSharedCollector/initSharedCollector 共享单例
- extensions/feedback-bank.ts: hooks tool_result+message_end+before_agent_start
  → 采集 → finalize → session_before_compact 喂 derive* + feedback_recall 工具
  (仿 orchestration-bank/skill-bank 范式)
- **4 bank 默认 derive* 读共享 collector lastFinalized 生产自驱**
  (experience/skill/orchestration/variant bank) — X.1+X.3 生产闭环

X.4 进化可观测面板:
- core/evolution-metrics.ts: aggregateMetrics/renderMetrics 纯函数
  (5 层注册表聚合 → EvolutionMetrics:规模/命中率/infoGain/risk/砖板率/forbidden 拒绝)
- bin/evolution-metrics.ts CLI(brickRate!=0 时 exit 1 fail-closed 自检)

X.1+X.3 全链路评估(tests/eval/feedback-chain-eval.test.ts, 合成 session 轨迹→
feedback-bank hooks→collector→derive*→真 5 层 manager→断 v_t 全链路 end-to-end):
  §4.6 分级正确率 8/8=100% (success+ground-truth/可逆/不可逆/partial/empty/mixed)
  同签名检索命中 100% + 注入块可见 100%
  5 链 end-to-end (skill/workflow/variant/tool-growth)
  forbidden §11.3/§4.8 拒绝 (命门 safety-gate/flash 工具)

X.4 评估(tests/eval/evolution-metrics-eval.test.ts, 合成 5 层注册表→精确指标):
  experienceHitRate=0.600/avgInfoGain=0.650/maxRisk=1 精确
  brickRate=0 EXACT 构造铁律 forbiddenRejected=7 精确

结论: X.1(4/5 时序钩子全接, stage-wise θ=L5.*) + X.3(v_t 全链路贯通: tool_result
采集→derive* 蒸馏→5 层入库→检索→注入→约束, §4.6 100% + 5 链 end-to-end + 4 bank
生产自驱) + X.4(可观测面板精确指标, brickRate=0 构造铁律) 全部完成。诚实边界
(M1-M5.* 跟进): live-LLM 读注入包真受益端到端门 + stage-wise 真 θ 更新 + 真 git
worktree 变体 + 真 CI §11.4 执行 = M1-M5.* 工程项(机制层全闭环+量化坐实)。

验证: tsc 0 / 全量 62 文件 3018 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot
过(barrel 28 扩展含 L1-L5 + 跨层反馈采集)
多 Agent 并行: K(feedback-collector+derive*)+L(feedback-bank+chain eval)+M(evolution-metrics)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-17 17:32:37 +08:00

405 lines
13 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* evolution-metrics.test.ts — X.4 进化可观测面板纯函数单测
*
* 设计文档 §13(证据)/ §16(评估)/ §19.4(X.4)。
* 覆盖 core/evolution-metrics.ts 的纯聚合函数 + 聚合器工厂:
* hitRate / avgInfoGain / avgSkillQuality / maxWorkflowRisk / evolutionBrickRate
* aggregateMetrics / renderMetrics / createEvolutionMetricsAggregator
*
* 纯逻辑,零 pi 运行时依赖。不修改 core/evolution-metrics.ts 或 shared/types.ts。
*/
import { describe, it, expect } from "vitest";
import type {
ExperienceEntry,
GrownTool,
HardwareSkill,
MetricsSources,
Variant,
WorkflowGraph,
} from "../src/shared/types.js";
import {
aggregateMetrics,
avgInfoGain,
avgSkillQuality,
createEvolutionMetricsAggregator,
evolutionBrickRate,
hitRate,
maxWorkflowRisk,
renderMetrics,
} from "../src/core/evolution-metrics.js";
// ============================================================================
// 测试夹具构造器(最小可用,只填聚合关心的字段)
// ============================================================================
const beta = { soc: "rdk-x5", kernel: "6.1.83", image: "v1", mac: "AA:01" };
const sig = { actionClass: "flash", target: "bootloader", constraints: ["aarch64"] };
function makeExperience(over: Partial<ExperienceEntry> = {}): ExperienceEntry {
const e: ExperienceEntry = {
id: "e-0",
boardFingerprint: beta,
taskSignature: sig,
outcome: "success",
actionable: "dd if=image of=/dev/mmcblk0",
evidenceRef: "seq#1",
contextTags: ["flash"],
validity: "repeatable",
usage: "actionable",
createdAt: 0,
hitCount: 0,
validatedBy: "physical-ground-truth",
...over,
};
return e;
}
function makeSkill(over: Partial<HardwareSkill> = {}): HardwareSkill {
return {
id: "s-0",
name: "i2c-recover",
trigger: { kind: "event", pattern: "i2c.*no such device", actionClass: "i2c-probe" },
procedure: [{ tool: "i2cdetect", argsTemplate: "0x3a" }],
granularity: "event-driven",
provenance: ["seq#1"],
quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 },
infoGain: 0.8,
createdAt: 0,
hitCount: 0,
active: true,
...over,
};
}
function makeWorkflow(over: Partial<WorkflowGraph> = {}): WorkflowGraph {
return {
id: "w-0",
taskClass: "flash",
nodes: [{ id: "n0", tool: "read", argsTemplate: "", irreversibility: "reversible" }],
edges: [],
risk: 0,
provenance: [],
createdAt: 0,
hitCount: 0,
active: true,
...over,
};
}
function makeGrownTool(over: Partial<GrownTool> = {}): GrownTool {
return {
id: "g-0",
name: "dump-reg",
toolClass: "read",
definitionHash: "abc123",
stage: "draft",
usageCount: 0,
passCount: 0,
validity: "repeatable",
createdAt: 0,
lastUsed: 0,
...over,
};
}
function makeVariant(over: Partial<Variant> = {}): Variant {
return {
id: "v-0",
name: "faster-probe",
targetExtension: "env-probe",
scaffoldClass: "probe",
baseRef: "HEAD",
factoryOverride: "override-1",
stage: "candidate",
invariants: {
tests: true, tsc: true, noSecrets: true, modelLaunch: true,
sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true,
},
createdAt: 0,
...over,
};
}
// ============================================================================
// hitRate
// ============================================================================
describe("hitRate —— §13 检索命中率(纯函数)", () => {
it("空集 → 0(无证据不算过)", () => {
expect(hitRate([])).toBe(0);
});
it("全 hitCount>0 → 1", () => {
const xs = [{ hitCount: 3 }, { hitCount: 1 }, { hitCount: 99 }];
expect(hitRate(xs)).toBe(1);
});
it("半数命中 → 0.5", () => {
const xs = [{ hitCount: 5 }, { hitCount: 0 }];
expect(hitRate(xs)).toBe(0.5);
});
it("全 hitCount=0 → 0", () => {
expect(hitRate([{ hitCount: 0 }, { hitCount: 0 }])).toBe(0);
});
});
// ============================================================================
// avgInfoGain
// ============================================================================
describe("avgInfoGain —— §4.4 平均可学信息增益(纯函数)", () => {
it("空集 → 0", () => {
expect(avgInfoGain([])).toBe(0);
});
it("多个值 → 均值", () => {
const skills = [{ infoGain: 0.8 }, { infoGain: 0.6 }, { infoGain: 0.9 }, { infoGain: 0.3 }];
// (0.8+0.6+0.9+0.3)/4 = 2.6/4 = 0.65
expect(avgInfoGain(skills)).toBeCloseTo(0.65, 10);
});
it("单值 → 该值", () => {
expect(avgInfoGain([{ infoGain: 0.42 }])).toBeCloseTo(0.42, 10);
});
});
// ============================================================================
// avgSkillQuality
// ============================================================================
describe("avgSkillQuality —— §8.4 双信号质量均值(exec 0.7 + rubric 0.3)", () => {
it("空集 → 0", () => {
expect(avgSkillQuality([])).toBe(0);
});
it("综合分 = 0.7·exec + 0.3·rubric,取均值", () => {
// skill0: exec=1.0, rubric=1.0 → 1.0
// skill1: exec=1.0, rubric=0.0 → 0.7
// skill2: exec=0.0, rubric=1.0 → 0.3
// skill3: exec=1.0, rubric=1.0 → 1.0
// 均值 = (1.0+0.7+0.3+1.0)/4 = 3.0/4 = 0.75
const skills = [
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
makeSkill({ quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 } }),
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } }),
makeSkill({ quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
];
expect(avgSkillQuality(skills)).toBeCloseTo(0.75, 10);
});
it("exec 为主:rubric 高但 exec=0 → 低分(fail-closed)", () => {
const skill = makeSkill({ quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } });
// 0.7*0 + 0.3*1 = 0.3
expect(avgSkillQuality([skill])).toBeCloseTo(0.3, 10);
});
});
// ============================================================================
// maxWorkflowRisk
// ============================================================================
describe("maxWorkflowRisk —— §4.8 不可逆暴露面最大值(纯函数)", () => {
it("空集 → 0", () => {
expect(maxWorkflowRisk([])).toBe(0);
});
it("多个 workflow → max(risk)", () => {
const wfs = [
makeWorkflow({ risk: 1 }),
makeWorkflow({ risk: 0 }),
makeWorkflow({ risk: 1 }),
makeWorkflow({ risk: 3 }),
];
expect(maxWorkflowRisk(wfs)).toBe(3);
});
it("全 risk=0 → 0", () => {
expect(maxWorkflowRisk([makeWorkflow({ risk: 0 }), makeWorkflow({ risk: 0 })])).toBe(0);
});
});
// ============================================================================
// evolutionBrickRate
// ============================================================================
describe("evolutionBrickRate —— 构造恒 0(§4.8/§11.4 不变式)", () => {
it("恒返回 0(L3+L5 forbidden 全拒+只长读/探+sandbox+§11.4 门)", () => {
expect(evolutionBrickRate()).toBe(0);
// 多次调用仍 0(非统计,构造不变式)
expect(evolutionBrickRate()).toBe(0);
});
});
// ============================================================================
// aggregateMetrics
// ============================================================================
describe("aggregateMetrics —— 5 层注册表快照 → EvolutionMetrics(纯函数)", () => {
it("全 15 字段精确(合成 MetricsSources)", () => {
const sources: MetricsSources = {
experiences: [
makeExperience({ hitCount: 3 }),
makeExperience({ hitCount: 0 }),
makeExperience({ hitCount: 2 }),
makeExperience({ hitCount: 1 }),
makeExperience({ hitCount: 0 }),
],
skills: [
makeSkill({ infoGain: 0.8, active: true, quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
makeSkill({ infoGain: 0.6, active: true, quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 } }),
makeSkill({ infoGain: 0.9, active: false, quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 } }),
makeSkill({ infoGain: 0.3, active: true, quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 } }),
],
workflows: [
makeWorkflow({ risk: 1, active: true }),
makeWorkflow({ risk: 0, active: false }),
makeWorkflow({ risk: 1, active: true }),
],
grownTools: [
makeGrownTool({ stage: "promoted" }),
makeGrownTool({ stage: "promoted" }),
makeGrownTool({ stage: "verified" }),
makeGrownTool({ stage: "draft" }),
makeGrownTool({ stage: "retired" }),
],
variants: [
makeVariant({ stage: "promoted" }),
makeVariant({ stage: "candidate" }),
makeVariant({ stage: "rolled-back" }),
makeVariant({ stage: "retired" }),
],
forbiddenRejected: 7,
};
const m = aggregateMetrics(sources, 42);
// 计数(来自各层数组长度)
expect(m.experienceCount).toBe(5);
expect(m.skillCount).toBe(4);
expect(m.activeSkillCount).toBe(3);
expect(m.workflowCount).toBe(3);
expect(m.activeWorkflowCount).toBe(2);
expect(m.grownToolCount).toBe(5);
expect(m.promotedToolCount).toBe(2);
expect(m.variantCount).toBe(4);
expect(m.promotedVariantCount).toBe(1);
// 派生指标
expect(m.experienceHitRate).toBeCloseTo(0.6, 10); // 3/5
expect(m.avgInfoGain).toBeCloseTo(0.65, 10); // (0.8+0.6+0.9+0.3)/4
expect(m.avgQualityScore).toBeCloseTo(0.75, 10); // (1.0+0.7+0.3+1.0)/4
expect(m.maxRisk).toBe(1); // max(1,0,1)
// 构造不变式
expect(m.brickRate).toBe(0);
// forbidden 拒绝数(注入,因拒绝不入库)
expect(m.forbiddenRejected).toBe(7);
// 时间戳透传
expect(m.ts).toBe(42);
});
it("forbiddenRejected 缺省 → 0(默认值)", () => {
const sources: MetricsSources = {
experiences: [],
skills: [],
workflows: [],
grownTools: [],
variants: [],
};
const m = aggregateMetrics(sources);
expect(m.forbiddenRejected).toBe(0);
expect(m.experienceCount).toBe(0);
expect(m.experienceHitRate).toBe(0);
expect(m.brickRate).toBe(0);
expect(m.ts).toBe(0);
});
it("空 5 层 → 全零指标(零状态安全)", () => {
const m = aggregateMetrics({
experiences: [], skills: [], workflows: [], grownTools: [], variants: [],
});
expect(m).toMatchObject({
experienceCount: 0, experienceHitRate: 0,
skillCount: 0, activeSkillCount: 0,
workflowCount: 0, activeWorkflowCount: 0,
grownToolCount: 0, promotedToolCount: 0,
variantCount: 0, promotedVariantCount: 0,
maxRisk: 0, brickRate: 0, forbiddenRejected: 0,
avgInfoGain: 0, avgQualityScore: 0, ts: 0,
});
});
});
// ============================================================================
// renderMetrics
// ============================================================================
describe("renderMetrics —— EvolutionMetrics → 可读多行串(纯函数)", () => {
it("产多行串,含关键指标行", () => {
const m = aggregateMetrics({
experiences: [makeExperience({ hitCount: 1 }), makeExperience({ hitCount: 0 })],
skills: [makeSkill({ infoGain: 0.5, active: true })],
workflows: [makeWorkflow({ risk: 2, active: true })],
grownTools: [makeGrownTool({ stage: "promoted" })],
variants: [makeVariant({ stage: "promoted" })],
forbiddenRejected: 3,
}, 0);
const out = renderMetrics(m);
expect(typeof out).toBe("string");
expect(out.split("\n").length).toBeGreaterThan(1);
// 关键指标行存在
expect(out).toContain("=== Evolution Metrics ===");
expect(out).toContain("experienceCount=2");
expect(out).toContain("hitRate=0.500");
expect(out).toContain("skillCount=1");
expect(out).toContain("active=1");
expect(out).toContain("maxRisk=2");
expect(out).toContain("grownToolCount=1");
expect(out).toContain("promoted=1");
expect(out).toContain("variantCount=1");
expect(out).toContain("brickRate=0");
expect(out).toContain("forbiddenRejected=3");
});
});
// ============================================================================
// createEvolutionMetricsAggregator
// ============================================================================
describe("createEvolutionMetricsAggregator —— 工厂 + .aggregate()", () => {
it("返回聚合器,aggregate 委托纯函数 aggregateMetrics", () => {
const agg = createEvolutionMetricsAggregator();
expect(typeof agg.aggregate).toBe("function");
const sources: MetricsSources = {
experiences: [makeExperience({ hitCount: 2 }), makeExperience({ hitCount: 0 })],
skills: [makeSkill({ infoGain: 0.4, active: true })],
workflows: [makeWorkflow({ risk: 1, active: true })],
grownTools: [makeGrownTool({ stage: "promoted" })],
variants: [makeVariant({ stage: "candidate" })],
forbiddenRejected: 5,
};
const viaAgg = agg.aggregate(sources, 9);
const viaFn = aggregateMetrics(sources, 9);
// 聚合器委托纯函数 → 完全一致
expect(viaAgg).toEqual(viaFn);
expect(viaAgg.experienceHitRate).toBeCloseTo(0.5, 10);
expect(viaAgg.maxRisk).toBe(1);
expect(viaAgg.brickRate).toBe(0);
expect(viaAgg.ts).toBe(9);
});
it("aggregate 默认 ts=0", () => {
const agg = createEvolutionMetricsAggregator();
const m = agg.aggregate({
experiences: [], skills: [], workflows: [], grownTools: [], variants: [],
});
expect(m.ts).toBe(0);
});
});