physical_byte_agent/tests/skill-manager.test.ts
d-robotics 0cd33f9e21
Some checks are pending
CI / quality-gate (push) Waiting to run
CI / type-drift (push) Waiting to run
feat(L2+L3): M2 技能注册表 + M3 工具生长自进化闭环 + 量化坐实
M2 L2 可学技能注册表层落地(设计文档 §8/§19.3 M2.1-M2.6):
- M2.1 shared/types.ts: HardwareSkill/SkillTrigger/SkillStep/Granularity/SkillQuality/
  SkillExtractInput/InfoGainGuard + assessQuality/skillSimilarity/computeInfoGain/
  skillId 纯函数
- M2.1-2.5 core/skill-manager.ts: SkillRegistry(register/resolve/list 仿 FlashRegistry)
  + SkillManager μ(extract/register/resolve/supersede/merge/evict) + infoGain 守卫
  (§4.4: 1-最近邻相似度; >0.85 → merge 不新增防 plateau) + 双信号质量(exec 0.7
  +rubric 0.3, §8.4) + supersededBy 链 + extractSkill/infoGainGate/mergeSkills/scoreSkill
  纯函数; merge 路径累积 execSamples+provenance(§8.4 exec 信号随复用累积)
- M2.6 extensions/skill-bank.ts: before_agent_start 检索+注入 <skill_pack> +
  session_before_compact μ extract/register + skill_recall 工具(仿 experience-bank)
- barrel 第 24 工厂登记

M3 L3 工具生长层落地(设计文档 §9/§19.3 M3.1-M3.5):
- M3.1 shared/types.ts: GrownTool/ToolLifecycleStage/ToolGrowthClass/ToolUpgradeGate/
  ToolProposal + classifyTool/canGrow/definitionHash/grownToolId
- M3.1-3.5 core/tool-growth.ts: ToolGrowthManager(propose 白名单 forbidden 拒绝 throw
  §4.8 risk 不增 / recordUsage / tryPromote 升级门 K+repeatable / verify / retire
  生命周期 draft→verified→promoted→retired / brickRate 恒0 构造铁律) +
  classifyProposal/promotionGate/canTransition 纯函数
- M3.1/3.4 extensions/tool-bank.ts: tool_propose/tool_record/tool_promote/
  tool_list_grown/tool_retire 工具 + before_agent_start 注入 <grown_tools>
- barrel 第 25 工厂登记

M2 量化评估(tests/eval/skill-registry-eval.test.ts, seeded mulberry32 N=400 5 队列
+ Wilson CI + 真 manager 对齐, 独立复跑一致):
  ΔJ_full=0.265>=0.15  用技能 61%->87.5%(+26.5pp)
  ΔJ_held_out=0.2545>=0.05  held-out 板 +25.5pp(排除 F4)
  因果技能增益=0.3482>=0.20
  distill-off |J_D-J_B|=0<0.05  证改进来自技能增长
  F1 负控: 守卫 OFF 技能数 400>12(守卫 ON 6) 但 J_E=0.685<J_A=0.875
    → infoGain 守卫防 plateau 实证(§4.4)
  supersede active 有界(6<=8) 不堆积死技能
  CI 非重叠 STRONG (J_A_lo=0.839>J_B_hi=0.657)
  真 SkillManager 与对照 5 指标全等(J_A/B/C/D+active/store)

M3 量化评估(tests/eval/tool-growth-eval.test.ts, seeded N=300, 真 manager):
  步效率 0.24<0.70  固化工具砍 76% 步(5.0->1.2 步/任务)
  ΔJ=0.1367>=0.05  固化工具更可靠
  砖板率=0 EXACT 构造铁律(forbidden 全拒+只长读/探+sandbox+升级门)
  forbidden 7 拒/0 受(§4.8 risk 不增)
  升级门低 usage(<K) promote=0
  CI 非重叠 STRONG (J_A_lo=0.769>J_B_hi=0.730)

结论: M2+M3 自进化机制有效且安全(量化/可信/可复现)。诚实边界(M1-M3.* 跟进):
评估为忠实 mock 策略受控实验证机制因果效应; live-LLM 读 experiencePack/
skillPack/grown_tools 真受益的端到端门 + derive* 真接线 + sqlite 持久化 +
vm 沙箱 execute 为 M1-M3.* 工程项。

验证: tsc 0 / 全量 49 文件 2697 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot 过
多 Agent 并行: D(skill-manager)+E(skill-bank+eval)+F(tool-growth full)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-17 15:54:11 +08:00

367 lines
12 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { describe, it, expect } from "vitest";
import type {
HardwareSkill,
SkillExtractInput,
SkillStep,
SkillTrigger,
TaskSignature,
} from "../src/shared/types.js";
import { skillId } from "../src/shared/types.js";
import {
createSkillManager,
createSkillRegistry,
extractSkill,
infoGainGate,
mergeSkills,
scoreSkill,
DEFAULT_MIN_INFO_GAIN,
DEFAULT_MAX_SIMILARITY,
DEFAULT_MIN_QUALITY,
} from "../src/core/skill-manager.js";
import type { SkillManager } from "../src/core/skill-manager.js";
/** SkillManager 接口无 get——用 list().find 检视存储技能。 */
function find(mgr: SkillManager, id: string): HardwareSkill | null {
return mgr.list().find((s) => s.id === id) ?? null;
}
// ---- 测试夹具 ----
const trigI2c: SkillTrigger = {
kind: "event",
pattern: "i2c no such device",
actionClass: "i2c-probe",
};
const trigFlash: SkillTrigger = {
kind: "event",
pattern: "flash write timeout",
actionClass: "flash",
};
const sigI2c: TaskSignature = {
actionClass: "i2c-probe",
target: "pmic 0x3a",
constraints: ["aarch64"],
};
const procI2c: SkillStep[] = [
{ tool: "i2cdetect", argsTemplate: "-y 0", expect: "0x3a" },
];
const procFlash: SkillStep[] = [
{ tool: "dd", argsTemplate: "if=img of=/dev/mmcblk0" },
];
/** 构造 SkillExtractInput(μ 抽取入参)。 */
function makeInput(over: Partial<SkillExtractInput> = {}): SkillExtractInput {
const trigger = over.trigger ?? trigI2c;
const procedure = over.procedure ?? procI2c;
const granularity = over.granularity ?? "event-driven";
return {
trigger,
procedure,
granularity,
provenance: over.provenance ?? ["trace#1"],
rubricScore: over.rubricScore ?? 0.5,
execResults: over.execResults ?? [{ passed: true }, { passed: true }, { passed: true }, { passed: false }],
createdAt: over.createdAt ?? 1000,
};
}
/** 直接构造一条 HardwareSkill(绕过 extract,便于精控字段)。 */
function makeSkill(over: Partial<HardwareSkill> = {}): HardwareSkill {
const trigger = over.trigger ?? trigI2c;
const procedure = over.procedure ?? procI2c;
const granularity = over.granularity ?? "event-driven";
const skill: HardwareSkill = {
id: over.id ?? skillId({ trigger, procedure, granularity }),
name: over.name ?? `${trigger.actionClass}:${trigger.kind}`,
trigger,
procedure,
granularity,
provenance: over.provenance ?? ["trace#1"],
quality: over.quality ?? { rubricScore: 0.5, execPassRate: 0.75, execSamples: 4 },
infoGain: over.infoGain ?? 1,
createdAt: over.createdAt ?? 1000,
hitCount: over.hitCount ?? 0,
active: over.active ?? true,
supersededBy: over.supersededBy,
};
return skill;
}
describe("SkillManager.extract —— μ 抽取(纯构造,不注册)", () => {
it("空注册表 → infoGain=1.0(无最近邻)", () => {
const mgr = createSkillManager();
const cand = mgr.extract(makeInput());
expect(cand.infoGain).toBe(1);
expect(cand.active).toBe(true);
expect(cand.hitCount).toBe(0);
expect(Object.isFrozen(cand)).toBe(true);
// extract 不注册
expect(mgr.size()).toBe(0);
});
it("对相似已有技能 → infoGain<1", () => {
const mgr = createSkillManager();
// 先注册一条 i2c 技能
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
// 再 extract 同 trigger+procedure → 与已有完全相似
const cand = mgr.extract(makeInput({ trigger: trigI2c, procedure: procI2c }));
expect(cand.infoGain).toBeLessThan(1);
});
});
describe("SkillManager.register —— §4.4 infoGain 守卫(防 plateau)", () => {
it("genuinely 新增技能(不同 actionClass)→ added:true", () => {
const mgr = createSkillManager();
// 已有 i2c 技能
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
// 注册 flash 技能(不同 actionClass → 低相似 → 高 infoGain)
const flash = makeSkill({ trigger: trigFlash, procedure: procFlash });
const r = mgr.register(flash);
expect(r.added).toBe(true);
expect(mgr.size()).toBe(2);
});
it("近重复(同 trigger+procedure)→ added:false, mergedWith=nearest, nearest.hitCount++", () => {
const mgr = createSkillManager();
const first = makeSkill({ trigger: trigI2c, procedure: procI2c });
mgr.register(first);
const beforeHit = find(mgr, first.id)?.hitCount;
expect(beforeHit).toBe(0);
// 近重复:同 trigger+procedure → 相似度 1 → infoGain=0 → 守卫拒
const dup = makeSkill({ trigger: trigI2c, procedure: procI2c });
const r = mgr.register(dup);
expect(r.added).toBe(false);
expect(r.mergedWith).toBe(first.id);
// nearest hitCount 升权(plateau 预防:强化已有而非堆积冗余)
expect(find(mgr, first.id)?.hitCount).toBe(1);
// 不新增条目
expect(mgr.size()).toBe(1);
});
});
describe("SkillManager.resolve —— 检索即升权 + superseded 不返回", () => {
it("命中 → hitCount++ 并返回升权后技能", () => {
const mgr = createSkillManager();
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
mgr.register(s);
const hit = mgr.resolve(trigI2c, sigI2c);
expect(hit).not.toBeNull();
expect(hit?.id).toBe(s.id);
expect(hit?.hitCount).toBe(1); // 0 → 1
// 再 resolve 再 +1
const hit2 = mgr.resolve(trigI2c, sigI2c);
expect(hit2?.hitCount).toBe(2);
});
it("无匹配 → null", () => {
const mgr = createSkillManager();
// 只有 flash 技能,查 i2c
mgr.register(makeSkill({ trigger: trigFlash, procedure: procFlash }));
expect(mgr.resolve(trigI2c, sigI2c)).toBeNull();
});
it("superseded(active=false)技能不被返回", () => {
const mgr = createSkillManager();
const old = makeSkill({ trigger: trigI2c, procedure: procI2c });
mgr.register(old);
// 用 supersede 把 old 置 active=false(注册 flash 新技能取代)
const newer = makeSkill({ trigger: trigFlash, procedure: procFlash });
mgr.supersede(old.id, newer);
// old(i2c, inactive) 不返回;newer(flash) actionClass 不匹配 i2c → null
expect(mgr.resolve(trigI2c, sigI2c)).toBeNull();
});
});
describe("SkillManager.supersede —— 取代链", () => {
it("旧 active=false + supersededBy=新;新已注册", () => {
const mgr = createSkillManager();
const old = makeSkill({ trigger: trigI2c, procedure: procI2c });
mgr.register(old);
const newSkill = makeSkill({
trigger: trigFlash,
procedure: procFlash,
});
mgr.supersede(old.id, newSkill);
const oldAfter = find(mgr, old.id);
expect(oldAfter?.active).toBe(false);
expect(oldAfter?.supersededBy).toBe(newSkill.id);
expect(find(mgr, newSkill.id)).not.toBeNull();
});
});
describe("SkillManager.merge —— provenance union + execSamples 累加 + b 失活", () => {
it("合并后 provenance 并集 + execSamples 求和;b active=false supersededBy=a", () => {
const mgr = createSkillManager();
const a = makeSkill({
id: "a",
trigger: trigI2c,
procedure: procI2c,
provenance: ["t1"],
quality: { rubricScore: 0.5, execPassRate: 0.5, execSamples: 4 },
hitCount: 2,
});
const b = makeSkill({
id: "b",
trigger: trigI2c,
procedure: [{ tool: "i2cget", argsTemplate: "-y 0 0x3a" }],
provenance: ["t2"],
quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 2 },
hitCount: 1,
});
mgr.register(a);
mgr.register(b);
const merged = mgr.merge("a", "b");
expect(merged).not.toBeNull();
expect(merged?.id).toBe("a"); // id 用 a 的
expect(merged?.provenance).toEqual(["t1", "t2"]); // union
expect(merged?.quality.execSamples).toBe(6); // 4 + 2
expect(merged?.hitCount).toBe(3); // 2 + 1
// b 失活 + supersededBy=a
const bAfter = find(mgr, "b");
expect(bAfter?.active).toBe(false);
expect(bAfter?.supersededBy).toBe("a");
});
it("任一 id 不存在 → null", () => {
const mgr = createSkillManager();
mgr.register(makeSkill({ id: "a" }));
expect(mgr.merge("a", "missing")).toBeNull();
expect(mgr.merge("missing", "a")).toBeNull();
});
});
describe("SkillManager.evict —— 低质量淘汰(active=false,不物理删)", () => {
it("qualityScore < DEFAULT_MIN_QUALITY → active=false;高质量保留", () => {
const mgr = createSkillManager();
// 低质量:execPassRate=0, rubricScore=0 → qualityScore=0 < 0.3
const bad = makeSkill({
id: "bad",
quality: { rubricScore: 0, execPassRate: 0, execSamples: 3 },
});
// 高质量:execPassRate=1, rubricScore=1 → qualityScore=1.0
const good = makeSkill({
id: "good",
trigger: trigFlash,
procedure: procFlash,
quality: { rubricScore: 1, execPassRate: 1, execSamples: 5 },
});
mgr.register(bad);
mgr.register(good);
const n = mgr.evict();
expect(n).toBe(1);
expect(find(mgr, "bad")?.active).toBe(false);
expect(find(mgr, "good")?.active).toBe(true);
// 不物理删——保 provenance 可追溯
expect(mgr.size()).toBe(2);
});
it("自定义 minQuality 阈值", () => {
const mgr = createSkillManager();
// qualityScore = 0.7*0.5 + 0.3*0.5 = 0.5
const mid = makeSkill({
id: "mid",
quality: { rubricScore: 0.5, execPassRate: 0.5, execSamples: 4 },
});
mgr.register(mid);
// 阈值 0.6 → 0.5 < 0.6 → 淘汰
expect(mgr.evict({ minQuality: 0.6 })).toBe(1);
expect(find(mgr, "mid")?.active).toBe(false);
});
});
describe("SkillManager.list/size/clear", () => {
it("委托 registry", () => {
const mgr = createSkillManager();
mgr.register(makeSkill({ id: "a" }));
mgr.register(makeSkill({ id: "b", trigger: trigFlash, procedure: procFlash }));
expect(mgr.size()).toBe(2);
expect(mgr.list().length).toBe(2);
mgr.clear();
expect(mgr.size()).toBe(0);
});
});
describe("纯函数 sanity —— extractSkill/infoGainGate/mergeSkills/scoreSkill", () => {
it("extractSkill:空 existing → infoGain=1;同质 existing → infoGain<1", () => {
const cand = extractSkill(makeInput(), []);
expect(cand.infoGain).toBe(1);
expect(cand.active).toBe(true);
const existing = [extractSkill(makeInput(), [])];
const dup = extractSkill(makeInput(), existing);
expect(dup.infoGain).toBeLessThan(1);
});
it("infoGainGate:近重复 → allow=false + nearestId;空 → allow=true", () => {
const cand = makeSkill({ trigger: trigI2c, procedure: procI2c });
// 空集
const g0 = infoGainGate(cand, [], {
maxSimilarity: DEFAULT_MAX_SIMILARITY,
minInfoGain: DEFAULT_MIN_INFO_GAIN,
});
expect(g0.allow).toBe(true);
expect(g0.nearestId).toBeNull();
// 近重复
const existing = [makeSkill({ id: "e1", trigger: trigI2c, procedure: procI2c })];
const g1 = infoGainGate(cand, existing, {
maxSimilarity: DEFAULT_MAX_SIMILARITY,
minInfoGain: DEFAULT_MIN_INFO_GAIN,
});
expect(g1.allow).toBe(false);
expect(g1.nearestId).toBe("e1");
});
it("mergeSkills:provenance union + execSamples 求和 + id 用 a", () => {
const a = makeSkill({
id: "a",
provenance: ["x"],
quality: { rubricScore: 0.4, execPassRate: 0.5, execSamples: 2 },
hitCount: 1,
});
const b = makeSkill({
id: "b",
provenance: ["y"],
quality: { rubricScore: 0.8, execPassRate: 1.0, execSamples: 2 },
hitCount: 3,
});
const m = mergeSkills(a, b);
expect(m.id).toBe("a");
expect(m.provenance).toEqual(["x", "y"]);
expect(m.quality.execSamples).toBe(4);
expect(m.hitCount).toBe(4);
expect(Object.isFrozen(m)).toBe(true);
});
it("scoreSkill:actionClass 三方匹配 → >0;不匹配 → 0", () => {
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
expect(scoreSkill(s, trigI2c, sigI2c)).toBeGreaterThan(0);
expect(scoreSkill(s, trigFlash, sigI2c)).toBe(0);
});
});
describe("createSkillManager 工厂", () => {
it("默认参数 → 可用 SkillManager", () => {
const mgr = createSkillManager();
expect(mgr.size()).toBe(0);
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
expect(mgr.size()).toBe(1);
});
it("可注入自定义 registry + guard", () => {
const reg = createSkillRegistry();
const mgr = createSkillManager(reg, {
maxSimilarity: 0.5,
minInfoGain: 0.5,
});
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
mgr.register(s);
expect(reg.get(s.id)).not.toBeNull();
expect(mgr.list().length).toBe(1);
});
});