M2 L2 可学技能注册表层落地(设计文档 §8/§19.3 M2.1-M2.6):
- M2.1 shared/types.ts: HardwareSkill/SkillTrigger/SkillStep/Granularity/SkillQuality/
SkillExtractInput/InfoGainGuard + assessQuality/skillSimilarity/computeInfoGain/
skillId 纯函数
- M2.1-2.5 core/skill-manager.ts: SkillRegistry(register/resolve/list 仿 FlashRegistry)
+ SkillManager μ(extract/register/resolve/supersede/merge/evict) + infoGain 守卫
(§4.4: 1-最近邻相似度; >0.85 → merge 不新增防 plateau) + 双信号质量(exec 0.7
+rubric 0.3, §8.4) + supersededBy 链 + extractSkill/infoGainGate/mergeSkills/scoreSkill
纯函数; merge 路径累积 execSamples+provenance(§8.4 exec 信号随复用累积)
- M2.6 extensions/skill-bank.ts: before_agent_start 检索+注入 <skill_pack> +
session_before_compact μ extract/register + skill_recall 工具(仿 experience-bank)
- barrel 第 24 工厂登记
M3 L3 工具生长层落地(设计文档 §9/§19.3 M3.1-M3.5):
- M3.1 shared/types.ts: GrownTool/ToolLifecycleStage/ToolGrowthClass/ToolUpgradeGate/
ToolProposal + classifyTool/canGrow/definitionHash/grownToolId
- M3.1-3.5 core/tool-growth.ts: ToolGrowthManager(propose 白名单 forbidden 拒绝 throw
§4.8 risk 不增 / recordUsage / tryPromote 升级门 K+repeatable / verify / retire
生命周期 draft→verified→promoted→retired / brickRate 恒0 构造铁律) +
classifyProposal/promotionGate/canTransition 纯函数
- M3.1/3.4 extensions/tool-bank.ts: tool_propose/tool_record/tool_promote/
tool_list_grown/tool_retire 工具 + before_agent_start 注入 <grown_tools>
- barrel 第 25 工厂登记
M2 量化评估(tests/eval/skill-registry-eval.test.ts, seeded mulberry32 N=400 5 队列
+ Wilson CI + 真 manager 对齐, 独立复跑一致):
ΔJ_full=0.265>=0.15 用技能 61%->87.5%(+26.5pp)
ΔJ_held_out=0.2545>=0.05 held-out 板 +25.5pp(排除 F4)
因果技能增益=0.3482>=0.20
distill-off |J_D-J_B|=0<0.05 证改进来自技能增长
F1 负控: 守卫 OFF 技能数 400>12(守卫 ON 6) 但 J_E=0.685<J_A=0.875
→ infoGain 守卫防 plateau 实证(§4.4)
supersede active 有界(6<=8) 不堆积死技能
CI 非重叠 STRONG (J_A_lo=0.839>J_B_hi=0.657)
真 SkillManager 与对照 5 指标全等(J_A/B/C/D+active/store)
M3 量化评估(tests/eval/tool-growth-eval.test.ts, seeded N=300, 真 manager):
步效率 0.24<0.70 固化工具砍 76% 步(5.0->1.2 步/任务)
ΔJ=0.1367>=0.05 固化工具更可靠
砖板率=0 EXACT 构造铁律(forbidden 全拒+只长读/探+sandbox+升级门)
forbidden 7 拒/0 受(§4.8 risk 不增)
升级门低 usage(<K) promote=0
CI 非重叠 STRONG (J_A_lo=0.769>J_B_hi=0.730)
结论: M2+M3 自进化机制有效且安全(量化/可信/可复现)。诚实边界(M1-M3.* 跟进):
评估为忠实 mock 策略受控实验证机制因果效应; live-LLM 读 experiencePack/
skillPack/grown_tools 真受益的端到端门 + derive* 真接线 + sqlite 持久化 +
vm 沙箱 execute 为 M1-M3.* 工程项。
验证: tsc 0 / 全量 49 文件 2697 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot 过
多 Agent 并行: D(skill-manager)+E(skill-bank+eval)+F(tool-growth full)
Co-Authored-By: Claude <noreply@anthropic.com>
367 lines
12 KiB
TypeScript
367 lines
12 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
||
import type {
|
||
HardwareSkill,
|
||
SkillExtractInput,
|
||
SkillStep,
|
||
SkillTrigger,
|
||
TaskSignature,
|
||
} from "../src/shared/types.js";
|
||
import { skillId } from "../src/shared/types.js";
|
||
import {
|
||
createSkillManager,
|
||
createSkillRegistry,
|
||
extractSkill,
|
||
infoGainGate,
|
||
mergeSkills,
|
||
scoreSkill,
|
||
DEFAULT_MIN_INFO_GAIN,
|
||
DEFAULT_MAX_SIMILARITY,
|
||
DEFAULT_MIN_QUALITY,
|
||
} from "../src/core/skill-manager.js";
|
||
import type { SkillManager } from "../src/core/skill-manager.js";
|
||
|
||
/** SkillManager 接口无 get——用 list().find 检视存储技能。 */
|
||
function find(mgr: SkillManager, id: string): HardwareSkill | null {
|
||
return mgr.list().find((s) => s.id === id) ?? null;
|
||
}
|
||
|
||
// ---- 测试夹具 ----
|
||
const trigI2c: SkillTrigger = {
|
||
kind: "event",
|
||
pattern: "i2c no such device",
|
||
actionClass: "i2c-probe",
|
||
};
|
||
const trigFlash: SkillTrigger = {
|
||
kind: "event",
|
||
pattern: "flash write timeout",
|
||
actionClass: "flash",
|
||
};
|
||
const sigI2c: TaskSignature = {
|
||
actionClass: "i2c-probe",
|
||
target: "pmic 0x3a",
|
||
constraints: ["aarch64"],
|
||
};
|
||
|
||
const procI2c: SkillStep[] = [
|
||
{ tool: "i2cdetect", argsTemplate: "-y 0", expect: "0x3a" },
|
||
];
|
||
const procFlash: SkillStep[] = [
|
||
{ tool: "dd", argsTemplate: "if=img of=/dev/mmcblk0" },
|
||
];
|
||
|
||
/** 构造 SkillExtractInput(μ 抽取入参)。 */
|
||
function makeInput(over: Partial<SkillExtractInput> = {}): SkillExtractInput {
|
||
const trigger = over.trigger ?? trigI2c;
|
||
const procedure = over.procedure ?? procI2c;
|
||
const granularity = over.granularity ?? "event-driven";
|
||
return {
|
||
trigger,
|
||
procedure,
|
||
granularity,
|
||
provenance: over.provenance ?? ["trace#1"],
|
||
rubricScore: over.rubricScore ?? 0.5,
|
||
execResults: over.execResults ?? [{ passed: true }, { passed: true }, { passed: true }, { passed: false }],
|
||
createdAt: over.createdAt ?? 1000,
|
||
};
|
||
}
|
||
|
||
/** 直接构造一条 HardwareSkill(绕过 extract,便于精控字段)。 */
|
||
function makeSkill(over: Partial<HardwareSkill> = {}): HardwareSkill {
|
||
const trigger = over.trigger ?? trigI2c;
|
||
const procedure = over.procedure ?? procI2c;
|
||
const granularity = over.granularity ?? "event-driven";
|
||
const skill: HardwareSkill = {
|
||
id: over.id ?? skillId({ trigger, procedure, granularity }),
|
||
name: over.name ?? `${trigger.actionClass}:${trigger.kind}`,
|
||
trigger,
|
||
procedure,
|
||
granularity,
|
||
provenance: over.provenance ?? ["trace#1"],
|
||
quality: over.quality ?? { rubricScore: 0.5, execPassRate: 0.75, execSamples: 4 },
|
||
infoGain: over.infoGain ?? 1,
|
||
createdAt: over.createdAt ?? 1000,
|
||
hitCount: over.hitCount ?? 0,
|
||
active: over.active ?? true,
|
||
supersededBy: over.supersededBy,
|
||
};
|
||
return skill;
|
||
}
|
||
|
||
describe("SkillManager.extract —— μ 抽取(纯构造,不注册)", () => {
|
||
it("空注册表 → infoGain=1.0(无最近邻)", () => {
|
||
const mgr = createSkillManager();
|
||
const cand = mgr.extract(makeInput());
|
||
expect(cand.infoGain).toBe(1);
|
||
expect(cand.active).toBe(true);
|
||
expect(cand.hitCount).toBe(0);
|
||
expect(Object.isFrozen(cand)).toBe(true);
|
||
// extract 不注册
|
||
expect(mgr.size()).toBe(0);
|
||
});
|
||
|
||
it("对相似已有技能 → infoGain<1", () => {
|
||
const mgr = createSkillManager();
|
||
// 先注册一条 i2c 技能
|
||
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
|
||
// 再 extract 同 trigger+procedure → 与已有完全相似
|
||
const cand = mgr.extract(makeInput({ trigger: trigI2c, procedure: procI2c }));
|
||
expect(cand.infoGain).toBeLessThan(1);
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.register —— §4.4 infoGain 守卫(防 plateau)", () => {
|
||
it("genuinely 新增技能(不同 actionClass)→ added:true", () => {
|
||
const mgr = createSkillManager();
|
||
// 已有 i2c 技能
|
||
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
|
||
// 注册 flash 技能(不同 actionClass → 低相似 → 高 infoGain)
|
||
const flash = makeSkill({ trigger: trigFlash, procedure: procFlash });
|
||
const r = mgr.register(flash);
|
||
expect(r.added).toBe(true);
|
||
expect(mgr.size()).toBe(2);
|
||
});
|
||
|
||
it("近重复(同 trigger+procedure)→ added:false, mergedWith=nearest, nearest.hitCount++", () => {
|
||
const mgr = createSkillManager();
|
||
const first = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
mgr.register(first);
|
||
const beforeHit = find(mgr, first.id)?.hitCount;
|
||
expect(beforeHit).toBe(0);
|
||
|
||
// 近重复:同 trigger+procedure → 相似度 1 → infoGain=0 → 守卫拒
|
||
const dup = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
const r = mgr.register(dup);
|
||
expect(r.added).toBe(false);
|
||
expect(r.mergedWith).toBe(first.id);
|
||
// nearest hitCount 升权(plateau 预防:强化已有而非堆积冗余)
|
||
expect(find(mgr, first.id)?.hitCount).toBe(1);
|
||
// 不新增条目
|
||
expect(mgr.size()).toBe(1);
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.resolve —— 检索即升权 + superseded 不返回", () => {
|
||
it("命中 → hitCount++ 并返回升权后技能", () => {
|
||
const mgr = createSkillManager();
|
||
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
mgr.register(s);
|
||
const hit = mgr.resolve(trigI2c, sigI2c);
|
||
expect(hit).not.toBeNull();
|
||
expect(hit?.id).toBe(s.id);
|
||
expect(hit?.hitCount).toBe(1); // 0 → 1
|
||
// 再 resolve 再 +1
|
||
const hit2 = mgr.resolve(trigI2c, sigI2c);
|
||
expect(hit2?.hitCount).toBe(2);
|
||
});
|
||
|
||
it("无匹配 → null", () => {
|
||
const mgr = createSkillManager();
|
||
// 只有 flash 技能,查 i2c
|
||
mgr.register(makeSkill({ trigger: trigFlash, procedure: procFlash }));
|
||
expect(mgr.resolve(trigI2c, sigI2c)).toBeNull();
|
||
});
|
||
|
||
it("superseded(active=false)技能不被返回", () => {
|
||
const mgr = createSkillManager();
|
||
const old = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
mgr.register(old);
|
||
// 用 supersede 把 old 置 active=false(注册 flash 新技能取代)
|
||
const newer = makeSkill({ trigger: trigFlash, procedure: procFlash });
|
||
mgr.supersede(old.id, newer);
|
||
// old(i2c, inactive) 不返回;newer(flash) actionClass 不匹配 i2c → null
|
||
expect(mgr.resolve(trigI2c, sigI2c)).toBeNull();
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.supersede —— 取代链", () => {
|
||
it("旧 active=false + supersededBy=新;新已注册", () => {
|
||
const mgr = createSkillManager();
|
||
const old = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
mgr.register(old);
|
||
const newSkill = makeSkill({
|
||
trigger: trigFlash,
|
||
procedure: procFlash,
|
||
});
|
||
mgr.supersede(old.id, newSkill);
|
||
const oldAfter = find(mgr, old.id);
|
||
expect(oldAfter?.active).toBe(false);
|
||
expect(oldAfter?.supersededBy).toBe(newSkill.id);
|
||
expect(find(mgr, newSkill.id)).not.toBeNull();
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.merge —— provenance union + execSamples 累加 + b 失活", () => {
|
||
it("合并后 provenance 并集 + execSamples 求和;b active=false supersededBy=a", () => {
|
||
const mgr = createSkillManager();
|
||
const a = makeSkill({
|
||
id: "a",
|
||
trigger: trigI2c,
|
||
procedure: procI2c,
|
||
provenance: ["t1"],
|
||
quality: { rubricScore: 0.5, execPassRate: 0.5, execSamples: 4 },
|
||
hitCount: 2,
|
||
});
|
||
const b = makeSkill({
|
||
id: "b",
|
||
trigger: trigI2c,
|
||
procedure: [{ tool: "i2cget", argsTemplate: "-y 0 0x3a" }],
|
||
provenance: ["t2"],
|
||
quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 2 },
|
||
hitCount: 1,
|
||
});
|
||
mgr.register(a);
|
||
mgr.register(b);
|
||
|
||
const merged = mgr.merge("a", "b");
|
||
expect(merged).not.toBeNull();
|
||
expect(merged?.id).toBe("a"); // id 用 a 的
|
||
expect(merged?.provenance).toEqual(["t1", "t2"]); // union
|
||
expect(merged?.quality.execSamples).toBe(6); // 4 + 2
|
||
expect(merged?.hitCount).toBe(3); // 2 + 1
|
||
|
||
// b 失活 + supersededBy=a
|
||
const bAfter = find(mgr, "b");
|
||
expect(bAfter?.active).toBe(false);
|
||
expect(bAfter?.supersededBy).toBe("a");
|
||
});
|
||
|
||
it("任一 id 不存在 → null", () => {
|
||
const mgr = createSkillManager();
|
||
mgr.register(makeSkill({ id: "a" }));
|
||
expect(mgr.merge("a", "missing")).toBeNull();
|
||
expect(mgr.merge("missing", "a")).toBeNull();
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.evict —— 低质量淘汰(active=false,不物理删)", () => {
|
||
it("qualityScore < DEFAULT_MIN_QUALITY → active=false;高质量保留", () => {
|
||
const mgr = createSkillManager();
|
||
// 低质量:execPassRate=0, rubricScore=0 → qualityScore=0 < 0.3
|
||
const bad = makeSkill({
|
||
id: "bad",
|
||
quality: { rubricScore: 0, execPassRate: 0, execSamples: 3 },
|
||
});
|
||
// 高质量:execPassRate=1, rubricScore=1 → qualityScore=1.0
|
||
const good = makeSkill({
|
||
id: "good",
|
||
trigger: trigFlash,
|
||
procedure: procFlash,
|
||
quality: { rubricScore: 1, execPassRate: 1, execSamples: 5 },
|
||
});
|
||
mgr.register(bad);
|
||
mgr.register(good);
|
||
|
||
const n = mgr.evict();
|
||
expect(n).toBe(1);
|
||
expect(find(mgr, "bad")?.active).toBe(false);
|
||
expect(find(mgr, "good")?.active).toBe(true);
|
||
// 不物理删——保 provenance 可追溯
|
||
expect(mgr.size()).toBe(2);
|
||
});
|
||
|
||
it("自定义 minQuality 阈值", () => {
|
||
const mgr = createSkillManager();
|
||
// qualityScore = 0.7*0.5 + 0.3*0.5 = 0.5
|
||
const mid = makeSkill({
|
||
id: "mid",
|
||
quality: { rubricScore: 0.5, execPassRate: 0.5, execSamples: 4 },
|
||
});
|
||
mgr.register(mid);
|
||
// 阈值 0.6 → 0.5 < 0.6 → 淘汰
|
||
expect(mgr.evict({ minQuality: 0.6 })).toBe(1);
|
||
expect(find(mgr, "mid")?.active).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe("SkillManager.list/size/clear", () => {
|
||
it("委托 registry", () => {
|
||
const mgr = createSkillManager();
|
||
mgr.register(makeSkill({ id: "a" }));
|
||
mgr.register(makeSkill({ id: "b", trigger: trigFlash, procedure: procFlash }));
|
||
expect(mgr.size()).toBe(2);
|
||
expect(mgr.list().length).toBe(2);
|
||
mgr.clear();
|
||
expect(mgr.size()).toBe(0);
|
||
});
|
||
});
|
||
|
||
describe("纯函数 sanity —— extractSkill/infoGainGate/mergeSkills/scoreSkill", () => {
|
||
it("extractSkill:空 existing → infoGain=1;同质 existing → infoGain<1", () => {
|
||
const cand = extractSkill(makeInput(), []);
|
||
expect(cand.infoGain).toBe(1);
|
||
expect(cand.active).toBe(true);
|
||
|
||
const existing = [extractSkill(makeInput(), [])];
|
||
const dup = extractSkill(makeInput(), existing);
|
||
expect(dup.infoGain).toBeLessThan(1);
|
||
});
|
||
|
||
it("infoGainGate:近重复 → allow=false + nearestId;空 → allow=true", () => {
|
||
const cand = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
// 空集
|
||
const g0 = infoGainGate(cand, [], {
|
||
maxSimilarity: DEFAULT_MAX_SIMILARITY,
|
||
minInfoGain: DEFAULT_MIN_INFO_GAIN,
|
||
});
|
||
expect(g0.allow).toBe(true);
|
||
expect(g0.nearestId).toBeNull();
|
||
|
||
// 近重复
|
||
const existing = [makeSkill({ id: "e1", trigger: trigI2c, procedure: procI2c })];
|
||
const g1 = infoGainGate(cand, existing, {
|
||
maxSimilarity: DEFAULT_MAX_SIMILARITY,
|
||
minInfoGain: DEFAULT_MIN_INFO_GAIN,
|
||
});
|
||
expect(g1.allow).toBe(false);
|
||
expect(g1.nearestId).toBe("e1");
|
||
});
|
||
|
||
it("mergeSkills:provenance union + execSamples 求和 + id 用 a", () => {
|
||
const a = makeSkill({
|
||
id: "a",
|
||
provenance: ["x"],
|
||
quality: { rubricScore: 0.4, execPassRate: 0.5, execSamples: 2 },
|
||
hitCount: 1,
|
||
});
|
||
const b = makeSkill({
|
||
id: "b",
|
||
provenance: ["y"],
|
||
quality: { rubricScore: 0.8, execPassRate: 1.0, execSamples: 2 },
|
||
hitCount: 3,
|
||
});
|
||
const m = mergeSkills(a, b);
|
||
expect(m.id).toBe("a");
|
||
expect(m.provenance).toEqual(["x", "y"]);
|
||
expect(m.quality.execSamples).toBe(4);
|
||
expect(m.hitCount).toBe(4);
|
||
expect(Object.isFrozen(m)).toBe(true);
|
||
});
|
||
|
||
it("scoreSkill:actionClass 三方匹配 → >0;不匹配 → 0", () => {
|
||
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
expect(scoreSkill(s, trigI2c, sigI2c)).toBeGreaterThan(0);
|
||
expect(scoreSkill(s, trigFlash, sigI2c)).toBe(0);
|
||
});
|
||
});
|
||
|
||
describe("createSkillManager 工厂", () => {
|
||
it("默认参数 → 可用 SkillManager", () => {
|
||
const mgr = createSkillManager();
|
||
expect(mgr.size()).toBe(0);
|
||
mgr.register(makeSkill({ trigger: trigI2c, procedure: procI2c }));
|
||
expect(mgr.size()).toBe(1);
|
||
});
|
||
|
||
it("可注入自定义 registry + guard", () => {
|
||
const reg = createSkillRegistry();
|
||
const mgr = createSkillManager(reg, {
|
||
maxSimilarity: 0.5,
|
||
minInfoGain: 0.5,
|
||
});
|
||
const s = makeSkill({ trigger: trigI2c, procedure: procI2c });
|
||
mgr.register(s);
|
||
expect(reg.get(s.id)).not.toBeNull();
|
||
expect(mgr.list().length).toBe(1);
|
||
});
|
||
});
|