M4 L4 编排进化层落地(设计文档 §10/§19.3 M4.1-M4.5):
- M4.1 shared/types.ts: DifficultyLevel/DifficultyEstimate/WorkflowNode/
WorkflowGraph/DifficultyCorrection + estimateDifficulty/riskOfWorkflow/
workflowSimilarity/backpropGradient/correctDifficultyPrior 纯函数
- M4.1-4.4 core/workflow-optimizer.ts: WorkflowRegistry(按 taskClass 分桶)
+ WorkflowOptimizer(estimateDifficulty/selectWorkflow升权/evolveWorkflow
符号反向传播 backpropGradient+applyGradient+§4.8 risk 不增守卫/
registerWorkflow supersede/correctPrior 滑动窗) + applyGradient 纯函数
- M4.5 extensions/orchestration-bank.ts: before_agent_start 难度估+workflow
注入 <orchestration_plan> + session_before_compact 先验校正 +
orchestration_recall 工具(仿 skill-bank)
- barrel 第 26 工厂登记
M5 L5 脚手架自改层落地(终局最强约束,设计文档 §11/§19.3 M5.1-M5.5):
- M5.1 shared/types.ts: VariantStage/ScaffoldClass/Variant/InvariantReport/
InvariantChecker + classifyScaffold/canSelfModify/allInvariantsPassed 纯函数
- M5.1-5.3 core/variant-manager.ts: VariantRegistry + VariantManager(propose
白名单 forbidden 命门 throw §11.3/runInvariants 注入 InvariantChecker/
abTest 注入 ABTestRunner/tryPromote 不变式全过+A/B 胜出→promoted 不变式
失败→rolled-back/rollback 可逆/resolve 只返 promoted) +
classifyProposalVariant/promotionDecision 纯函数
- M5.4-5.5 extensions/variant-bank.ts: variant_propose/variant_list/
variant_promote/variant_rollback 工具 + 注入 <active_variants>(仿 tool-bank)
- barrel 第 27 工厂登记
M4 量化评估(tests/eval/orchestration-eval.test.ts, seeded mulberry32 N=400 5 队列
+ Wilson CI + 真 optimizer 对齐, 独立复跑一致):
ΔJ_G=0.2975>=0.10 workflow 进化 49.25%->79%(+29.75pp, θ 固定归因 G 非模型)
ΔJ_held_out=0.3182>=0.03 held-out 板 +31.82pp(泛化)
同类不同 workflow max 2/class>=2 非平凡进化
难度估与实际步数 Pearson corr=0.6121>0 估计器有意义
evolve-off |J_D-J_B|=0<0.05 证改进来自 G 增长非 select 路径
risk 不增 EXACT 构造铁律 max(evolved risk)=1<=default=1(§4.8)
risk-increase 负控 242 退化候选全被守卫拒绝
CI 非重叠 STRONG (J_A_lo=0.7474>J_B_hi=0.5413)
真 WorkflowOptimizer 与对照 selectWorkflow+thresholds 对齐
M5 量化评估(tests/eval/variant-eval.test.ts, seeded N=200, 注入 checker+runner
+真 manager 对齐):
ΔJ_selfmod=0.15>=0.05 有益自改 67%->82%(+15pp)
forbidden 命门 5 拒/0 受 EXACT 构造铁律(§11.3 safety-gate/sandbox/flash/
glm-provider 永不自改)
§11.4 门捕有害变体 2>=1(不变式失败被拒)
回滚触发 1>=1(§11.6 可逆性实证 promoted->rolled-back)
promoted 有益 3>=1(非零自改) + 全 promoted allPassed
CI 非重叠 STRONG (J_A_lo=0.7609>J_B_hi=0.7314)
真 VariantManager 与对照 J_A/B/ΔJ 全等
结论: M4+M5 自进化机制有效且安全(量化/可信/可复现)。M1-M5 五层机制闭环
全部完成, pba 从'静态装配 agent'->'L1-L5 全栈自进化 agent'。诚实边界
(M1-M5.* 跟进): 评估为忠实 mock 策略受控实验证机制因果效应; live-LLM
读注入包真受益的端到端门 + derive* 真接线 + sqlite 持久化 + vm 沙箱 execute
变体工厂 + 真 CI §11.4 不变式执行(真 tsc/单测/launch/砖板率) 为 M1-M5.*
工程项。
验证: tsc 0 / 全量 57 文件 2875 passed 5 skipped / sk- 明文 0 / 真 pi T01
boot 过(barrel 27 扩展含 L1-L5 全栈自进化层)
多 Agent 并行: G(workflow-optimizer)+H(orchestration-bank+eval)+
I(variant-manager)+J(variant-bank+eval)
Co-Authored-By: Claude <noreply@anthropic.com>
375 lines
14 KiB
TypeScript
375 lines
14 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
||
import type {
|
||
InvariantChecker,
|
||
InvariantReport,
|
||
Variant,
|
||
} from "../src/shared/types.js";
|
||
import {
|
||
DEFAULT_AB_THRESHOLD,
|
||
createVariantManager,
|
||
createVariantRegistry,
|
||
classifyProposalVariant,
|
||
promotionDecision,
|
||
} from "../src/core/variant-manager.js";
|
||
import type { ABTestRunner, VariantManager, VariantProposal } from "../src/core/variant-manager.js";
|
||
|
||
// ---- 测试夹具 ----
|
||
|
||
/** 构造一条 VariantProposal(默认指向可自改的 pdf-reader 扩展)。 */
|
||
function makeProposal(over: Partial<VariantProposal> = {}): VariantProposal {
|
||
return {
|
||
name: over.name ?? "pdf-reader-tweak",
|
||
targetExtension: over.targetExtension ?? "pdf-reader",
|
||
description: over.description ?? "tweak pdf parsing logic",
|
||
baseRef: over.baseRef ?? "main",
|
||
factoryOverride: over.factoryOverride ?? "override-A",
|
||
createdAt: over.createdAt ?? 1000,
|
||
};
|
||
}
|
||
|
||
/** 全过不变式报告。 */
|
||
const ALL_PASS: InvariantReport = {
|
||
tests: true,
|
||
tsc: true,
|
||
noSecrets: true,
|
||
modelLaunch: true,
|
||
sandboxGate: true,
|
||
brickRateZero: true,
|
||
riskNonIncrease: true,
|
||
allPassed: true,
|
||
};
|
||
|
||
/** 不变式不过报告(tests 失败)。 */
|
||
const SOME_FAIL: InvariantReport = {
|
||
tests: false,
|
||
tsc: true,
|
||
noSecrets: true,
|
||
modelLaunch: true,
|
||
sandboxGate: true,
|
||
brickRateZero: true,
|
||
riskNonIncrease: true,
|
||
allPassed: false,
|
||
};
|
||
|
||
/** 构造一个 fake A/B runner,返回固定结果。 */
|
||
function fakeRunner(succVariant: number, succBaseline: number, N = 100): ABTestRunner {
|
||
return {
|
||
run: (_v: Variant, _n: number) => ({ succVariant, succBaseline, N }),
|
||
};
|
||
}
|
||
|
||
/** VariantManager 无 get——用 list().find 检视存储变体。 */
|
||
function find(mgr: VariantManager, id: string): Variant | null {
|
||
return mgr.list().find((v) => v.id === id) ?? null;
|
||
}
|
||
|
||
describe("VariantManager.propose —— §11.3 白名单预检", () => {
|
||
it("可自改扩展(read/probe/analyze/orchestrate)→ candidate", () => {
|
||
const mgr = createVariantManager();
|
||
// read 类(pdf-reader 默认归 read)
|
||
const v1 = mgr.propose(makeProposal({ name: "pdf", targetExtension: "pdf-reader", description: "tweak parse", factoryOverride: "pdf-A" }));
|
||
expect(v1.stage).toBe("candidate");
|
||
expect(v1.scaffoldClass).toBe("read");
|
||
expect(Object.isFrozen(v1)).toBe(true);
|
||
// probe 类(env-probe)
|
||
const v2 = mgr.propose(makeProposal({ name: "env", targetExtension: "env-probe", description: "probe i2c bus", factoryOverride: "env-A" }));
|
||
expect(v2.scaffoldClass).toBe("probe");
|
||
// analyze 类(hardware-verify)
|
||
const v3 = mgr.propose(makeProposal({ name: "verify", targetExtension: "hardware-verify", description: "verify signal", factoryOverride: "verify-A" }));
|
||
expect(v3.scaffoldClass).toBe("analyze");
|
||
// orchestrate 类(loop-orchestrator)
|
||
const v4 = mgr.propose(makeProposal({ name: "loop", targetExtension: "loop-orchestrator", description: "workflow", factoryOverride: "loop-A" }));
|
||
expect(v4.scaffoldClass).toBe("orchestrate");
|
||
expect(mgr.size()).toBe(4);
|
||
});
|
||
|
||
it("forbidden 命门(safety-gate/sandbox-policy/flash/glm-provider/reg-tool)→ throws §11.3", () => {
|
||
const mgr = createVariantManager();
|
||
const forbidden: Array<[string, string]> = [
|
||
["safety-gate", "tweak safety gate"],
|
||
["sandbox-policy", "relax sandbox"],
|
||
["flash", "change flash three-stage"],
|
||
["glm-provider", "patch glm provider"],
|
||
["reg-tool", "modify register write"],
|
||
];
|
||
for (const [ext, desc] of forbidden) {
|
||
expect(() =>
|
||
mgr.propose(makeProposal({ targetExtension: ext, description: desc, name: `${ext}-tweak` })),
|
||
).toThrowError(/forbidden scaffold.*§11\.3/);
|
||
}
|
||
// 全部被拒,注册表为空
|
||
expect(mgr.size()).toBe(0);
|
||
});
|
||
|
||
it("变体初建 invariants 全 false + allPassed=false", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
expect(v.invariants.allPassed).toBe(false);
|
||
expect(v.invariants.tests).toBe(false);
|
||
expect(v.invariants.tsc).toBe(false);
|
||
expect(v.invariants.brickRateZero).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.propose —— 幂等(同 name+factoryOverride → 同 id,不重复)", () => {
|
||
it("同 name+factoryOverride → 同 id,size 不增", () => {
|
||
const mgr = createVariantManager();
|
||
const v1 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F" }));
|
||
const v2 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F" }));
|
||
expect(v1.id).toBe(v2.id);
|
||
expect(mgr.size()).toBe(1);
|
||
});
|
||
|
||
it("不同 factoryOverride → 不同 id", () => {
|
||
const mgr = createVariantManager();
|
||
const v1 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F1" }));
|
||
const v2 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F2" }));
|
||
expect(v1.id).not.toBe(v2.id);
|
||
expect(mgr.size()).toBe(2);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.runInvariants —— 委托注入 checker + 更新存储", () => {
|
||
it("默认 checker 全过 → allPassed=true;存储变体 invariants 更新", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
expect(v.invariants.allPassed).toBe(false); // 初建全 false
|
||
const report = mgr.runInvariants(v);
|
||
expect(report.allPassed).toBe(true);
|
||
// 存储副本更新
|
||
expect(find(mgr, v.id)?.invariants.allPassed).toBe(true);
|
||
});
|
||
|
||
it("fake checker 返回 some-false → allPassed=false;存储更新", () => {
|
||
const fakeChecker: InvariantChecker = {
|
||
runChecks: (_v: Variant) => SOME_FAIL,
|
||
};
|
||
const mgr = createVariantManager(createVariantRegistry(), fakeChecker);
|
||
const v = mgr.propose(makeProposal());
|
||
const report = mgr.runInvariants(v);
|
||
expect(report.allPassed).toBe(false);
|
||
expect(report.tests).toBe(false);
|
||
expect(find(mgr, v.id)?.invariants.tests).toBe(false);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.abTest —— 委托注入 runner + 更新 shadowResults", () => {
|
||
it("runner.run 被调用,shadowResults 更新到存储变体", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
expect(v.shadowResults).toBeUndefined();
|
||
const runner = fakeRunner(80, 60, 100);
|
||
const result = mgr.abTest(v, runner);
|
||
expect(result.succVariant).toBe(80);
|
||
expect(result.succBaseline).toBe(60);
|
||
expect(result.N).toBe(100);
|
||
expect(find(mgr, v.id)?.shadowResults).toEqual({ succVariant: 80, succBaseline: 60, N: 100 });
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.tryPromote —— §11.4 + A/B 提升门", () => {
|
||
it("不变式全过 + A/B 胜出 → promoted", () => {
|
||
const mgr = createVariantManager(); // 默认 checker 全过
|
||
const v = mgr.propose(makeProposal());
|
||
const updated = mgr.tryPromote(v.id, fakeRunner(80, 50)); // 80 > 50 → 胜出
|
||
expect(updated.stage).toBe("promoted");
|
||
});
|
||
|
||
it("不变式不过 → rolled-back(不 promoted)", () => {
|
||
const fakeChecker: InvariantChecker = {
|
||
runChecks: (_v: Variant) => SOME_FAIL,
|
||
};
|
||
const mgr = createVariantManager(createVariantRegistry(), fakeChecker);
|
||
const v = mgr.propose(makeProposal());
|
||
const updated = mgr.tryPromote(v.id, fakeRunner(99, 1)); // A/B 胜出但不变式不过
|
||
expect(updated.stage).toBe("rolled-back");
|
||
expect(updated.invariants.allPassed).toBe(false);
|
||
});
|
||
|
||
it("不变式全过但 A/B 未胜出 → 留 candidate(不回滚)", () => {
|
||
const mgr = createVariantManager(); // 默认全过
|
||
const v = mgr.propose(makeProposal());
|
||
// 50 <= 60 → 未胜出(rate=50/60≈0.83 <= 1+0)
|
||
const updated = mgr.tryPromote(v.id, fakeRunner(50, 60));
|
||
expect(updated.stage).toBe("candidate");
|
||
expect(updated.shadowResults).toEqual({ succVariant: 50, succBaseline: 60, N: 100 });
|
||
});
|
||
|
||
it("A/B 平局(双方相等)→ 留 candidate(未严格胜出)", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
// 60 === 60 → rate=1 <= 1+0 → 未胜出
|
||
const updated = mgr.tryPromote(v.id, fakeRunner(60, 60));
|
||
expect(updated.stage).toBe("candidate");
|
||
});
|
||
|
||
it("无 A/B 数据且无 runner → throws §11.4(fail-closed)", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
expect(() => mgr.tryPromote(v.id)).toThrowError(/cannot promote without A\/B data.*§11\.4/);
|
||
// 仍 candidate
|
||
expect(find(mgr, v.id)?.stage).toBe("candidate");
|
||
});
|
||
|
||
it("已有 shadowResults 时无 runner 也能 promote(用已有数据)", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
// 先采 A/B
|
||
mgr.abTest(v, fakeRunner(80, 50));
|
||
// 无 runner tryPromote 用已有 shadowResults
|
||
const updated = mgr.tryPromote(v.id);
|
||
expect(updated.stage).toBe("promoted");
|
||
});
|
||
|
||
it("未知 id → throws", () => {
|
||
const mgr = createVariantManager();
|
||
expect(() => mgr.tryPromote("nope")).toThrowError(/unknown id nope/);
|
||
});
|
||
|
||
it("A/B 双方都 0 → 无信号 → 留 candidate(不变式全过)", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
// 0 vs 0 → promotionDecision 返回 not promotable "both zero"
|
||
const updated = mgr.tryPromote(v.id, fakeRunner(0, 0));
|
||
expect(updated.stage).toBe("candidate");
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.rollback —— §14.3 可逆性", () => {
|
||
it("→ rolled-back;变体留注册表保可追溯", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
mgr.rollback(v.id);
|
||
expect(find(mgr, v.id)?.stage).toBe("rolled-back");
|
||
expect(mgr.size()).toBe(1); // 不物理删
|
||
});
|
||
|
||
it("未知 id → throws", () => {
|
||
const mgr = createVariantManager();
|
||
expect(() => mgr.rollback("nope")).toThrowError(/unknown id nope/);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.retire —— 终态", () => {
|
||
it("→ retired(终态)", () => {
|
||
const mgr = createVariantManager();
|
||
const v = mgr.propose(makeProposal());
|
||
mgr.retire(v.id);
|
||
expect(find(mgr, v.id)?.stage).toBe("retired");
|
||
});
|
||
|
||
it("未知 id → throws", () => {
|
||
const mgr = createVariantManager();
|
||
expect(() => mgr.retire("nope")).toThrowError(/unknown id nope/);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.resolve —— 只返 promoted", () => {
|
||
it("candidate/rolled-back/retired 不返;promoted 返", () => {
|
||
const mgr = createVariantManager();
|
||
const cand = mgr.propose(makeProposal({ name: "v-cand", factoryOverride: "cand" }));
|
||
expect(mgr.resolve("v-cand")).toBeNull(); // candidate
|
||
|
||
const rb = mgr.propose(makeProposal({ name: "v-rb", factoryOverride: "rb" }));
|
||
mgr.rollback(rb.id);
|
||
expect(mgr.resolve("v-rb")).toBeNull(); // rolled-back
|
||
|
||
const rt = mgr.propose(makeProposal({ name: "v-rt", factoryOverride: "rt" }));
|
||
mgr.retire(rt.id);
|
||
expect(mgr.resolve("v-rt")).toBeNull(); // retired
|
||
|
||
const promoted = mgr.propose(makeProposal({ name: "v-pro", factoryOverride: "pro" }));
|
||
mgr.tryPromote(promoted.id, fakeRunner(80, 50));
|
||
const hit = mgr.resolve("v-pro");
|
||
expect(hit).not.toBeNull();
|
||
expect(hit?.stage).toBe("promoted");
|
||
expect(hit?.id).toBe(promoted.id);
|
||
});
|
||
|
||
it("无匹配名 → null", () => {
|
||
const mgr = createVariantManager();
|
||
expect(mgr.resolve("nonexistent")).toBeNull();
|
||
});
|
||
});
|
||
|
||
describe("纯函数 sanity —— classifyProposalVariant / promotionDecision", () => {
|
||
it("classifyProposalVariant:forbidden 关键词 → selfModifiable=false", () => {
|
||
expect(classifyProposalVariant("safety-gate", "tweak").selfModifiable).toBe(false);
|
||
expect(classifyProposalVariant("sandbox-policy", "relax").selfModifiable).toBe(false);
|
||
expect(classifyProposalVariant("flash", "change").selfModifiable).toBe(false);
|
||
expect(classifyProposalVariant("glm-provider", "patch").selfModifiable).toBe(false);
|
||
expect(classifyProposalVariant("reg-tool", "modify").selfModifiable).toBe(false);
|
||
// scaffoldClass === "forbidden"
|
||
expect(classifyProposalVariant("flash", "").scaffoldClass).toBe("forbidden");
|
||
});
|
||
|
||
it("classifyProposalVariant:可改扩展 → selfModifiable=true", () => {
|
||
expect(classifyProposalVariant("pdf-reader", "parse").selfModifiable).toBe(true);
|
||
expect(classifyProposalVariant("env-probe", "probe i2c").selfModifiable).toBe(true);
|
||
expect(classifyProposalVariant("hardware-verify", "verify").selfModifiable).toBe(true);
|
||
expect(classifyProposalVariant("loop-orchestrator", "workflow").selfModifiable).toBe(true);
|
||
});
|
||
|
||
it("promotionDecision:不变式不过 → not promotable", () => {
|
||
const d = promotionDecision(SOME_FAIL, { succVariant: 99, succBaseline: 1, N: 100 }, DEFAULT_AB_THRESHOLD);
|
||
expect(d.promotable).toBe(false);
|
||
expect(d.reason).toMatch(/invariants/);
|
||
});
|
||
|
||
it("promotionDecision:A/B 平局(rate=1)→ not promotable", () => {
|
||
const d = promotionDecision(ALL_PASS, { succVariant: 60, succBaseline: 60, N: 100 }, DEFAULT_AB_THRESHOLD);
|
||
expect(d.promotable).toBe(false);
|
||
expect(d.reason).toMatch(/not winning/);
|
||
});
|
||
|
||
it("promotionDecision:双方都 0 → not promotable(no signal)", () => {
|
||
const d = promotionDecision(ALL_PASS, { succVariant: 0, succBaseline: 0, N: 100 }, DEFAULT_AB_THRESHOLD);
|
||
expect(d.promotable).toBe(false);
|
||
expect(d.reason).toMatch(/both zero/);
|
||
});
|
||
|
||
it("promotionDecision:不变式全过 + A/B 胜出 → promotable", () => {
|
||
const d = promotionDecision(ALL_PASS, { succVariant: 80, succBaseline: 50, N: 100 }, DEFAULT_AB_THRESHOLD);
|
||
expect(d.promotable).toBe(true);
|
||
});
|
||
|
||
it("promotionDecision:基线 0 但变体 >0 → promotable(Infinity rate)", () => {
|
||
const d = promotionDecision(ALL_PASS, { succVariant: 5, succBaseline: 0, N: 100 }, DEFAULT_AB_THRESHOLD);
|
||
expect(d.promotable).toBe(true);
|
||
});
|
||
});
|
||
|
||
describe("createVariantManager / createVariantRegistry 工厂", () => {
|
||
it("createVariantManager() 默认参数 → 可用 VariantManager", () => {
|
||
const mgr = createVariantManager();
|
||
expect(mgr.size()).toBe(0);
|
||
const v = mgr.propose(makeProposal());
|
||
expect(mgr.size()).toBe(1);
|
||
expect(Object.isFrozen(v)).toBe(true);
|
||
});
|
||
|
||
it("可注入自定义 registry + checker", () => {
|
||
const reg = createVariantRegistry();
|
||
const checker: InvariantChecker = {
|
||
runChecks: (_v: Variant) => ALL_PASS,
|
||
};
|
||
const mgr = createVariantManager(reg, checker);
|
||
const v = mgr.propose(makeProposal());
|
||
expect(reg.get(v.id)).not.toBeNull();
|
||
const report = mgr.runInvariants(v);
|
||
expect(report.allPassed).toBe(true);
|
||
});
|
||
});
|
||
|
||
describe("VariantManager.list/size/clear", () => {
|
||
it("委托 registry", () => {
|
||
const mgr = createVariantManager();
|
||
mgr.propose(makeProposal({ name: "a", factoryOverride: "a" }));
|
||
mgr.propose(makeProposal({ name: "b", factoryOverride: "b", targetExtension: "env-probe", description: "probe" }));
|
||
expect(mgr.size()).toBe(2);
|
||
expect(mgr.list().length).toBe(2);
|
||
mgr.clear();
|
||
expect(mgr.size()).toBe(0);
|
||
});
|
||
});
|