physical_byte_agent/tests/variant-manager.test.ts
d-robotics 80e89c9841
Some checks are pending
CI / quality-gate (push) Waiting to run
CI / type-drift (push) Waiting to run
feat(L4+L5): M4 编排进化 + M5 脚手架自改自进化闭环 + 量化坐实 (M1-M5 全栈完成)
M4 L4 编排进化层落地(设计文档 §10/§19.3 M4.1-M4.5):
- M4.1 shared/types.ts: DifficultyLevel/DifficultyEstimate/WorkflowNode/
  WorkflowGraph/DifficultyCorrection + estimateDifficulty/riskOfWorkflow/
  workflowSimilarity/backpropGradient/correctDifficultyPrior 纯函数
- M4.1-4.4 core/workflow-optimizer.ts: WorkflowRegistry(按 taskClass 分桶)
  + WorkflowOptimizer(estimateDifficulty/selectWorkflow升权/evolveWorkflow
  符号反向传播 backpropGradient+applyGradient+§4.8 risk 不增守卫/
  registerWorkflow supersede/correctPrior 滑动窗) + applyGradient 纯函数
- M4.5 extensions/orchestration-bank.ts: before_agent_start 难度估+workflow
  注入 <orchestration_plan> + session_before_compact 先验校正 +
  orchestration_recall 工具(仿 skill-bank)
- barrel 第 26 工厂登记

M5 L5 脚手架自改层落地(终局最强约束,设计文档 §11/§19.3 M5.1-M5.5):
- M5.1 shared/types.ts: VariantStage/ScaffoldClass/Variant/InvariantReport/
  InvariantChecker + classifyScaffold/canSelfModify/allInvariantsPassed 纯函数
- M5.1-5.3 core/variant-manager.ts: VariantRegistry + VariantManager(propose
  白名单 forbidden 命门 throw §11.3/runInvariants 注入 InvariantChecker/
  abTest 注入 ABTestRunner/tryPromote 不变式全过+A/B 胜出→promoted 不变式
  失败→rolled-back/rollback 可逆/resolve 只返 promoted) +
  classifyProposalVariant/promotionDecision 纯函数
- M5.4-5.5 extensions/variant-bank.ts: variant_propose/variant_list/
  variant_promote/variant_rollback 工具 + 注入 <active_variants>(仿 tool-bank)
- barrel 第 27 工厂登记

M4 量化评估(tests/eval/orchestration-eval.test.ts, seeded mulberry32 N=400 5 队列
+ Wilson CI + 真 optimizer 对齐, 独立复跑一致):
  ΔJ_G=0.2975>=0.10  workflow 进化 49.25%->79%(+29.75pp, θ 固定归因 G 非模型)
  ΔJ_held_out=0.3182>=0.03  held-out 板 +31.82pp(泛化)
  同类不同 workflow max 2/class>=2  非平凡进化
  难度估与实际步数 Pearson corr=0.6121>0  估计器有意义
  evolve-off |J_D-J_B|=0<0.05  证改进来自 G 增长非 select 路径
  risk 不增 EXACT 构造铁律 max(evolved risk)=1<=default=1(§4.8)
  risk-increase 负控 242 退化候选全被守卫拒绝
  CI 非重叠 STRONG (J_A_lo=0.7474>J_B_hi=0.5413)
  真 WorkflowOptimizer 与对照 selectWorkflow+thresholds 对齐

M5 量化评估(tests/eval/variant-eval.test.ts, seeded N=200, 注入 checker+runner
+真 manager 对齐):
  ΔJ_selfmod=0.15>=0.05  有益自改 67%->82%(+15pp)
  forbidden 命门 5 拒/0 受 EXACT 构造铁律(§11.3 safety-gate/sandbox/flash/
    glm-provider 永不自改)
  §11.4 门捕有害变体 2>=1(不变式失败被拒)
  回滚触发 1>=1(§11.6 可逆性实证 promoted->rolled-back)
  promoted 有益 3>=1(非零自改) + 全 promoted allPassed
  CI 非重叠 STRONG (J_A_lo=0.7609>J_B_hi=0.7314)
  真 VariantManager 与对照 J_A/B/ΔJ 全等

结论: M4+M5 自进化机制有效且安全(量化/可信/可复现)。M1-M5 五层机制闭环
全部完成, pba 从'静态装配 agent'->'L1-L5 全栈自进化 agent'。诚实边界
(M1-M5.* 跟进): 评估为忠实 mock 策略受控实验证机制因果效应; live-LLM
读注入包真受益的端到端门 + derive* 真接线 + sqlite 持久化 + vm 沙箱 execute
变体工厂 + 真 CI §11.4 不变式执行(真 tsc/单测/launch/砖板率) 为 M1-M5.*
工程项。

验证: tsc 0 / 全量 57 文件 2875 passed 5 skipped / sk- 明文 0 / 真 pi T01
boot 过(barrel 27 扩展含 L1-L5 全栈自进化层)
多 Agent 并行: G(workflow-optimizer)+H(orchestration-bank+eval)+
I(variant-manager)+J(variant-bank+eval)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-17 16:36:50 +08:00

375 lines
14 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { describe, it, expect } from "vitest";
import type {
InvariantChecker,
InvariantReport,
Variant,
} from "../src/shared/types.js";
import {
DEFAULT_AB_THRESHOLD,
createVariantManager,
createVariantRegistry,
classifyProposalVariant,
promotionDecision,
} from "../src/core/variant-manager.js";
import type { ABTestRunner, VariantManager, VariantProposal } from "../src/core/variant-manager.js";
// ---- 测试夹具 ----
/** 构造一条 VariantProposal(默认指向可自改的 pdf-reader 扩展)。 */
function makeProposal(over: Partial<VariantProposal> = {}): VariantProposal {
return {
name: over.name ?? "pdf-reader-tweak",
targetExtension: over.targetExtension ?? "pdf-reader",
description: over.description ?? "tweak pdf parsing logic",
baseRef: over.baseRef ?? "main",
factoryOverride: over.factoryOverride ?? "override-A",
createdAt: over.createdAt ?? 1000,
};
}
/** 全过不变式报告。 */
const ALL_PASS: InvariantReport = {
tests: true,
tsc: true,
noSecrets: true,
modelLaunch: true,
sandboxGate: true,
brickRateZero: true,
riskNonIncrease: true,
allPassed: true,
};
/** 不变式不过报告(tests 失败)。 */
const SOME_FAIL: InvariantReport = {
tests: false,
tsc: true,
noSecrets: true,
modelLaunch: true,
sandboxGate: true,
brickRateZero: true,
riskNonIncrease: true,
allPassed: false,
};
/** 构造一个 fake A/B runner,返回固定结果。 */
function fakeRunner(succVariant: number, succBaseline: number, N = 100): ABTestRunner {
return {
run: (_v: Variant, _n: number) => ({ succVariant, succBaseline, N }),
};
}
/** VariantManager 无 get——用 list().find 检视存储变体。 */
function find(mgr: VariantManager, id: string): Variant | null {
return mgr.list().find((v) => v.id === id) ?? null;
}
describe("VariantManager.propose —— §11.3 白名单预检", () => {
it("可自改扩展(read/probe/analyze/orchestrate)→ candidate", () => {
const mgr = createVariantManager();
// read 类(pdf-reader 默认归 read)
const v1 = mgr.propose(makeProposal({ name: "pdf", targetExtension: "pdf-reader", description: "tweak parse", factoryOverride: "pdf-A" }));
expect(v1.stage).toBe("candidate");
expect(v1.scaffoldClass).toBe("read");
expect(Object.isFrozen(v1)).toBe(true);
// probe 类(env-probe)
const v2 = mgr.propose(makeProposal({ name: "env", targetExtension: "env-probe", description: "probe i2c bus", factoryOverride: "env-A" }));
expect(v2.scaffoldClass).toBe("probe");
// analyze 类(hardware-verify)
const v3 = mgr.propose(makeProposal({ name: "verify", targetExtension: "hardware-verify", description: "verify signal", factoryOverride: "verify-A" }));
expect(v3.scaffoldClass).toBe("analyze");
// orchestrate 类(loop-orchestrator)
const v4 = mgr.propose(makeProposal({ name: "loop", targetExtension: "loop-orchestrator", description: "workflow", factoryOverride: "loop-A" }));
expect(v4.scaffoldClass).toBe("orchestrate");
expect(mgr.size()).toBe(4);
});
it("forbidden 命门(safety-gate/sandbox-policy/flash/glm-provider/reg-tool)→ throws §11.3", () => {
const mgr = createVariantManager();
const forbidden: Array<[string, string]> = [
["safety-gate", "tweak safety gate"],
["sandbox-policy", "relax sandbox"],
["flash", "change flash three-stage"],
["glm-provider", "patch glm provider"],
["reg-tool", "modify register write"],
];
for (const [ext, desc] of forbidden) {
expect(() =>
mgr.propose(makeProposal({ targetExtension: ext, description: desc, name: `${ext}-tweak` })),
).toThrowError(/forbidden scaffold.*§11\.3/);
}
// 全部被拒,注册表为空
expect(mgr.size()).toBe(0);
});
it("变体初建 invariants 全 false + allPassed=false", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
expect(v.invariants.allPassed).toBe(false);
expect(v.invariants.tests).toBe(false);
expect(v.invariants.tsc).toBe(false);
expect(v.invariants.brickRateZero).toBe(false);
});
});
describe("VariantManager.propose —— 幂等(同 name+factoryOverride → 同 id,不重复)", () => {
it("同 name+factoryOverride → 同 id,size 不增", () => {
const mgr = createVariantManager();
const v1 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F" }));
const v2 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F" }));
expect(v1.id).toBe(v2.id);
expect(mgr.size()).toBe(1);
});
it("不同 factoryOverride → 不同 id", () => {
const mgr = createVariantManager();
const v1 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F1" }));
const v2 = mgr.propose(makeProposal({ name: "t", factoryOverride: "F2" }));
expect(v1.id).not.toBe(v2.id);
expect(mgr.size()).toBe(2);
});
});
describe("VariantManager.runInvariants —— 委托注入 checker + 更新存储", () => {
it("默认 checker 全过 → allPassed=true;存储变体 invariants 更新", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
expect(v.invariants.allPassed).toBe(false); // 初建全 false
const report = mgr.runInvariants(v);
expect(report.allPassed).toBe(true);
// 存储副本更新
expect(find(mgr, v.id)?.invariants.allPassed).toBe(true);
});
it("fake checker 返回 some-false → allPassed=false;存储更新", () => {
const fakeChecker: InvariantChecker = {
runChecks: (_v: Variant) => SOME_FAIL,
};
const mgr = createVariantManager(createVariantRegistry(), fakeChecker);
const v = mgr.propose(makeProposal());
const report = mgr.runInvariants(v);
expect(report.allPassed).toBe(false);
expect(report.tests).toBe(false);
expect(find(mgr, v.id)?.invariants.tests).toBe(false);
});
});
describe("VariantManager.abTest —— 委托注入 runner + 更新 shadowResults", () => {
it("runner.run 被调用,shadowResults 更新到存储变体", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
expect(v.shadowResults).toBeUndefined();
const runner = fakeRunner(80, 60, 100);
const result = mgr.abTest(v, runner);
expect(result.succVariant).toBe(80);
expect(result.succBaseline).toBe(60);
expect(result.N).toBe(100);
expect(find(mgr, v.id)?.shadowResults).toEqual({ succVariant: 80, succBaseline: 60, N: 100 });
});
});
describe("VariantManager.tryPromote —— §11.4 + A/B 提升门", () => {
it("不变式全过 + A/B 胜出 → promoted", () => {
const mgr = createVariantManager(); // 默认 checker 全过
const v = mgr.propose(makeProposal());
const updated = mgr.tryPromote(v.id, fakeRunner(80, 50)); // 80 > 50 → 胜出
expect(updated.stage).toBe("promoted");
});
it("不变式不过 → rolled-back(不 promoted)", () => {
const fakeChecker: InvariantChecker = {
runChecks: (_v: Variant) => SOME_FAIL,
};
const mgr = createVariantManager(createVariantRegistry(), fakeChecker);
const v = mgr.propose(makeProposal());
const updated = mgr.tryPromote(v.id, fakeRunner(99, 1)); // A/B 胜出但不变式不过
expect(updated.stage).toBe("rolled-back");
expect(updated.invariants.allPassed).toBe(false);
});
it("不变式全过但 A/B 未胜出 → 留 candidate(不回滚)", () => {
const mgr = createVariantManager(); // 默认全过
const v = mgr.propose(makeProposal());
// 50 <= 60 → 未胜出(rate=50/60≈0.83 <= 1+0)
const updated = mgr.tryPromote(v.id, fakeRunner(50, 60));
expect(updated.stage).toBe("candidate");
expect(updated.shadowResults).toEqual({ succVariant: 50, succBaseline: 60, N: 100 });
});
it("A/B 平局(双方相等)→ 留 candidate(未严格胜出)", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
// 60 === 60 → rate=1 <= 1+0 → 未胜出
const updated = mgr.tryPromote(v.id, fakeRunner(60, 60));
expect(updated.stage).toBe("candidate");
});
it("无 A/B 数据且无 runner → throws §11.4(fail-closed)", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
expect(() => mgr.tryPromote(v.id)).toThrowError(/cannot promote without A\/B data.*§11\.4/);
// 仍 candidate
expect(find(mgr, v.id)?.stage).toBe("candidate");
});
it("已有 shadowResults 时无 runner 也能 promote(用已有数据)", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
// 先采 A/B
mgr.abTest(v, fakeRunner(80, 50));
// 无 runner tryPromote 用已有 shadowResults
const updated = mgr.tryPromote(v.id);
expect(updated.stage).toBe("promoted");
});
it("未知 id → throws", () => {
const mgr = createVariantManager();
expect(() => mgr.tryPromote("nope")).toThrowError(/unknown id nope/);
});
it("A/B 双方都 0 → 无信号 → 留 candidate(不变式全过)", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
// 0 vs 0 → promotionDecision 返回 not promotable "both zero"
const updated = mgr.tryPromote(v.id, fakeRunner(0, 0));
expect(updated.stage).toBe("candidate");
});
});
describe("VariantManager.rollback —— §14.3 可逆性", () => {
it("→ rolled-back;变体留注册表保可追溯", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
mgr.rollback(v.id);
expect(find(mgr, v.id)?.stage).toBe("rolled-back");
expect(mgr.size()).toBe(1); // 不物理删
});
it("未知 id → throws", () => {
const mgr = createVariantManager();
expect(() => mgr.rollback("nope")).toThrowError(/unknown id nope/);
});
});
describe("VariantManager.retire —— 终态", () => {
it("→ retired(终态)", () => {
const mgr = createVariantManager();
const v = mgr.propose(makeProposal());
mgr.retire(v.id);
expect(find(mgr, v.id)?.stage).toBe("retired");
});
it("未知 id → throws", () => {
const mgr = createVariantManager();
expect(() => mgr.retire("nope")).toThrowError(/unknown id nope/);
});
});
describe("VariantManager.resolve —— 只返 promoted", () => {
it("candidate/rolled-back/retired 不返;promoted 返", () => {
const mgr = createVariantManager();
const cand = mgr.propose(makeProposal({ name: "v-cand", factoryOverride: "cand" }));
expect(mgr.resolve("v-cand")).toBeNull(); // candidate
const rb = mgr.propose(makeProposal({ name: "v-rb", factoryOverride: "rb" }));
mgr.rollback(rb.id);
expect(mgr.resolve("v-rb")).toBeNull(); // rolled-back
const rt = mgr.propose(makeProposal({ name: "v-rt", factoryOverride: "rt" }));
mgr.retire(rt.id);
expect(mgr.resolve("v-rt")).toBeNull(); // retired
const promoted = mgr.propose(makeProposal({ name: "v-pro", factoryOverride: "pro" }));
mgr.tryPromote(promoted.id, fakeRunner(80, 50));
const hit = mgr.resolve("v-pro");
expect(hit).not.toBeNull();
expect(hit?.stage).toBe("promoted");
expect(hit?.id).toBe(promoted.id);
});
it("无匹配名 → null", () => {
const mgr = createVariantManager();
expect(mgr.resolve("nonexistent")).toBeNull();
});
});
describe("纯函数 sanity —— classifyProposalVariant / promotionDecision", () => {
it("classifyProposalVariant:forbidden 关键词 → selfModifiable=false", () => {
expect(classifyProposalVariant("safety-gate", "tweak").selfModifiable).toBe(false);
expect(classifyProposalVariant("sandbox-policy", "relax").selfModifiable).toBe(false);
expect(classifyProposalVariant("flash", "change").selfModifiable).toBe(false);
expect(classifyProposalVariant("glm-provider", "patch").selfModifiable).toBe(false);
expect(classifyProposalVariant("reg-tool", "modify").selfModifiable).toBe(false);
// scaffoldClass === "forbidden"
expect(classifyProposalVariant("flash", "").scaffoldClass).toBe("forbidden");
});
it("classifyProposalVariant:可改扩展 → selfModifiable=true", () => {
expect(classifyProposalVariant("pdf-reader", "parse").selfModifiable).toBe(true);
expect(classifyProposalVariant("env-probe", "probe i2c").selfModifiable).toBe(true);
expect(classifyProposalVariant("hardware-verify", "verify").selfModifiable).toBe(true);
expect(classifyProposalVariant("loop-orchestrator", "workflow").selfModifiable).toBe(true);
});
it("promotionDecision:不变式不过 → not promotable", () => {
const d = promotionDecision(SOME_FAIL, { succVariant: 99, succBaseline: 1, N: 100 }, DEFAULT_AB_THRESHOLD);
expect(d.promotable).toBe(false);
expect(d.reason).toMatch(/invariants/);
});
it("promotionDecision:A/B 平局(rate=1)→ not promotable", () => {
const d = promotionDecision(ALL_PASS, { succVariant: 60, succBaseline: 60, N: 100 }, DEFAULT_AB_THRESHOLD);
expect(d.promotable).toBe(false);
expect(d.reason).toMatch(/not winning/);
});
it("promotionDecision:双方都 0 → not promotable(no signal)", () => {
const d = promotionDecision(ALL_PASS, { succVariant: 0, succBaseline: 0, N: 100 }, DEFAULT_AB_THRESHOLD);
expect(d.promotable).toBe(false);
expect(d.reason).toMatch(/both zero/);
});
it("promotionDecision:不变式全过 + A/B 胜出 → promotable", () => {
const d = promotionDecision(ALL_PASS, { succVariant: 80, succBaseline: 50, N: 100 }, DEFAULT_AB_THRESHOLD);
expect(d.promotable).toBe(true);
});
it("promotionDecision:基线 0 但变体 >0 → promotable(Infinity rate)", () => {
const d = promotionDecision(ALL_PASS, { succVariant: 5, succBaseline: 0, N: 100 }, DEFAULT_AB_THRESHOLD);
expect(d.promotable).toBe(true);
});
});
describe("createVariantManager / createVariantRegistry 工厂", () => {
it("createVariantManager() 默认参数 → 可用 VariantManager", () => {
const mgr = createVariantManager();
expect(mgr.size()).toBe(0);
const v = mgr.propose(makeProposal());
expect(mgr.size()).toBe(1);
expect(Object.isFrozen(v)).toBe(true);
});
it("可注入自定义 registry + checker", () => {
const reg = createVariantRegistry();
const checker: InvariantChecker = {
runChecks: (_v: Variant) => ALL_PASS,
};
const mgr = createVariantManager(reg, checker);
const v = mgr.propose(makeProposal());
expect(reg.get(v.id)).not.toBeNull();
const report = mgr.runInvariants(v);
expect(report.allPassed).toBe(true);
});
});
describe("VariantManager.list/size/clear", () => {
it("委托 registry", () => {
const mgr = createVariantManager();
mgr.propose(makeProposal({ name: "a", factoryOverride: "a" }));
mgr.propose(makeProposal({ name: "b", factoryOverride: "b", targetExtension: "env-probe", description: "probe" }));
expect(mgr.size()).toBe(2);
expect(mgr.list().length).toBe(2);
mgr.clear();
expect(mgr.size()).toBe(0);
});
});