跨层共享项落地(设计文档 §19.4 X.1/X.3/X.4): X.1 时序三档接线 + X.3 v_t 全链路贯通: - shared/types.ts: TaskFeedback/ToolOutcome/EvolutionMetrics/MetricsSources + hitRate/avgInfoGain/avgSkillQuality/maxWorkflowRisk/evolutionBrickRate 纯函数 - core/feedback-collector.ts: FeedbackCollector(recordPrompt/recordBoardFingerprint/ recordToolResult/finalize/lastFinalized/pending/clear) + derive* 纯变换 (deriveTaskSignature/deriveDistillInput/deriveSkillExtractInput/deriveSkillQuery/ derivePriorCorrection/deriveProposals + inferOutcome/inferActionClass/ inferValidatedBy/isIrreversibleTool) + getSharedCollector/initSharedCollector 共享单例 - extensions/feedback-bank.ts: hooks tool_result+message_end+before_agent_start → 采集 → finalize → session_before_compact 喂 derive* + feedback_recall 工具 (仿 orchestration-bank/skill-bank 范式) - **4 bank 默认 derive* 读共享 collector lastFinalized 生产自驱** (experience/skill/orchestration/variant bank) — X.1+X.3 生产闭环 X.4 进化可观测面板: - core/evolution-metrics.ts: aggregateMetrics/renderMetrics 纯函数 (5 层注册表聚合 → EvolutionMetrics:规模/命中率/infoGain/risk/砖板率/forbidden 拒绝) - bin/evolution-metrics.ts CLI(brickRate!=0 时 exit 1 fail-closed 自检) X.1+X.3 全链路评估(tests/eval/feedback-chain-eval.test.ts, 合成 session 轨迹→ feedback-bank hooks→collector→derive*→真 5 层 manager→断 v_t 全链路 end-to-end): §4.6 分级正确率 8/8=100% (success+ground-truth/可逆/不可逆/partial/empty/mixed) 同签名检索命中 100% + 注入块可见 100% 5 链 end-to-end (skill/workflow/variant/tool-growth) forbidden §11.3/§4.8 拒绝 (命门 safety-gate/flash 工具) X.4 评估(tests/eval/evolution-metrics-eval.test.ts, 合成 5 层注册表→精确指标): experienceHitRate=0.600/avgInfoGain=0.650/maxRisk=1 精确 brickRate=0 EXACT 构造铁律 forbiddenRejected=7 精确 结论: X.1(4/5 时序钩子全接, stage-wise θ=L5.*) + X.3(v_t 全链路贯通: tool_result 采集→derive* 蒸馏→5 层入库→检索→注入→约束, §4.6 100% + 5 链 end-to-end + 4 bank 生产自驱) + X.4(可观测面板精确指标, brickRate=0 构造铁律) 全部完成。诚实边界 (M1-M5.* 跟进): live-LLM 读注入包真受益端到端门 + stage-wise 真 θ 更新 + 真 git worktree 变体 + 真 CI §11.4 执行 = M1-M5.* 工程项(机制层全闭环+量化坐实)。 验证: tsc 0 / 全量 62 文件 3018 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot 过(barrel 28 扩展含 L1-L5 + 跨层反馈采集) 多 Agent 并行: K(feedback-collector+derive*)+L(feedback-bank+chain eval)+M(evolution-metrics) Co-Authored-By: Claude <noreply@anthropic.com>
127 lines
9.0 KiB
JavaScript
127 lines
9.0 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* @file bin/evolution-metrics.ts
|
||
* @module bin/evolution-metrics
|
||
*
|
||
* X.4 进化可观测面板 CLI(设计文档 §13 证据 / §16 评估 / §19.4 X.4)。
|
||
*
|
||
* 用法:
|
||
* npx tsx bin/evolution-metrics.ts [metrics-sources.json]
|
||
*
|
||
* - 不带参数:用内置确定性合成 5 层注册表快照演示面板输出。
|
||
* - 带参数:读 JSON 文件(序列化 MetricsSources),聚合 + 渲染面板。
|
||
*
|
||
* 生产接线(X.* 跟进):真实 5 层管理器是 in-memory 单例,进程内聚合;
|
||
* 本 CLI 演示纯函数聚合 + 渲染。生产可由 session/registry 钩子落
|
||
* MetricsSources 快照到 JSON,再用本 CLI 离线查看;或直接调
|
||
* aggregateMetrics(renderMetrics) 渲染实时快照。不读真实单例。
|
||
*/
|
||
|
||
import { readFileSync } from "node:fs";
|
||
import { resolve } from "node:path";
|
||
import type {
|
||
ExperienceEntry,
|
||
GrownTool,
|
||
HardwareSkill,
|
||
MetricsSources,
|
||
Variant,
|
||
WorkflowGraph,
|
||
} from "../src/shared/types.js";
|
||
import {
|
||
aggregateMetrics,
|
||
renderMetrics,
|
||
} from "../src/core/evolution-metrics.js";
|
||
|
||
// ============================================================================
|
||
// 内置确定性合成 5 层注册表快照(演示用;固定数字,无 RNG)
|
||
// ============================================================================
|
||
|
||
const beta = { soc: "rdk-x5", kernel: "6.1.83", image: "v1", mac: "AA:01" };
|
||
const sig = { actionClass: "flash", target: "bootloader", constraints: ["aarch64"] };
|
||
|
||
/** 5 经验,hitCounts [3,0,2,1,0] → hitRate = 0.6 */
|
||
const DEMO_EXPERIENCES: ExperienceEntry[] = [
|
||
{ id: "e0", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "dd if=img of=/dev/mmcblk0", evidenceRef: "s1", contextTags: ["flash"], validity: "repeatable", usage: "actionable", createdAt: 0, hitCount: 3, validatedBy: "physical-ground-truth" },
|
||
{ id: "e1", boardFingerprint: beta, taskSignature: sig, outcome: "failure", actionable: "n/a", antiPattern: "don't skip verify", evidenceRef: "s2", contextTags: ["flash"], validity: "irreversible-fail", usage: "anti-pattern", createdAt: 1, hitCount: 0, validatedBy: "physical-ground-truth" },
|
||
{ id: "e2", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "i2cdetect -y 0", evidenceRef: "s3", contextTags: ["i2c"], validity: "repeatable", usage: "actionable", createdAt: 2, hitCount: 2, validatedBy: "executable-stdout" },
|
||
{ id: "e3", boardFingerprint: beta, taskSignature: sig, outcome: "partial", actionable: "probe pmic", evidenceRef: "s4", contextTags: ["i2c"], validity: "once", usage: "hint", createdAt: 3, hitCount: 1, validatedBy: "executable-stdout" },
|
||
{ id: "e4", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "read dmesg", evidenceRef: "s5", contextTags: ["dmesg"], validity: "inferred", usage: "hint", createdAt: 4, hitCount: 0, validatedBy: "model-claim" },
|
||
];
|
||
|
||
/** 4 技能:avgInfoGain=0.65, avgQualityScore=0.75, 3 active */
|
||
const DEMO_SKILLS: HardwareSkill[] = [
|
||
{ id: "sk0", name: "i2c-recover", trigger: { kind: "event", pattern: "i2c.*no such device", actionClass: "i2c-probe" }, procedure: [{ tool: "i2cdetect", argsTemplate: "0x3a" }], granularity: "event-driven", provenance: ["s1"], quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.8, createdAt: 0, hitCount: 4, active: true },
|
||
{ id: "sk1", name: "reg-read", trigger: { kind: "state", statePredicate: "reg locked", actionClass: "reg-read" }, procedure: [{ tool: "devmem", argsTemplate: "0x40020000" }], granularity: "event-driven", provenance: ["s2"], quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.6, createdAt: 1, hitCount: 2, active: true },
|
||
{ id: "sk2", name: "old-probe", trigger: { kind: "event", pattern: "timeout", actionClass: "probe" }, procedure: [{ tool: "scan", argsTemplate: "" }], granularity: "task-level", provenance: ["s3"], quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 }, infoGain: 0.9, createdAt: 2, hitCount: 0, active: false, supersededBy: "sk0" },
|
||
{ id: "sk3", name: "dt-parse", trigger: { kind: "event", pattern: "dtb error", actionClass: "device-tree" }, procedure: [{ tool: "fdtdump", argsTemplate: "" }], granularity: "event-driven", provenance: ["s4"], quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.3, createdAt: 3, hitCount: 1, active: true },
|
||
];
|
||
|
||
/** 3 workflow,risks [1,0,1] → maxRisk=1, 2 active */
|
||
const DEMO_WORKFLOWS: WorkflowGraph[] = [
|
||
{ id: "w0", taskClass: "flash", nodes: [{ id: "n0", tool: "verify", argsTemplate: "", irreversibility: "irreversible" }], edges: [], risk: 1, provenance: ["s1"], createdAt: 0, hitCount: 3, active: true },
|
||
{ id: "w1", taskClass: "i2c", nodes: [{ id: "n1", tool: "i2cdetect", argsTemplate: "", irreversibility: "reversible" }], edges: [], risk: 0, provenance: ["s3"], createdAt: 1, hitCount: 0, active: false, supersededBy: "w2" },
|
||
{ id: "w2", taskClass: "i2c", nodes: [{ id: "n2", tool: "i2cget", argsTemplate: "", irreversibility: "irreversible" }], edges: [], risk: 1, provenance: ["s3"], createdAt: 2, hitCount: 2, active: true },
|
||
];
|
||
|
||
/** 5 grownTools,2 promoted */
|
||
const DEMO_GROWN_TOOLS: GrownTool[] = [
|
||
{ id: "g0", name: "dump-reg", toolClass: "read", definitionHash: "h0", stage: "promoted", usageCount: 10, passCount: 9, validity: "repeatable", createdAt: 0, lastUsed: 5 },
|
||
{ id: "g1", name: "scan-i2c", toolClass: "probe", definitionHash: "h1", stage: "promoted", usageCount: 12, passCount: 11, validity: "repeatable", createdAt: 1, lastUsed: 6 },
|
||
{ id: "g2", name: "parse-dtb", toolClass: "analyze", definitionHash: "h2", stage: "verified", usageCount: 4, passCount: 3, validity: "repeatable", createdAt: 2, lastUsed: 3 },
|
||
{ id: "g3", name: "probe-gpio", toolClass: "probe", definitionHash: "h3", stage: "draft", usageCount: 1, passCount: 1, validity: "once", createdAt: 3, lastUsed: 1 },
|
||
{ id: "g4", name: "old-tool", toolClass: "read", definitionHash: "h4", stage: "retired", usageCount: 20, passCount: 15, validity: "repeatable", createdAt: 4, lastUsed: 0, supersededBy: "g0" },
|
||
];
|
||
|
||
/** 4 variants,1 promoted */
|
||
const DEMO_VARIANTS: Variant[] = [
|
||
{ id: "v0", name: "faster-probe", targetExtension: "env-probe", scaffoldClass: "probe", baseRef: "HEAD", factoryOverride: "o0", stage: "promoted", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true }, createdAt: 0, shadowResults: { succVariant: 15, succBaseline: 10, N: 20 } },
|
||
{ id: "v1", name: "better-dt", targetExtension: "device-tree", scaffoldClass: "analyze", baseRef: "HEAD", factoryOverride: "o1", stage: "candidate", invariants: { tests: false, tsc: false, noSecrets: true, modelLaunch: false, sandboxGate: true, brickRateZero: false, riskNonIncrease: true, allPassed: false }, createdAt: 1 },
|
||
{ id: "v2", name: "risky-build", targetExtension: "build-tracker", scaffoldClass: "analyze", baseRef: "HEAD", factoryOverride: "o2", stage: "rolled-back", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: false, allPassed: false }, createdAt: 2 },
|
||
{ id: "v3", name: "old-variant", targetExtension: "pdf-reader", scaffoldClass: "read", baseRef: "HEAD", factoryOverride: "o3", stage: "retired", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true }, createdAt: 3, supersededBy: "v0" },
|
||
];
|
||
|
||
const DEMO_SOURCES: MetricsSources = {
|
||
experiences: DEMO_EXPERIENCES,
|
||
skills: DEMO_SKILLS,
|
||
workflows: DEMO_WORKFLOWS,
|
||
grownTools: DEMO_GROWN_TOOLS,
|
||
variants: DEMO_VARIANTS,
|
||
forbiddenRejected: 7,
|
||
};
|
||
|
||
// ============================================================================
|
||
// CLI 入口
|
||
// ============================================================================
|
||
|
||
function loadSourcesFromJson(path: string): MetricsSources {
|
||
const raw = readFileSync(resolve(path), "utf8");
|
||
const parsed = JSON.parse(raw) as MetricsSources;
|
||
// fail-closed:基本结构校验(5 层数组必须存在)
|
||
if (!parsed || !Array.isArray(parsed.experiences) || !Array.isArray(parsed.skills)
|
||
|| !Array.isArray(parsed.workflows) || !Array.isArray(parsed.grownTools)
|
||
|| !Array.isArray(parsed.variants)) {
|
||
throw new Error(`evolution-metrics: JSON 文件 ${path} 缺 5 层数组之一(MetricsSources 结构不完整)`);
|
||
}
|
||
return parsed;
|
||
}
|
||
|
||
function main(): void {
|
||
const arg = process.argv[2];
|
||
const sources = arg ? loadSourcesFromJson(arg) : DEMO_SOURCES;
|
||
const ts = Number(process.env.PBA_METRICS_TS ?? 0);
|
||
const m = aggregateMetrics(sources, ts);
|
||
const out = renderMetrics(m);
|
||
// stdout 仅渲染表,stderr 打元信息(便于管道)
|
||
const origin = arg ? `file:${arg}` : "demo-synthetic";
|
||
process.stderr.write(`[evolution-metrics] sources=${origin} ts=${ts}\n`);
|
||
process.stdout.write(out + "\n");
|
||
|
||
// 构造不变式自检(fail-closed):brickRate 必须为 0
|
||
if (m.brickRate !== 0) {
|
||
process.stderr.write(`[evolution-metrics] CRITICAL: brickRate=${m.brickRate} 非零(构造不变式破坏,§4.8/§11.4)\n`);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
main();
|