physical_byte_agent/bin/evolution-metrics.ts
d-robotics 74f4cd428e
Some checks are pending
CI / quality-gate (push) Waiting to run
CI / type-drift (push) Waiting to run
feat(跨层 X.1+X.3+X.4): 反馈采集全链路 + 可观测面板 + 量化坐实 (M1-M5+X 全闭环)
跨层共享项落地(设计文档 §19.4 X.1/X.3/X.4):

X.1 时序三档接线 + X.3 v_t 全链路贯通:
- shared/types.ts: TaskFeedback/ToolOutcome/EvolutionMetrics/MetricsSources
  + hitRate/avgInfoGain/avgSkillQuality/maxWorkflowRisk/evolutionBrickRate 纯函数
- core/feedback-collector.ts: FeedbackCollector(recordPrompt/recordBoardFingerprint/
  recordToolResult/finalize/lastFinalized/pending/clear) + derive* 纯变换
  (deriveTaskSignature/deriveDistillInput/deriveSkillExtractInput/deriveSkillQuery/
  derivePriorCorrection/deriveProposals + inferOutcome/inferActionClass/
  inferValidatedBy/isIrreversibleTool) + getSharedCollector/initSharedCollector 共享单例
- extensions/feedback-bank.ts: hooks tool_result+message_end+before_agent_start
  → 采集 → finalize → session_before_compact 喂 derive* + feedback_recall 工具
  (仿 orchestration-bank/skill-bank 范式)
- **4 bank 默认 derive* 读共享 collector lastFinalized 生产自驱**
  (experience/skill/orchestration/variant bank) — X.1+X.3 生产闭环

X.4 进化可观测面板:
- core/evolution-metrics.ts: aggregateMetrics/renderMetrics 纯函数
  (5 层注册表聚合 → EvolutionMetrics:规模/命中率/infoGain/risk/砖板率/forbidden 拒绝)
- bin/evolution-metrics.ts CLI(brickRate!=0 时 exit 1 fail-closed 自检)

X.1+X.3 全链路评估(tests/eval/feedback-chain-eval.test.ts, 合成 session 轨迹→
feedback-bank hooks→collector→derive*→真 5 层 manager→断 v_t 全链路 end-to-end):
  §4.6 分级正确率 8/8=100% (success+ground-truth/可逆/不可逆/partial/empty/mixed)
  同签名检索命中 100% + 注入块可见 100%
  5 链 end-to-end (skill/workflow/variant/tool-growth)
  forbidden §11.3/§4.8 拒绝 (命门 safety-gate/flash 工具)

X.4 评估(tests/eval/evolution-metrics-eval.test.ts, 合成 5 层注册表→精确指标):
  experienceHitRate=0.600/avgInfoGain=0.650/maxRisk=1 精确
  brickRate=0 EXACT 构造铁律 forbiddenRejected=7 精确

结论: X.1(4/5 时序钩子全接, stage-wise θ=L5.*) + X.3(v_t 全链路贯通: tool_result
采集→derive* 蒸馏→5 层入库→检索→注入→约束, §4.6 100% + 5 链 end-to-end + 4 bank
生产自驱) + X.4(可观测面板精确指标, brickRate=0 构造铁律) 全部完成。诚实边界
(M1-M5.* 跟进): live-LLM 读注入包真受益端到端门 + stage-wise 真 θ 更新 + 真 git
worktree 变体 + 真 CI §11.4 执行 = M1-M5.* 工程项(机制层全闭环+量化坐实)。

验证: tsc 0 / 全量 62 文件 3018 passed 5 skipped / sk- 明文 0 / 真 pi T01 boot
过(barrel 28 扩展含 L1-L5 + 跨层反馈采集)
多 Agent 并行: K(feedback-collector+derive*)+L(feedback-bank+chain eval)+M(evolution-metrics)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-17 17:32:37 +08:00

127 lines
9.0 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env node
/**
* @file bin/evolution-metrics.ts
* @module bin/evolution-metrics
*
* X.4 进化可观测面板 CLI(设计文档 §13 证据 / §16 评估 / §19.4 X.4)。
*
* 用法:
* npx tsx bin/evolution-metrics.ts [metrics-sources.json]
*
* - 不带参数:用内置确定性合成 5 层注册表快照演示面板输出。
* - 带参数:读 JSON 文件(序列化 MetricsSources),聚合 + 渲染面板。
*
* 生产接线(X.* 跟进):真实 5 层管理器是 in-memory 单例,进程内聚合;
* 本 CLI 演示纯函数聚合 + 渲染。生产可由 session/registry 钩子落
* MetricsSources 快照到 JSON,再用本 CLI 离线查看;或直接调
* aggregateMetrics(renderMetrics) 渲染实时快照。不读真实单例。
*/
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import type {
ExperienceEntry,
GrownTool,
HardwareSkill,
MetricsSources,
Variant,
WorkflowGraph,
} from "../src/shared/types.js";
import {
aggregateMetrics,
renderMetrics,
} from "../src/core/evolution-metrics.js";
// ============================================================================
// 内置确定性合成 5 层注册表快照(演示用;固定数字,无 RNG)
// ============================================================================
const beta = { soc: "rdk-x5", kernel: "6.1.83", image: "v1", mac: "AA:01" };
const sig = { actionClass: "flash", target: "bootloader", constraints: ["aarch64"] };
/** 5 经验,hitCounts [3,0,2,1,0] → hitRate = 0.6 */
const DEMO_EXPERIENCES: ExperienceEntry[] = [
{ id: "e0", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "dd if=img of=/dev/mmcblk0", evidenceRef: "s1", contextTags: ["flash"], validity: "repeatable", usage: "actionable", createdAt: 0, hitCount: 3, validatedBy: "physical-ground-truth" },
{ id: "e1", boardFingerprint: beta, taskSignature: sig, outcome: "failure", actionable: "n/a", antiPattern: "don't skip verify", evidenceRef: "s2", contextTags: ["flash"], validity: "irreversible-fail", usage: "anti-pattern", createdAt: 1, hitCount: 0, validatedBy: "physical-ground-truth" },
{ id: "e2", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "i2cdetect -y 0", evidenceRef: "s3", contextTags: ["i2c"], validity: "repeatable", usage: "actionable", createdAt: 2, hitCount: 2, validatedBy: "executable-stdout" },
{ id: "e3", boardFingerprint: beta, taskSignature: sig, outcome: "partial", actionable: "probe pmic", evidenceRef: "s4", contextTags: ["i2c"], validity: "once", usage: "hint", createdAt: 3, hitCount: 1, validatedBy: "executable-stdout" },
{ id: "e4", boardFingerprint: beta, taskSignature: sig, outcome: "success", actionable: "read dmesg", evidenceRef: "s5", contextTags: ["dmesg"], validity: "inferred", usage: "hint", createdAt: 4, hitCount: 0, validatedBy: "model-claim" },
];
/** 4 技能:avgInfoGain=0.65, avgQualityScore=0.75, 3 active */
const DEMO_SKILLS: HardwareSkill[] = [
{ id: "sk0", name: "i2c-recover", trigger: { kind: "event", pattern: "i2c.*no such device", actionClass: "i2c-probe" }, procedure: [{ tool: "i2cdetect", argsTemplate: "0x3a" }], granularity: "event-driven", provenance: ["s1"], quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.8, createdAt: 0, hitCount: 4, active: true },
{ id: "sk1", name: "reg-read", trigger: { kind: "state", statePredicate: "reg locked", actionClass: "reg-read" }, procedure: [{ tool: "devmem", argsTemplate: "0x40020000" }], granularity: "event-driven", provenance: ["s2"], quality: { rubricScore: 0.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.6, createdAt: 1, hitCount: 2, active: true },
{ id: "sk2", name: "old-probe", trigger: { kind: "event", pattern: "timeout", actionClass: "probe" }, procedure: [{ tool: "scan", argsTemplate: "" }], granularity: "task-level", provenance: ["s3"], quality: { rubricScore: 1.0, execPassRate: 0.0, execSamples: 5 }, infoGain: 0.9, createdAt: 2, hitCount: 0, active: false, supersededBy: "sk0" },
{ id: "sk3", name: "dt-parse", trigger: { kind: "event", pattern: "dtb error", actionClass: "device-tree" }, procedure: [{ tool: "fdtdump", argsTemplate: "" }], granularity: "event-driven", provenance: ["s4"], quality: { rubricScore: 1.0, execPassRate: 1.0, execSamples: 5 }, infoGain: 0.3, createdAt: 3, hitCount: 1, active: true },
];
/** 3 workflow,risks [1,0,1] → maxRisk=1, 2 active */
const DEMO_WORKFLOWS: WorkflowGraph[] = [
{ id: "w0", taskClass: "flash", nodes: [{ id: "n0", tool: "verify", argsTemplate: "", irreversibility: "irreversible" }], edges: [], risk: 1, provenance: ["s1"], createdAt: 0, hitCount: 3, active: true },
{ id: "w1", taskClass: "i2c", nodes: [{ id: "n1", tool: "i2cdetect", argsTemplate: "", irreversibility: "reversible" }], edges: [], risk: 0, provenance: ["s3"], createdAt: 1, hitCount: 0, active: false, supersededBy: "w2" },
{ id: "w2", taskClass: "i2c", nodes: [{ id: "n2", tool: "i2cget", argsTemplate: "", irreversibility: "irreversible" }], edges: [], risk: 1, provenance: ["s3"], createdAt: 2, hitCount: 2, active: true },
];
/** 5 grownTools,2 promoted */
const DEMO_GROWN_TOOLS: GrownTool[] = [
{ id: "g0", name: "dump-reg", toolClass: "read", definitionHash: "h0", stage: "promoted", usageCount: 10, passCount: 9, validity: "repeatable", createdAt: 0, lastUsed: 5 },
{ id: "g1", name: "scan-i2c", toolClass: "probe", definitionHash: "h1", stage: "promoted", usageCount: 12, passCount: 11, validity: "repeatable", createdAt: 1, lastUsed: 6 },
{ id: "g2", name: "parse-dtb", toolClass: "analyze", definitionHash: "h2", stage: "verified", usageCount: 4, passCount: 3, validity: "repeatable", createdAt: 2, lastUsed: 3 },
{ id: "g3", name: "probe-gpio", toolClass: "probe", definitionHash: "h3", stage: "draft", usageCount: 1, passCount: 1, validity: "once", createdAt: 3, lastUsed: 1 },
{ id: "g4", name: "old-tool", toolClass: "read", definitionHash: "h4", stage: "retired", usageCount: 20, passCount: 15, validity: "repeatable", createdAt: 4, lastUsed: 0, supersededBy: "g0" },
];
/** 4 variants,1 promoted */
const DEMO_VARIANTS: Variant[] = [
{ id: "v0", name: "faster-probe", targetExtension: "env-probe", scaffoldClass: "probe", baseRef: "HEAD", factoryOverride: "o0", stage: "promoted", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true }, createdAt: 0, shadowResults: { succVariant: 15, succBaseline: 10, N: 20 } },
{ id: "v1", name: "better-dt", targetExtension: "device-tree", scaffoldClass: "analyze", baseRef: "HEAD", factoryOverride: "o1", stage: "candidate", invariants: { tests: false, tsc: false, noSecrets: true, modelLaunch: false, sandboxGate: true, brickRateZero: false, riskNonIncrease: true, allPassed: false }, createdAt: 1 },
{ id: "v2", name: "risky-build", targetExtension: "build-tracker", scaffoldClass: "analyze", baseRef: "HEAD", factoryOverride: "o2", stage: "rolled-back", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: false, allPassed: false }, createdAt: 2 },
{ id: "v3", name: "old-variant", targetExtension: "pdf-reader", scaffoldClass: "read", baseRef: "HEAD", factoryOverride: "o3", stage: "retired", invariants: { tests: true, tsc: true, noSecrets: true, modelLaunch: true, sandboxGate: true, brickRateZero: true, riskNonIncrease: true, allPassed: true }, createdAt: 3, supersededBy: "v0" },
];
const DEMO_SOURCES: MetricsSources = {
experiences: DEMO_EXPERIENCES,
skills: DEMO_SKILLS,
workflows: DEMO_WORKFLOWS,
grownTools: DEMO_GROWN_TOOLS,
variants: DEMO_VARIANTS,
forbiddenRejected: 7,
};
// ============================================================================
// CLI 入口
// ============================================================================
function loadSourcesFromJson(path: string): MetricsSources {
const raw = readFileSync(resolve(path), "utf8");
const parsed = JSON.parse(raw) as MetricsSources;
// fail-closed:基本结构校验(5 层数组必须存在)
if (!parsed || !Array.isArray(parsed.experiences) || !Array.isArray(parsed.skills)
|| !Array.isArray(parsed.workflows) || !Array.isArray(parsed.grownTools)
|| !Array.isArray(parsed.variants)) {
throw new Error(`evolution-metrics: JSON 文件 ${path} 缺 5 层数组之一(MetricsSources 结构不完整)`);
}
return parsed;
}
function main(): void {
const arg = process.argv[2];
const sources = arg ? loadSourcesFromJson(arg) : DEMO_SOURCES;
const ts = Number(process.env.PBA_METRICS_TS ?? 0);
const m = aggregateMetrics(sources, ts);
const out = renderMetrics(m);
// stdout 仅渲染表,stderr 打元信息(便于管道)
const origin = arg ? `file:${arg}` : "demo-synthetic";
process.stderr.write(`[evolution-metrics] sources=${origin} ts=${ts}\n`);
process.stdout.write(out + "\n");
// 构造不变式自检(fail-closed):brickRate 必须为 0
if (m.brickRate !== 0) {
process.stderr.write(`[evolution-metrics] CRITICAL: brickRate=${m.brickRate} 非零(构造不变式破坏,§4.8/§11.4)\n`);
process.exit(1);
}
}
main();