2026-10-09 16:41:13 +08:00

175 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""主通道维度对比: fp (torch float) vs 量化目标 (伪量化 calib / 板端 hbm), 逐 action 维度。
主通道 = 模型主输出通道 (action 张量 [action_horizon, action_dim], 有效维 valid_action_dim, 其余为 padding 维)。
输入:
OUTPUT_DIR 必填; 需含 eval_dump/{stage}/torch/{sid}.npy 与 eval_dump/{stage}/{stage}/{sid}.npy
(torch_eval 阶段 vla_eval 产物, unified_dumper 逐 chunk dump)
SAMPLE_INDEX 选样参数: "0" (默认) / 逗号列表 "0,3,7" / "all"
STAGES 对比对象, 默认 "calib" (伪量化); M2 后可加 "hbm"
VALID_ACTION_DIM 有效动作维 (默认 14, 契约 state.dim; >= 该维的为 padding 维单独标注)
产物 (OUTPUT_DIR/compare/):
sample_{sid}/per_dim.csv dim, kind(valid/padding), cosine, mae, rmse, max_abs
sample_{sid}/per_dim.png 逐维 cosine 柱状 + 逐时间步误差曲线
sample_{sid}/summary.json 整体指标 + PASS 判定
summary_all.json 全部取样汇总
PASS 初值: 整体 tensor cosine >= 0.98 且 逐维 cosine 中位数 >= 0.95 (quant_design_v1 §8)
"""
from __future__ import annotations
import csv
import json
import logging
import os
import sys
from pathlib import Path
import numpy as np
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger("sample_compare")
PASS_TENSOR_COS = 0.98
PASS_DIM_MEDIAN_COS = 0.95
def cosine(a: np.ndarray, b: np.ndarray) -> float:
d = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-12))
return d
def compare_pair(fp: np.ndarray, qt: np.ndarray, valid_dim: int) -> dict:
if fp.ndim == 3 and fp.shape[0] == 1: # dump 带 batch 维 [1, horizon, dim]
fp = fp[0]
if qt.ndim == 3 and qt.shape[0] == 1:
qt = qt[0]
h, dim = fp.shape
err = qt - fp
per_dim = []
for d in range(dim):
per_dim.append({
"dim": d,
"kind": "valid" if d < valid_dim else "padding",
"cosine": round(cosine(fp[:, d], qt[:, d]), 6),
"mae": round(float(np.abs(err[:, d]).mean()), 6),
"rmse": round(float(np.sqrt((err[:, d] ** 2).mean())), 6),
"max_abs": round(float(np.abs(err[:, d]).max()), 6),
})
valid_cos = [p["cosine"] for p in per_dim if p["kind"] == "valid"]
return {
"shape": [h, dim],
"tensor": {
"cosine": round(cosine(fp.reshape(-1), qt.reshape(-1)), 6),
"mae": round(float(np.abs(err).mean()), 6),
"rmse": round(float(np.sqrt((err ** 2).mean())), 6),
"max_abs": round(float(np.abs(err).max()), 6),
},
"valid_dim_median_cosine": round(float(np.median(valid_cos)), 6),
"valid_dim_min_cosine": round(float(np.min(valid_cos)), 6),
"per_step_mae": np.abs(err).mean(axis=1).round(6).tolist(),
"per_dim": per_dim,
}
def plot_sample(out_dir: Path, res: dict) -> None:
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError:
return
per_dim = res["per_dim"]
fig, axes = plt.subplots(1, 2, figsize=(12, 4))
dims = [p["dim"] for p in per_dim]
cos = [p["cosine"] for p in per_dim]
colors = ["tab:blue" if p["kind"] == "valid" else "tab:gray" for p in per_dim]
axes[0].bar(dims, cos, color=colors)
axes[0].axhline(PASS_DIM_MEDIAN_COS, ls="--", c="r", lw=1)
axes[0].set_xlabel("action dim"); axes[0].set_ylabel("cosine"); axes[0].set_ylim(min(0.0, min(cos) - 0.05), 1.05)
axes[0].set_title("per-dim cosine (gray=padding)")
steps = np.arange(len(res["per_step_mae"]))
axes[1].plot(steps, res["per_step_mae"])
axes[1].set_xlabel("horizon step"); axes[1].set_title("per-step MAE")
fig.tight_layout()
fig.savefig(out_dir / "per_dim.png", dpi=120)
def main() -> None:
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
if not output_dir:
sys.exit("[hyperparam] OUTPUT_DIR not set")
stages = [s.strip() for s in os.environ.get("STAGES", "calib").split(",") if s.strip()]
valid_dim = int(os.environ.get("VALID_ACTION_DIM", "14"))
sel = os.environ.get("SAMPLE_INDEX", "0").strip()
dump_root = output_dir / "eval_dump"
if not dump_root.is_dir():
sys.exit(f"[input] 无 eval_dump: {dump_root} (先跑 run_quant STAGES=eval)")
# 可用样本 = 任一 stage torch 目录下的 {sid}.npy
torch_dir0 = dump_root / stages[0] / "torch"
sids_all = sorted(p.stem for p in torch_dir0.glob("*.npy"))
if not sids_all:
sys.exit(f"[input] {torch_dir0} 无 npy")
if sel == "all":
sids = sids_all
else:
want = [int(x) for x in sel.split(",") if x.strip() != ""]
bad = [w for w in want if str(w) not in sids_all]
if bad:
sys.exit(f"[input] 样本不存在: {bad} (可用: {sids_all[:8]}...)")
sids = [str(w) for w in want]
logger.info("[compare] samples=%s stages=%s valid_dim=%d", sids, stages, valid_dim)
out_root = output_dir / "compare"
out_root.mkdir(parents=True, exist_ok=True)
all_summary = {"pass_thresholds": {"tensor_cosine": PASS_TENSOR_COS, "valid_dim_median_cosine": PASS_DIM_MEDIAN_COS},
"valid_action_dim": valid_dim, "stages": stages, "samples": {}}
for sid in sids:
fp_path = dump_root / stages[0] / "torch" / f"{sid}.npy"
fp = np.load(fp_path)
sample_out = out_root / f"sample_{sid}"
sample_out.mkdir(exist_ok=True)
res = {"fp_source": str(fp_path), "targets": {}}
for stage in stages:
qt_path = dump_root / stage / stage / f"{sid}.npy"
if not qt_path.is_file():
sys.exit(f"[input] 缺 {qt_path}")
qt = np.load(qt_path)
if qt.shape != fp.shape:
sys.exit(f"[input] 形状不一致 fp{fp.shape} vs {stage}{qt.shape}")
res["targets"][stage] = compare_pair(fp.astype(np.float64), qt.astype(np.float64), valid_dim)
verdict = all(
t["tensor"]["cosine"] >= PASS_TENSOR_COS and t["valid_dim_median_cosine"] >= PASS_DIM_MEDIAN_COS
for t in res["targets"].values()
)
res["verdict"] = "pass" if verdict else "fail"
# 取首个 stage 画图与 csv (多 stage 时逐 stage 各写一份 csv)
for stage, r in res["targets"].items():
with open(sample_out / f"per_dim_{stage}.csv", "w", newline="") as f:
w = csv.DictWriter(f, fieldnames=["dim", "kind", "cosine", "mae", "rmse", "max_abs"])
w.writeheader()
w.writerows(r["per_dim"])
plot_dir = sample_out if stage == stages[0] else (sample_out / stage)
plot_dir.mkdir(exist_ok=True)
plot_sample(plot_dir, r)
(sample_out / "summary.json").write_text(json.dumps(res, indent=2, ensure_ascii=False), encoding="utf-8")
all_summary["samples"][sid] = {
"verdict": res["verdict"],
**{stage: {"tensor_cosine": r["tensor"]["cosine"], "valid_dim_median_cosine": r["valid_dim_median_cosine"]}
for stage, r in res["targets"].items()},
}
logger.info("[compare] sid=%s %s", sid, all_summary["samples"][sid])
(out_root / "summary_all.json").write_text(json.dumps(all_summary, indent=2, ensure_ascii=False), encoding="utf-8")
n_pass = sum(1 for s in all_summary["samples"].values() if s["verdict"] == "pass")
print(f"DROBOTICFLOW_PROGRESS 100 sample_compare {n_pass}/{len(sids)} pass")
if n_pass != len(sids):
logger.warning("[compare] 存在未达标样本, 醒目提示 (不阻断): 见 summary_all.json")
if __name__ == "__main__":
main()