175 lines
7.6 KiB
Python
175 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""主通道维度对比: fp (torch float) vs 量化目标 (伪量化 calib / 板端 hbm), 逐 action 维度。
|
|
|
|
主通道 = 模型主输出通道 (action 张量 [action_horizon, action_dim], 有效维 valid_action_dim, 其余为 padding 维)。
|
|
|
|
输入:
|
|
OUTPUT_DIR 必填; 需含 eval_dump/{stage}/torch/{sid}.npy 与 eval_dump/{stage}/{stage}/{sid}.npy
|
|
(torch_eval 阶段 vla_eval 产物, unified_dumper 逐 chunk dump)
|
|
SAMPLE_INDEX 选样参数: "0" (默认) / 逗号列表 "0,3,7" / "all"
|
|
STAGES 对比对象, 默认 "calib" (伪量化); M2 后可加 "hbm"
|
|
VALID_ACTION_DIM 有效动作维 (默认 14, 契约 state.dim; >= 该维的为 padding 维单独标注)
|
|
产物 (OUTPUT_DIR/compare/):
|
|
sample_{sid}/per_dim.csv dim, kind(valid/padding), cosine, mae, rmse, max_abs
|
|
sample_{sid}/per_dim.png 逐维 cosine 柱状 + 逐时间步误差曲线
|
|
sample_{sid}/summary.json 整体指标 + PASS 判定
|
|
summary_all.json 全部取样汇总
|
|
PASS 初值: 整体 tensor cosine >= 0.98 且 逐维 cosine 中位数 >= 0.95 (quant_design_v1 §8)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
logger = logging.getLogger("sample_compare")
|
|
|
|
PASS_TENSOR_COS = 0.98
|
|
PASS_DIM_MEDIAN_COS = 0.95
|
|
|
|
|
|
def cosine(a: np.ndarray, b: np.ndarray) -> float:
|
|
d = float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-12))
|
|
return d
|
|
|
|
|
|
def compare_pair(fp: np.ndarray, qt: np.ndarray, valid_dim: int) -> dict:
|
|
if fp.ndim == 3 and fp.shape[0] == 1: # dump 带 batch 维 [1, horizon, dim]
|
|
fp = fp[0]
|
|
if qt.ndim == 3 and qt.shape[0] == 1:
|
|
qt = qt[0]
|
|
h, dim = fp.shape
|
|
err = qt - fp
|
|
per_dim = []
|
|
for d in range(dim):
|
|
per_dim.append({
|
|
"dim": d,
|
|
"kind": "valid" if d < valid_dim else "padding",
|
|
"cosine": round(cosine(fp[:, d], qt[:, d]), 6),
|
|
"mae": round(float(np.abs(err[:, d]).mean()), 6),
|
|
"rmse": round(float(np.sqrt((err[:, d] ** 2).mean())), 6),
|
|
"max_abs": round(float(np.abs(err[:, d]).max()), 6),
|
|
})
|
|
valid_cos = [p["cosine"] for p in per_dim if p["kind"] == "valid"]
|
|
return {
|
|
"shape": [h, dim],
|
|
"tensor": {
|
|
"cosine": round(cosine(fp.reshape(-1), qt.reshape(-1)), 6),
|
|
"mae": round(float(np.abs(err).mean()), 6),
|
|
"rmse": round(float(np.sqrt((err ** 2).mean())), 6),
|
|
"max_abs": round(float(np.abs(err).max()), 6),
|
|
},
|
|
"valid_dim_median_cosine": round(float(np.median(valid_cos)), 6),
|
|
"valid_dim_min_cosine": round(float(np.min(valid_cos)), 6),
|
|
"per_step_mae": np.abs(err).mean(axis=1).round(6).tolist(),
|
|
"per_dim": per_dim,
|
|
}
|
|
|
|
|
|
def plot_sample(out_dir: Path, res: dict) -> None:
|
|
try:
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
except ImportError:
|
|
return
|
|
per_dim = res["per_dim"]
|
|
fig, axes = plt.subplots(1, 2, figsize=(12, 4))
|
|
dims = [p["dim"] for p in per_dim]
|
|
cos = [p["cosine"] for p in per_dim]
|
|
colors = ["tab:blue" if p["kind"] == "valid" else "tab:gray" for p in per_dim]
|
|
axes[0].bar(dims, cos, color=colors)
|
|
axes[0].axhline(PASS_DIM_MEDIAN_COS, ls="--", c="r", lw=1)
|
|
axes[0].set_xlabel("action dim"); axes[0].set_ylabel("cosine"); axes[0].set_ylim(min(0.0, min(cos) - 0.05), 1.05)
|
|
axes[0].set_title("per-dim cosine (gray=padding)")
|
|
steps = np.arange(len(res["per_step_mae"]))
|
|
axes[1].plot(steps, res["per_step_mae"])
|
|
axes[1].set_xlabel("horizon step"); axes[1].set_title("per-step MAE")
|
|
fig.tight_layout()
|
|
fig.savefig(out_dir / "per_dim.png", dpi=120)
|
|
|
|
|
|
def main() -> None:
|
|
output_dir = Path(os.environ.get("OUTPUT_DIR", "").strip())
|
|
if not output_dir:
|
|
sys.exit("[hyperparam] OUTPUT_DIR not set")
|
|
stages = [s.strip() for s in os.environ.get("STAGES", "calib").split(",") if s.strip()]
|
|
valid_dim = int(os.environ.get("VALID_ACTION_DIM", "14"))
|
|
sel = os.environ.get("SAMPLE_INDEX", "0").strip()
|
|
|
|
dump_root = output_dir / "eval_dump"
|
|
if not dump_root.is_dir():
|
|
sys.exit(f"[input] 无 eval_dump: {dump_root} (先跑 run_quant STAGES=eval)")
|
|
|
|
# 可用样本 = 任一 stage torch 目录下的 {sid}.npy
|
|
torch_dir0 = dump_root / stages[0] / "torch"
|
|
sids_all = sorted(p.stem for p in torch_dir0.glob("*.npy"))
|
|
if not sids_all:
|
|
sys.exit(f"[input] {torch_dir0} 无 npy")
|
|
if sel == "all":
|
|
sids = sids_all
|
|
else:
|
|
want = [int(x) for x in sel.split(",") if x.strip() != ""]
|
|
bad = [w for w in want if str(w) not in sids_all]
|
|
if bad:
|
|
sys.exit(f"[input] 样本不存在: {bad} (可用: {sids_all[:8]}...)")
|
|
sids = [str(w) for w in want]
|
|
logger.info("[compare] samples=%s stages=%s valid_dim=%d", sids, stages, valid_dim)
|
|
|
|
out_root = output_dir / "compare"
|
|
out_root.mkdir(parents=True, exist_ok=True)
|
|
all_summary = {"pass_thresholds": {"tensor_cosine": PASS_TENSOR_COS, "valid_dim_median_cosine": PASS_DIM_MEDIAN_COS},
|
|
"valid_action_dim": valid_dim, "stages": stages, "samples": {}}
|
|
for sid in sids:
|
|
fp_path = dump_root / stages[0] / "torch" / f"{sid}.npy"
|
|
fp = np.load(fp_path)
|
|
sample_out = out_root / f"sample_{sid}"
|
|
sample_out.mkdir(exist_ok=True)
|
|
res = {"fp_source": str(fp_path), "targets": {}}
|
|
for stage in stages:
|
|
qt_path = dump_root / stage / stage / f"{sid}.npy"
|
|
if not qt_path.is_file():
|
|
sys.exit(f"[input] 缺 {qt_path}")
|
|
qt = np.load(qt_path)
|
|
if qt.shape != fp.shape:
|
|
sys.exit(f"[input] 形状不一致 fp{fp.shape} vs {stage}{qt.shape}")
|
|
res["targets"][stage] = compare_pair(fp.astype(np.float64), qt.astype(np.float64), valid_dim)
|
|
verdict = all(
|
|
t["tensor"]["cosine"] >= PASS_TENSOR_COS and t["valid_dim_median_cosine"] >= PASS_DIM_MEDIAN_COS
|
|
for t in res["targets"].values()
|
|
)
|
|
res["verdict"] = "pass" if verdict else "fail"
|
|
# 取首个 stage 画图与 csv (多 stage 时逐 stage 各写一份 csv)
|
|
for stage, r in res["targets"].items():
|
|
with open(sample_out / f"per_dim_{stage}.csv", "w", newline="") as f:
|
|
w = csv.DictWriter(f, fieldnames=["dim", "kind", "cosine", "mae", "rmse", "max_abs"])
|
|
w.writeheader()
|
|
w.writerows(r["per_dim"])
|
|
plot_dir = sample_out if stage == stages[0] else (sample_out / stage)
|
|
plot_dir.mkdir(exist_ok=True)
|
|
plot_sample(plot_dir, r)
|
|
(sample_out / "summary.json").write_text(json.dumps(res, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
all_summary["samples"][sid] = {
|
|
"verdict": res["verdict"],
|
|
**{stage: {"tensor_cosine": r["tensor"]["cosine"], "valid_dim_median_cosine": r["valid_dim_median_cosine"]}
|
|
for stage, r in res["targets"].items()},
|
|
}
|
|
logger.info("[compare] sid=%s %s", sid, all_summary["samples"][sid])
|
|
|
|
(out_root / "summary_all.json").write_text(json.dumps(all_summary, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
n_pass = sum(1 for s in all_summary["samples"].values() if s["verdict"] == "pass")
|
|
print(f"DROBOTICFLOW_PROGRESS 100 sample_compare {n_pass}/{len(sids)} pass")
|
|
if n_pass != len(sids):
|
|
logger.warning("[compare] 存在未达标样本, 醒目提示 (不阻断): 见 summary_all.json")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|