Gates: compute must run on the BPU; board latency budget and the original model on the board CPU as speed floors
- 06-hbm records the convolutions/matmuls hb_compile left on the CPU (its node table's ON column); G1 fails on any when the batch's gates say require_bpu_compute (new in gates.yaml; frozen batches without it are not judged) - batch creation freezes agent/speed_ref/<job>.json into G4: latency_budget_ms = 1.5 x board_latency_ms (median board latency of plain int8 builds) and cpu_fp32_latency_ms; G4 also fails when the model is not faster than the original FP32 model on the board CPU, and records the speedup - scripts/board_refs.py computes both; the CPU number is onnxruntime on the board itself under the board lease - e2e prompt states the three rules; agent/VERSION 25 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f6e7dfe56a
commit
1dcf331c90
@ -1 +1 @@
|
||||
24
|
||||
25
|
||||
|
||||
@ -7,11 +7,12 @@ job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的
|
||||
1. G1 编译:产出 `.hbm`,march = {march},输入 {input_spec},`hb_verifier` 通过
|
||||
2. G2 数值:校准外样本的逐输出余弦,参考阈值 {g2_threshold},只作诊断
|
||||
3. G3 任务精度:板上 T1 子集({t1_n} 张)的 {g3_metric} 与同一子集的 FP32 基线比较,容差 {g3_tol}({g3_kind})
|
||||
4. G4 速度:不比平台默认量化配置(compile.yaml 只写两项必填)编出的模型慢 1.5 倍以上。参考在 `$BENCH_ROOT/agent/speed_ref/{job_id}.json`:`latency_ms_est` 是默认配置的编译器静态估计,不含落到 CPU 的算子;文件还没有表示还在算,值为 null 表示默认配置没编出来、本题没有参考。
|
||||
4. G4 速度:以 job.json 的 `gates.G4` 为准。`latency_budget_ms` 是本题板上实测延迟的上限(同题普通 int8 配置板上实测 `latency_ref_ms` 的 1.5 倍),`cpu_fp32_latency_ms` 是原始 FP32 模型在本题板卡 CPU 上的延迟,你的模型必须比它快。没有这两个字段的题不判速度。`$BENCH_ROOT/agent/speed_ref/{job_id}.json` 里还有默认配置的编译器静态估计 `latency_ms_est`,可以当量级参考。
|
||||
|
||||
两条硬规定,违反任何一条都不算通过:
|
||||
三条硬规定,违反任何一条都不算通过:
|
||||
- 本题是 int8 量化。`all_node_type`,或者整类计算算子(Conv、MatMul、Gemm 等;用 op_config,或者逐节点列出超过同类节点的一半)设成 int16、float16 或 float32,都判不通过。只把少数敏感节点设成 int16 可以。
|
||||
- 板上实测延迟超过同题默认 int8 配置板上实测的 1.5 倍,判不通过(G4 参考文件里是默认配置的编译器静态估计,可以当量级参考)。
|
||||
- 卷积、矩阵乘(Conv、ConvTranspose、MatMul、Gemm)必须跑在 BPU 上:hb_compile 节点表 `ON` 一列里有这类节点是 `CPU`,G1 判不通过。
|
||||
- 板上实测延迟超过 `latency_budget_ms`,或者不比 `cpu_fp32_latency_ms` 快,G4 判不通过。
|
||||
|
||||
缺少板上精度结果不能报完整 PASS;缺权重、工具或材料时如实保留实际错误。
|
||||
|
||||
|
||||
@ -53,9 +53,13 @@ def g1(m: dict | None, cfg: dict) -> dict:
|
||||
"verifier_ok": m.get("verifier_ok") if cfg.get("run_hb_verifier", True) else True,
|
||||
"log_clean": (not m.get("log_has_error")) if m.get("log_has_error") is not None else None,
|
||||
}
|
||||
if cfg.get("require_bpu_compute") and "cpu_compute_nodes" in m: # absent: 06-hbm predates the check
|
||||
cpu = m["cpu_compute_nodes"]
|
||||
checks["bpu_compute"] = None if cpu is None else not cpu
|
||||
if any(v is False for v in checks.values()):
|
||||
failed = [k for k, v in checks.items() if v is False]
|
||||
return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read")}
|
||||
return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read"),
|
||||
**({"cpu_compute_nodes": m["cpu_compute_nodes"][:20]} if m.get("cpu_compute_nodes") else {})}
|
||||
if any(v is None for v in checks.values()):
|
||||
return {"verdict": "unverified", "missing": [k for k, v in checks.items() if v is None], **checks}
|
||||
return {"verdict": "pass", **checks, "march_read": m.get("march_read")}
|
||||
@ -267,7 +271,12 @@ def g4(m: dict | None, cfg: dict) -> dict:
|
||||
return {"verdict": "na", **rec}
|
||||
if not m or m.get("latency_ms_mean") is None:
|
||||
return {"verdict": "unverified", **rec}
|
||||
return {"verdict": "pass" if m["latency_ms_mean"] <= budget else "fail", "budget_ms": budget, **rec}
|
||||
lat, cpu = m["latency_ms_mean"], cfg.get("cpu_fp32_latency_ms")
|
||||
if cpu:
|
||||
rec.update(cpu_fp32_latency_ms=cpu, speedup_vs_cpu=round(cpu / lat, 2) if lat else None)
|
||||
ok = lat <= budget and (not cpu or lat < cpu)
|
||||
return {"verdict": "pass" if ok else "fail", "budget_ms": budget, **rec,
|
||||
**({"latency_ref_ms": cfg["latency_ref_ms"]} if cfg.get("latency_ref_ms") else {})}
|
||||
|
||||
|
||||
def evaluate(measurements: dict, gates_cfg: dict, *, provenance: dict | None = None) -> dict:
|
||||
|
||||
@ -377,11 +377,28 @@ def gate_snapshot(spec: JobSpec) -> dict:
|
||||
"G1": g["G1"],
|
||||
"G2": g2,
|
||||
"G3": g3,
|
||||
"G4": {"mandatory": False, "latency_budget_ms": spec.overrides.get("latency_budget_ms"),
|
||||
"record": g["G4"]["record"]},
|
||||
"G4": {"mandatory": False, "record": g["G4"]["record"], **speed_budget(spec, g["G4"])},
|
||||
}
|
||||
|
||||
|
||||
def speed_budget(spec: JobSpec, cfg: dict) -> dict:
|
||||
"""agent/speed_ref/<job>.json (scripts/board_refs.py): the board latency of a plain int8 build and the original
|
||||
FP32 model's latency on the board CPU, frozen at batch creation. An explicit override still wins."""
|
||||
if spec.overrides.get("latency_budget_ms"):
|
||||
return {"latency_budget_ms": spec.overrides["latency_budget_ms"]}
|
||||
try:
|
||||
ref = json.loads((paths.AGENT / "speed_ref" / f"{spec.job_id}.json").read_text())
|
||||
except (OSError, ValueError):
|
||||
return {"latency_budget_ms": None}
|
||||
out = {"latency_budget_ms": None}
|
||||
if ref.get("board_latency_ms") and cfg.get("latency_ratio_max"):
|
||||
out.update(latency_ref_ms=ref["board_latency_ms"],
|
||||
latency_budget_ms=round(ref["board_latency_ms"] * cfg["latency_ratio_max"], 3))
|
||||
if ref.get("cpu_fp32_latency_ms"):
|
||||
out["cpu_fp32_latency_ms"] = ref["cpu_fp32_latency_ms"]
|
||||
return out
|
||||
|
||||
|
||||
def data_snapshot(spec: JobSpec, row: dict | None = None) -> dict:
|
||||
"""job.json.data:calib / t1 / t2 的 pool 路径与 manifest sha。缺的记 null,preflight 会报。"""
|
||||
row = model_row(spec.model) if row is None else row
|
||||
|
||||
@ -135,7 +135,8 @@ class Hbm(Stage):
|
||||
last = clog.rfind("Start hb_compile")
|
||||
tail = clog[last:] if last >= 0 else clog
|
||||
m["log_has_error"] = bool(re.search(r"\bERROR\b", tail))
|
||||
m["hmct_output_cosine"] = _hmct_output_cosine(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到
|
||||
m["hmct_output_cosine"] = _hmct_output_cosine(tail)
|
||||
m["cpu_compute_nodes"] = _cpu_compute_nodes(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到
|
||||
|
||||
(stage_dir / "stdout.log").write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n")
|
||||
(stage_dir / "stderr.log").touch()
|
||||
@ -215,6 +216,17 @@ def _check_io(io: dict, input_spec: str) -> bool | None:
|
||||
return any(i["shape"] == dims for i in ins)
|
||||
|
||||
|
||||
COMPUTE_OPS = {"Conv", "ConvTranspose", "MatMul", "Gemm"}
|
||||
_NODE_ROW = re.compile(r"\|\s*(\S+)\s*\|\s*(\S+)\s*\|\s*(BPU|CPU|CPU\+BPU)\s*\|")
|
||||
|
||||
|
||||
def _cpu_compute_nodes(text: str) -> list[str] | None:
|
||||
"""hb_compile's node table (| Node | NodeType | ON | ...): the convolutions/matmuls it left on the CPU, where a
|
||||
float16/float32 or unsupported layer ends up (10-14: FPN convs in float16 ran 50-87x slower). None: no table."""
|
||||
rows = _NODE_ROW.findall(text)
|
||||
return [node for node, kind, on in rows if on == "CPU" and kind in COMPUTE_OPS] if rows else None
|
||||
|
||||
|
||||
def _parse_static_perf(info: str) -> dict:
|
||||
"""hb_model_info stdout:FPS=2318.51, latency = 431.3 us, DDR = 12709504 bytes(编译器静态估算,不是板端实测)"""
|
||||
d = {}
|
||||
|
||||
@ -10,6 +10,7 @@ G1:
|
||||
run_hb_verifier: true
|
||||
verifier_cosine_threshold: 0.99 # 编译前后量化产物一致性,与 G2 的 FP32 比较分开
|
||||
log_must_not_contain: ["ERROR"]
|
||||
require_bpu_compute: true # 卷积/矩阵乘不许留在 CPU(hb_compile 节点表的 ON 列)
|
||||
|
||||
G2:
|
||||
default_threshold: 0.99
|
||||
@ -87,3 +88,4 @@ G4:
|
||||
mandatory: false
|
||||
judge_only_if: latency_budget_ms # job.json 声明才判
|
||||
record: [latency_ms_mean, latency_ms_p95, bpu_util, ddr_bw_mbps]
|
||||
latency_ratio_max: 1.5 # 预算 = 同题普通 int8 板上实测 × 1.5;另须快于原始 FP32 模型在板上 CPU 的速度
|
||||
|
||||
132
scripts/board_refs.py
Normal file
132
scripts/board_refs.py
Normal file
@ -0,0 +1,132 @@
|
||||
"""Board speed references per job, merged into agent/speed_ref/<job_id>.json; registry.gate_snapshot freezes them into
|
||||
each new batch's G4 (budget = 1.5 x board_latency_ms; the model must also beat cpu_fp32_latency_ms).
|
||||
|
||||
- board_latency_ms: median board latency (07-board, latency_ms_mean) of this job's plain int8 builds in any run:
|
||||
compile.yaml without int16/float qtypes or run_on_cpu. Rollouts whose compile.yaml was offloaded are not seen.
|
||||
- cpu_fp32_latency_ms: the platform's FP32 export (03-export cache, fetched from Salty storage) run with onnxruntime on
|
||||
the board's own CPU, all cores, under the board lease so no stage shares the board. onnxruntime lives in
|
||||
/userdata/bench-ort on the board (installed 10-24 with pip --target; the system python is untouched).
|
||||
|
||||
BENCH_CONFIG=<a run config> python3 scripts/board_refs.py RUN [--only JOB_ID ...] [--no-cpu]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import sqlite3
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from harness import paths, scheduler # noqa: E402
|
||||
from harness.stages.s03_export import onnx_path # noqa: E402
|
||||
from harness.tools.board_perf import ssh_base # noqa: E402
|
||||
|
||||
ORT_DIR = "/userdata/bench-ort"
|
||||
CPU_BENCH = r'''
|
||||
import json, os, sys, time
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
o = ort.SessionOptions(); o.intra_op_num_threads = os.cpu_count()
|
||||
s = ort.InferenceSession(sys.argv[1], o, providers=["CPUExecutionProvider"])
|
||||
feed = {i.name: np.random.rand(*[d if isinstance(d, int) else 1 for d in i.shape]).astype(np.float32) for i in s.get_inputs()}
|
||||
s.run(None, feed) # warm-up
|
||||
t, start = [], time.time()
|
||||
while len(t) < 10 and (len(t) < 3 or time.time() - start < 120):
|
||||
a = time.perf_counter(); s.run(None, feed); t.append((time.perf_counter() - a) * 1000)
|
||||
print(json.dumps({"latency_ms_median": sorted(t)[len(t) // 2], "runs": len(t), "threads": os.cpu_count()}))
|
||||
'''
|
||||
|
||||
|
||||
def plain_int8(yaml_text: str | None) -> bool:
|
||||
if not yaml_text:
|
||||
return False
|
||||
body = re.sub(r"model_output_type:.*", "", yaml_text)
|
||||
return not re.search(r"int16|float", body) and "run_on_cpu" not in body
|
||||
|
||||
|
||||
def board_latency(job_id: str) -> tuple[float | None, int]:
|
||||
db = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True, timeout=60)
|
||||
roots = [paths.RUNS, paths.BENCH_ROOT.parent / "bench" / "runs", paths.BENCH_ROOT.parent / "bench-staging" / "runs"]
|
||||
lat = []
|
||||
for run, kk, verdict in db.execute("SELECT run_id, kk, verdict FROM rollouts WHERE job_id=? AND verdict IS NOT NULL", (job_id,)):
|
||||
value = ((json.loads(verdict).get("gates") or {}).get("G4") or {}).get("latency_ms_mean")
|
||||
yaml_text = next((p.read_text() for r in dict.fromkeys(roots)
|
||||
if (p := r / run / job_id / "rollouts" / f"{kk:02d}" / "stages/04-calibrate/compile.yaml").exists()), None)
|
||||
if value and plain_int8(yaml_text):
|
||||
lat.append(value)
|
||||
return (round(statistics.median(lat), 3), len(lat)) if lat else (None, 0)
|
||||
|
||||
|
||||
def cpu_latency(job: dict) -> dict:
|
||||
onnx = onnx_path(job)
|
||||
if onnx is None:
|
||||
raise ValueError("no platform export for this model")
|
||||
remote = "personal:/bench/cache/" + onnx.relative_to(paths.CACHE).as_posix()
|
||||
salty = paths.config()["execution"]["salty_remote"]["salty"]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
local = Path(tmp) / "model.onnx"
|
||||
subprocess.run([salty, "storage", "cp", remote, str(local)], check=True, capture_output=True, timeout=1800)
|
||||
(Path(tmp) / "cpu_bench.py").write_text(CPU_BENCH)
|
||||
with scheduler._acquire_board(job["march"], 5.0, "normal") as (slot, board):
|
||||
if board is None:
|
||||
raise ValueError(f"no enabled board for {job['march']}")
|
||||
env = {k: os.environ.get(board[f"{k.split('_', 1)[1].lower()}_env"], "")
|
||||
for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER")}
|
||||
env["BOARD_SSH_KEY"] = os.environ.get(board["ssh_key_env"], "")
|
||||
os.environ.update(env)
|
||||
opts, target = ssh_base()
|
||||
work = f"{ORT_DIR}/run-{job['job_id']}"
|
||||
subprocess.run(["ssh", *opts, target, f"mkdir -p {shlex.quote(work)}"], check=True, timeout=120)
|
||||
subprocess.run(["scp", *[o.replace("-p", "-P") if o == "-p" else o for o in opts],
|
||||
str(local), str(Path(tmp) / "cpu_bench.py"), f"{target}:{work}/"], check=True, timeout=3600)
|
||||
try:
|
||||
out = subprocess.run(["ssh", *opts, target, f"cd {shlex.quote(work)} && PYTHONPATH={ORT_DIR} python3 cpu_bench.py model.onnx"],
|
||||
check=True, capture_output=True, text=True, timeout=3600).stdout
|
||||
finally:
|
||||
subprocess.run(["ssh", *opts, target, f"rm -rf {shlex.quote(work)}"], timeout=120)
|
||||
return {**json.loads(out.strip().splitlines()[-1]), "board": board.get("name")}
|
||||
|
||||
|
||||
def merge(job_id: str, fields: dict) -> None:
|
||||
out = paths.AGENT / "speed_ref" / f"{job_id}.json"
|
||||
ref = json.loads(out.read_text()) if out.exists() else {"schema": "bench.speed_ref/1", "job_id": job_id}
|
||||
ref.update(fields)
|
||||
tmp = out.with_name(f".{out.name}.tmp")
|
||||
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
|
||||
tmp.replace(out)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("run")
|
||||
ap.add_argument("--only", action="append")
|
||||
ap.add_argument("--no-cpu", action="store_true")
|
||||
a = ap.parse_args()
|
||||
for jd in sorted((paths.RUNS / a.run).iterdir()):
|
||||
job = json.loads((jd / "job.json").read_text())
|
||||
if a.only and job["job_id"] not in a.only:
|
||||
continue
|
||||
lat, n = board_latency(job["job_id"])
|
||||
fields = {"board_latency_ms": lat, "board_latency_n": n,
|
||||
"board_latency_rule": "median 07-board latency of plain int8 builds", "board_refs_at": time.time()}
|
||||
if not a.no_cpu:
|
||||
try:
|
||||
cpu = cpu_latency(job)
|
||||
fields.update(cpu_fp32_latency_ms=round(cpu["latency_ms_median"], 3), cpu_fp32_measured=cpu)
|
||||
except Exception as e: # one job's missing export or board hiccup must not stop the others
|
||||
fields["cpu_fp32_error"] = f"{type(e).__name__}: {e}"[:300]
|
||||
merge(job["job_id"], fields)
|
||||
print(job["job_id"], json.dumps({k: v for k, v in fields.items() if k != "board_refs_at"}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
58
scripts/test_bpu_speed_gates.py
Normal file
58
scripts/test_bpu_speed_gates.py
Normal file
@ -0,0 +1,58 @@
|
||||
"""BPU-compute (G1) and board speed (G4) gates. Run: python -m unittest scripts.test_bpu_speed_gates"""
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from harness import gates, registry
|
||||
from harness.stages.s06_hbm import _cpu_compute_nodes
|
||||
|
||||
TABLE = """INFO | Node | NodeType | ON | Threshold |
|
||||
INFO | /backbone/Conv | Conv | BPU | 1.0 |
|
||||
INFO | /head/Conv_1 | Conv | CPU | -- |
|
||||
INFO | /Softmax | Softmax | CPU+BPU | -- |
|
||||
INFO | /post/Mul | Mul | CPU | -- |
|
||||
"""
|
||||
G1 = {"check_march": True, "check_input_dtype_layout": True, "run_hb_verifier": True}
|
||||
M1 = {"hbm_exists": True, "march_ok": True, "io_ok": True, "verifier_ok": True, "log_has_error": False}
|
||||
|
||||
|
||||
class BpuCompute(unittest.TestCase):
|
||||
def test_table_lists_only_compute_nodes_on_cpu(self):
|
||||
self.assertEqual(_cpu_compute_nodes(TABLE), ["/head/Conv_1"])
|
||||
self.assertIsNone(_cpu_compute_nodes("no table here"))
|
||||
|
||||
def test_g1_fails_compute_on_cpu_only_when_required(self):
|
||||
m = {**M1, "cpu_compute_nodes": ["/head/Conv_1"]}
|
||||
self.assertEqual(gates.g1(m, G1)["verdict"], "pass") # old batches: not judged
|
||||
r = gates.g1(m, {**G1, "require_bpu_compute": True})
|
||||
self.assertEqual((r["verdict"], r["failed"]), ("fail", ["bpu_compute"]))
|
||||
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": []}, {**G1, "require_bpu_compute": True})["verdict"], "pass")
|
||||
self.assertEqual(gates.g1({**M1, "cpu_compute_nodes": None}, {**G1, "require_bpu_compute": True})["verdict"], "unverified")
|
||||
self.assertEqual(gates.g1(M1, {**G1, "require_bpu_compute": True})["verdict"], "pass") # measured before the check
|
||||
|
||||
|
||||
class BoardSpeed(unittest.TestCase):
|
||||
CFG = {"record": ["latency_ms_mean"], "latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0}
|
||||
|
||||
def test_budget_and_cpu(self):
|
||||
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["verdict"], "pass")
|
||||
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, self.CFG)["speedup_vs_cpu"], 16.0)
|
||||
self.assertEqual(gates.g4({"latency_ms_mean": 3.5}, self.CFG)["verdict"], "fail") # over 1.5x reference
|
||||
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {**self.CFG, "cpu_fp32_latency_ms": 2.0})["verdict"], "fail")
|
||||
self.assertEqual(gates.g4({"latency_ms_mean": 2.5}, {"record": []})["verdict"], "na")
|
||||
|
||||
def test_batch_creation_freezes_the_reference(self):
|
||||
spec = registry.JobSpec("det--m--nash-m--ptq--int8", "det", "m", "nash-m", "ptq", "int8", "A", "e2e")
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(registry.paths, "AGENT", Path(tmp)):
|
||||
(Path(tmp) / "speed_ref").mkdir()
|
||||
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}), {"latency_budget_ms": None})
|
||||
(Path(tmp) / "speed_ref" / f"{spec.job_id}.json").write_text(json.dumps(
|
||||
{"board_latency_ms": 2.0, "cpu_fp32_latency_ms": 40.0}))
|
||||
self.assertEqual(registry.speed_budget(spec, {"latency_ratio_max": 1.5}),
|
||||
{"latency_budget_ms": 3.0, "latency_ref_ms": 2.0, "cpu_fp32_latency_ms": 40.0})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user