bench b3be5cb102 板上每次调用按实测字节分批、G2 余弦按组算:大输出模型不再一次 12 GB(只取 f56ac2f 中不在冻结指纹里的部分)
task_contract 只哈希 harness/eval/<任务>.py;board.py、common.py、run.py 不在指纹里,在跑的 run 可以上。
各任务文件里的分组调用(分割/深度/光流/特征一次多张)仍等新建 run。
10-15 05 solo.r50 的 G2 一批 32 张 12 GB,容器内存撑爆,回执带不回来,整条轨迹作废。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 13:43:16 +08:00

113 lines
4.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""真实板端推理后端;复用评测器的预处理和指标计算,仅替换 runner。"""
from __future__ import annotations
import hashlib
import os
import tempfile
from pathlib import Path
import numpy as np
from harness.tools.board_perf import RemoteSession
class BoardRunner:
"""每次评测传一次 HBM,以小块 NPZ 传输入/输出,退出时清理临时文件。"""
CHUNK = 32
MAX_BYTES = 1 << 30 # inputs + outputs per board call (10-15: one 32-sample solo.r50 call was 12 GB, filled an S600)
def __init__(self, hbm: str, runtime_dir: str = "/userdata/bench"):
self.hbm = str(hbm)
self.runtime_dir = runtime_dir
h = hashlib.sha256()
with open(hbm, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
self.metadata = {"source": "board", "hbm_sha256": h.hexdigest()}
if os.environ.get("BOARD_NAME"):
self.metadata["board_name"] = os.environ["BOARD_NAME"]
self._local = tempfile.TemporaryDirectory(prefix="board-eval-")
self.tmp = Path(self._local.name)
self._session: RemoteSession | None = None
self._closed = False
self._samples = 0
self._sample_bytes = 0
def group_size(self) -> int:
"""Samples per board call: one until a sample's inputs + outputs are measured, then as many as fit
MAX_BYTES. Evaluators group by it too, so one group's outputs also fit in memory."""
return max(1, min(self.CHUNK, self.MAX_BYTES // self._sample_bytes)) if self._sample_bytes else 1
def __enter__(self):
if self._closed:
raise RuntimeError("BoardRunner is closed")
if self._session is None:
try:
self._session = RemoteSession(self.hbm, self.runtime_dir)
self._session.__enter__()
except BaseException:
self.close()
raise
return self
def batch(self, xs: list[np.ndarray]) -> list[list[np.ndarray]]:
if not xs:
return []
self.__enter__()
session = self._session
results: list[list[np.ndarray]] = []
fin, fout = self.tmp / "inputs.npz", self.tmp / "outputs.npz"
try:
start = 0
while start < len(xs):
chunk = xs[start:start + self.group_size()]
start += len(chunk)
for x in chunk:
if x.dtype != np.float32:
raise ValueError(f"board evaluation requires float32 inputs, found {x.dtype}")
np.savez(fin, **{f"s{i:04d}": np.ascontiguousarray(x) for i, x in enumerate(chunk)})
session.upload(fin, "inputs.npz")
info = session.bench(["--inputs", session.path("inputs.npz"), "--out", session.path("outputs.npz")])
if info.get("hbm_sha256") != self.metadata["hbm_sha256"]:
raise RuntimeError("board HBM SHA256 does not match local HBM")
if info.get("samples") != len(chunk):
raise RuntimeError("board returned an unexpected sample count")
session.download("outputs.npz", fout)
self._sample_bytes = max(self._sample_bytes, (fin.stat().st_size + fout.stat().st_size) // len(chunk))
output_count = len(info.get("outputs", []))
if output_count == 0:
raise RuntimeError("board returned no model outputs")
expected = {f"s{i:04d}/out{j}" for i in range(len(chunk)) for j in range(output_count)}
with np.load(fout, allow_pickle=False) as archive:
if set(archive.files) != expected:
raise RuntimeError("board output NPZ is missing samples or outputs")
for i in range(len(chunk)):
results.append([np.array(archive[f"s{i:04d}/out{j}"], copy=True) for j in range(output_count)])
self.metadata.update({k: v for k, v in info.items() if k not in ("samples", "load_s")})
self.metadata["source"] = "board"
session.remove("inputs.npz", "outputs.npz")
fin.unlink(missing_ok=True)
fout.unlink(missing_ok=True)
self._samples += len(chunk)
print(f"board inference: {self._samples} samples completed", flush=True)
except BaseException:
self.close()
raise
return results
def __call__(self, x: np.ndarray) -> list[np.ndarray]:
return self.batch([x])[0]
def close(self):
if not self._closed:
self._closed = True
try:
if self._session is not None:
self._session.close()
finally:
self._local.cleanup()
def __exit__(self, *_):
self.close()