task_contract 只哈希 harness/eval/<任务>.py;board.py、common.py、run.py 不在指纹里,在跑的 run 可以上。 各任务文件里的分组调用(分割/深度/光流/特征一次多张)仍等新建 run。 10-15 05 solo.r50 的 G2 一批 32 张 12 GB,容器内存撑爆,回执带不回来,整条轨迹作废。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
4.8 KiB
Python
113 lines
4.8 KiB
Python
"""真实板端推理后端;复用评测器的预处理和指标计算,仅替换 runner。"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import os
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
|
||
from harness.tools.board_perf import RemoteSession
|
||
|
||
|
||
class BoardRunner:
|
||
"""每次评测传一次 HBM,以小块 NPZ 传输入/输出,退出时清理临时文件。"""
|
||
|
||
CHUNK = 32
|
||
MAX_BYTES = 1 << 30 # inputs + outputs per board call (10-15: one 32-sample solo.r50 call was 12 GB, filled an S600)
|
||
|
||
def __init__(self, hbm: str, runtime_dir: str = "/userdata/bench"):
|
||
self.hbm = str(hbm)
|
||
self.runtime_dir = runtime_dir
|
||
h = hashlib.sha256()
|
||
with open(hbm, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||
h.update(chunk)
|
||
self.metadata = {"source": "board", "hbm_sha256": h.hexdigest()}
|
||
if os.environ.get("BOARD_NAME"):
|
||
self.metadata["board_name"] = os.environ["BOARD_NAME"]
|
||
self._local = tempfile.TemporaryDirectory(prefix="board-eval-")
|
||
self.tmp = Path(self._local.name)
|
||
self._session: RemoteSession | None = None
|
||
self._closed = False
|
||
self._samples = 0
|
||
self._sample_bytes = 0
|
||
|
||
def group_size(self) -> int:
|
||
"""Samples per board call: one until a sample's inputs + outputs are measured, then as many as fit
|
||
MAX_BYTES. Evaluators group by it too, so one group's outputs also fit in memory."""
|
||
return max(1, min(self.CHUNK, self.MAX_BYTES // self._sample_bytes)) if self._sample_bytes else 1
|
||
|
||
def __enter__(self):
|
||
if self._closed:
|
||
raise RuntimeError("BoardRunner is closed")
|
||
if self._session is None:
|
||
try:
|
||
self._session = RemoteSession(self.hbm, self.runtime_dir)
|
||
self._session.__enter__()
|
||
except BaseException:
|
||
self.close()
|
||
raise
|
||
return self
|
||
|
||
def batch(self, xs: list[np.ndarray]) -> list[list[np.ndarray]]:
|
||
if not xs:
|
||
return []
|
||
self.__enter__()
|
||
session = self._session
|
||
results: list[list[np.ndarray]] = []
|
||
fin, fout = self.tmp / "inputs.npz", self.tmp / "outputs.npz"
|
||
try:
|
||
start = 0
|
||
while start < len(xs):
|
||
chunk = xs[start:start + self.group_size()]
|
||
start += len(chunk)
|
||
for x in chunk:
|
||
if x.dtype != np.float32:
|
||
raise ValueError(f"board evaluation requires float32 inputs, found {x.dtype}")
|
||
np.savez(fin, **{f"s{i:04d}": np.ascontiguousarray(x) for i, x in enumerate(chunk)})
|
||
session.upload(fin, "inputs.npz")
|
||
info = session.bench(["--inputs", session.path("inputs.npz"), "--out", session.path("outputs.npz")])
|
||
if info.get("hbm_sha256") != self.metadata["hbm_sha256"]:
|
||
raise RuntimeError("board HBM SHA256 does not match local HBM")
|
||
if info.get("samples") != len(chunk):
|
||
raise RuntimeError("board returned an unexpected sample count")
|
||
session.download("outputs.npz", fout)
|
||
self._sample_bytes = max(self._sample_bytes, (fin.stat().st_size + fout.stat().st_size) // len(chunk))
|
||
output_count = len(info.get("outputs", []))
|
||
if output_count == 0:
|
||
raise RuntimeError("board returned no model outputs")
|
||
expected = {f"s{i:04d}/out{j}" for i in range(len(chunk)) for j in range(output_count)}
|
||
with np.load(fout, allow_pickle=False) as archive:
|
||
if set(archive.files) != expected:
|
||
raise RuntimeError("board output NPZ is missing samples or outputs")
|
||
for i in range(len(chunk)):
|
||
results.append([np.array(archive[f"s{i:04d}/out{j}"], copy=True) for j in range(output_count)])
|
||
self.metadata.update({k: v for k, v in info.items() if k not in ("samples", "load_s")})
|
||
self.metadata["source"] = "board"
|
||
session.remove("inputs.npz", "outputs.npz")
|
||
fin.unlink(missing_ok=True)
|
||
fout.unlink(missing_ok=True)
|
||
self._samples += len(chunk)
|
||
print(f"board inference: {self._samples} samples completed", flush=True)
|
||
except BaseException:
|
||
self.close()
|
||
raise
|
||
return results
|
||
|
||
def __call__(self, x: np.ndarray) -> list[np.ndarray]:
|
||
return self.batch([x])[0]
|
||
|
||
def close(self):
|
||
if not self._closed:
|
||
self._closed = True
|
||
try:
|
||
if self._session is not None:
|
||
self._session.close()
|
||
finally:
|
||
self._local.cleanup()
|
||
|
||
def __exit__(self, *_):
|
||
self.close()
|