task_contract 只哈希 harness/eval/<任务>.py;board.py、common.py、run.py 不在指纹里,在跑的 run 可以上。 各任务文件里的分组调用(分割/深度/光流/特征一次多张)仍等新建 run。 10-15 05 solo.r50 的 G2 一批 32 张 12 GB,容器内存撑爆,回执带不回来,整条轨迹作废。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
198 lines
10 KiB
Python
198 lines
10 KiB
Python
"""评测入口(在 backend venv 里跑)。
|
||
|
||
--mode fp32 : 02-baseline。T1 上算指标 + 抽 G2 样本保存输入与 FP32 输出。→ <out>/metrics.json, g2_inputs.npz, g2_fp32.npz
|
||
--mode quant: 08-eval。hbm 在 G2 样本上算余弦、在 T1 上算指标、与基线配对。→ <out>(eval.json)
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import importlib
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
|
||
from . import EVALUATOR_VERSION, COMPATIBLE_BASELINE_VERSIONS, TASK_MODULES, get as task_module
|
||
from .common import T1, TorchRunner, HbmRunner, cosine, preprocess_cls, letterbox, run_batched
|
||
|
||
|
||
def g2_inputs(task: str, t1: T1, input_spec: str, n: int, resize: int = 0,
|
||
model: str = "") -> dict[str, np.ndarray]:
|
||
size = int(input_spec.split("x")[-1])
|
||
xs = {}
|
||
module = task_module(task)
|
||
if model:
|
||
from ..adapters import get
|
||
custom = getattr(get(model), "g2_input", None)
|
||
if custom is not None:
|
||
# A top-down model cannot infer an image with no detector crop.
|
||
# Select from real eligible inputs; never insert None or a GT box.
|
||
eligible = [value for item in t1.items
|
||
if (value := custom(t1, item, input_spec)) is not None]
|
||
count = min(n, len(t1.items))
|
||
if len(eligible) < count:
|
||
raise ValueError(f"only {len(eligible)} valid {model} diagnostic inputs; require {count}")
|
||
selected = sorted(np.random.default_rng(20260914).permutation(len(eligible))[:count])
|
||
return {f"s{i:03d}": eligible[index] for i, index in enumerate(selected)}
|
||
for i, it in enumerate(t1.g2_subset(n)):
|
||
if callable(getattr(module, "inputs", None)):
|
||
xs[f"s{i:03d}"] = module.inputs(t1, it, input_spec, model)
|
||
continue
|
||
img = t1.image(it)
|
||
if task == "cls":
|
||
xs[f"s{i:03d}"] = preprocess_cls(img, size=size, resize=resize or round(size * 256 / 224), model=model)
|
||
elif task == "det" and model:
|
||
from ..adapters import get
|
||
transform = getattr(get(model), "preprocess", None)
|
||
xs[f"s{i:03d}"] = transform(img, input_spec) if transform else letterbox(img, size)[0]
|
||
else:
|
||
xs[f"s{i:03d}"] = letterbox(img, size)[0]
|
||
return xs
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--mode", choices=["fp32", "quant"], required=True)
|
||
ap.add_argument("--task", required=True)
|
||
ap.add_argument("--model", required=True)
|
||
ap.add_argument("--source")
|
||
ap.add_argument("--weights")
|
||
ap.add_argument("--hbm")
|
||
ap.add_argument("--onnx")
|
||
ap.add_argument("--backend")
|
||
ap.add_argument("--input-spec", required=True)
|
||
ap.add_argument("--resize", type=int, default=0, help="cls:resize 短边(torchvision 默认权重各不同);0 = size*256/224")
|
||
ap.add_argument("--t1", required=True)
|
||
ap.add_argument("--t1-sha")
|
||
ap.add_argument("--baseline")
|
||
ap.add_argument("--recipe", help="protected task recipe snapshot, never a workspace import")
|
||
ap.add_argument("--g2-samples", type=int, default=32)
|
||
ap.add_argument("--calib-source", default="t0")
|
||
ap.add_argument("--runtime-python", help="模拟器所在 venv(仅 sim 后端需要)")
|
||
ap.add_argument("--runner", choices=["sim", "board"], default="sim")
|
||
ap.add_argument("--runtime-dir", default="/userdata/bench")
|
||
ap.add_argument("--evaluator-version", default=EVALUATOR_VERSION)
|
||
ap.add_argument("--out", required=True)
|
||
a = ap.parse_args()
|
||
if a.mode == "quant" and a.runner == "board":
|
||
from ..tools.board_perf import not_started
|
||
not_started("BOARD_LOCAL_VALIDATION")
|
||
|
||
if a.evaluator_version not in COMPATIBLE_BASELINE_VERSIONS:
|
||
print(f"evaluator version mismatch: job wants {a.evaluator_version}, code is {EVALUATOR_VERSION}", file=sys.stderr)
|
||
return 66
|
||
if a.task not in TASK_MODULES:
|
||
print(f"task {a.task} evaluator not implemented (P4+)", file=sys.stderr)
|
||
return 65
|
||
mod = task_module(a.task)
|
||
recipe = None
|
||
if a.recipe:
|
||
if a.task != "seg":
|
||
raise ValueError("task recipes currently support semantic segmentation only")
|
||
from ..recipes import load
|
||
from .common import RecipeRunner
|
||
recipe = load(Path(a.recipe))
|
||
t1 = T1.open(a.t1, a.t1_sha)
|
||
out: dict = {"task": a.task, "model": a.model, "evaluator_version": EVALUATOR_VERSION,
|
||
"t1_manifest_sha": t1.manifest.get("sha256"), "n": len(t1.items)}
|
||
|
||
if a.mode == "fp32":
|
||
od = Path(a.out); od.mkdir(parents=True, exist_ok=True)
|
||
runner = (RecipeRunner(recipe, sys.executable, od.parent / "model_calls", t1.manifest["dataset"], reference=True)
|
||
if recipe else TorchRunner(a.model, a.source, a.weights, a.input_spec))
|
||
if recipe:
|
||
images = [t1.image(item) for item in t1.g2_subset(a.g2_samples)]
|
||
xs = {f"s{i:03d}": x for i, x in enumerate(runner.prepare(images))}
|
||
np.savez(od / "g2_images.npz", **{f"s{i:03d}": np.asarray(im) for i, im in enumerate(images)})
|
||
else:
|
||
xs = g2_inputs(a.task, t1, a.input_spec, a.g2_samples, a.resize, a.model)
|
||
np.savez(od / "g2_inputs.npz", **xs)
|
||
values = run_batched(runner, list(xs.values()))
|
||
if len(values) != len(xs):
|
||
raise ValueError("reference runner changed sample count")
|
||
np.savez(od / "g2_fp32.npz", **{f"{k}/out{i}": o for k, outputs in zip(xs, values) for i, o in enumerate(outputs)})
|
||
mod.run_fp32(runner, t1, runner.input_spec if recipe else a.input_spec, out, resize=a.resize)
|
||
(od / "metrics.json").write_text(json.dumps(out, ensure_ascii=False, indent=2))
|
||
print(json.dumps(out["metrics"]))
|
||
return 0
|
||
|
||
# quant:后端由受信任的阶段明确选择;板卡出错不回退模拟器。
|
||
bdir = Path(a.baseline)
|
||
baseline = json.loads((bdir / "metrics.json").read_text())
|
||
if baseline.get("t1_manifest_sha") != t1.manifest.get("sha256"):
|
||
print("baseline T1 sha != current T1 sha:基线与实测子集不一致,delta 不可比", file=sys.stderr)
|
||
return 67
|
||
if baseline.get("evaluator_version") not in COMPATIBLE_BASELINE_VERSIONS:
|
||
print("baseline evaluator version incompatible", file=sys.stderr)
|
||
return 66
|
||
if a.runner == "board":
|
||
from .board import BoardRunner
|
||
runner = BoardRunner(a.hbm, runtime_dir=a.runtime_dir)
|
||
else:
|
||
if not a.runtime_python:
|
||
print("--runtime-python required for sim runner", file=sys.stderr)
|
||
return 2
|
||
runner = HbmRunner(a.hbm, a.runtime_python)
|
||
if recipe:
|
||
runner = RecipeRunner(recipe, sys.executable, Path(a.out).parent / "_calls",
|
||
t1.manifest["dataset"], backend=runner)
|
||
with runner:
|
||
out["G2"] = numeric_diagnostic(runner, bdir, a.calib_source)
|
||
mod.run_quant(runner, t1, a.input_spec, baseline, out, resize=a.resize)
|
||
execution = {"source": a.runner, **getattr(runner, "metadata", {})}
|
||
if recipe:
|
||
execution["host_processing_seconds"] = dict(runner.host_seconds)
|
||
execution["host_timing_scope"] = "observed calls including process startup; not board-only latency"
|
||
out["execution"] = execution
|
||
if out.get("G3"):
|
||
out["G3"].update(execution)
|
||
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
|
||
if '_sample_predictions' in out:
|
||
import hashlib
|
||
prediction_path = Path(a.out).with_name('predictions.jsonl')
|
||
raw = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in out.pop('_sample_predictions')).encode()
|
||
prediction_path.write_bytes(raw)
|
||
out['G3']['predictions'] = {'schema': 'bench.classification-predictions/1',
|
||
'path': str(prediction_path), 'sha256': hashlib.sha256(raw).hexdigest(),
|
||
'n': out['n']}
|
||
Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=2))
|
||
print(json.dumps({"G2": out["G2"], "G3": out.get("G3")}))
|
||
return 0
|
||
|
||
|
||
def numeric_diagnostic(runner, bdir: Path, calib_source: str) -> dict:
|
||
"""缺少诊断样本或输出形状不同不拦任务指标;推理失败仍由调用方报告。"""
|
||
if not all((bdir / name).exists() for name in ("g2_inputs.npz", "g2_fp32.npz")):
|
||
return {"cos_min": None, "reason": "G2 reference samples missing"}
|
||
with np.load(bdir / "g2_inputs.npz", allow_pickle=False) as xs, np.load(bdir / "g2_fp32.npz", allow_pickle=False) as fp:
|
||
per_out: dict[str, list[float]] = {}
|
||
keys = list(xs.files)
|
||
if hasattr(runner, "recipe_adapter"):
|
||
if not (bdir / "g2_images.npz").is_file():
|
||
return {"cos_min": None, "reason": "G2 original images missing for deployment preprocessing"}
|
||
with np.load(bdir / "g2_images.npz", allow_pickle=False) as images:
|
||
if set(images.files) != set(keys):
|
||
return {"cos_min": None, "reason": "G2 original image identities differ"}
|
||
prepared = runner.prepare([images[key] for key in keys])
|
||
else:
|
||
prepared = [xs[k] for k in keys]
|
||
from .common import groups
|
||
for group in groups(runner, list(range(len(keys))), len(keys)):
|
||
results = run_batched(runner, [prepared[j] for j in group])
|
||
if len(results) != len(group):
|
||
raise ValueError("G2 runner changed sample count")
|
||
for j, outs in zip(group, results):
|
||
for i, o in enumerate(outs):
|
||
ref_key = f"{keys[j]}/out{i}"
|
||
if ref_key not in fp or fp[ref_key].shape != o.shape:
|
||
return {"cos_min": None, "reason": "G2 output shapes differ"}
|
||
per_out.setdefault(f"out{i}", []).append(cosine(fp[ref_key], o))
|
||
cos_per_output = {k: float(min(v)) for k, v in per_out.items()}
|
||
return {"cos_min": min(cos_per_output.values()) if cos_per_output else None,
|
||
"cos_per_output": cos_per_output, "n": len(keys), "source": calib_source}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|