196 lines
9.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""评测入口(在 backend venv 里跑)。
--mode fp32 : 02-baseline。T1 上算指标 + 抽 G2 样本保存输入与 FP32 输出。→ <out>/metrics.json, g2_inputs.npz, g2_fp32.npz
--mode quant: 08-eval。hbm 在 G2 样本上算余弦、在 T1 上算指标、与基线配对。→ <out>(eval.json)
"""
from __future__ import annotations
import argparse
import importlib
import json
import sys
from pathlib import Path
import numpy as np
from . import EVALUATOR_VERSION, COMPATIBLE_BASELINE_VERSIONS, TASK_MODULES, get as task_module
from .common import T1, TorchRunner, HbmRunner, cosine, preprocess_cls, letterbox, run_batched
def g2_inputs(task: str, t1: T1, input_spec: str, n: int, resize: int = 0,
model: str = "") -> dict[str, np.ndarray]:
size = int(input_spec.split("x")[-1])
xs = {}
module = task_module(task)
if model:
from ..adapters import get
custom = getattr(get(model), "g2_input", None)
if custom is not None:
# A top-down model cannot infer an image with no detector crop.
# Select from real eligible inputs; never insert None or a GT box.
eligible = [value for item in t1.items
if (value := custom(t1, item, input_spec)) is not None]
count = min(n, len(t1.items))
if len(eligible) < count:
raise ValueError(f"only {len(eligible)} valid {model} diagnostic inputs; require {count}")
selected = sorted(np.random.default_rng(20260914).permutation(len(eligible))[:count])
return {f"s{i:03d}": eligible[index] for i, index in enumerate(selected)}
for i, it in enumerate(t1.g2_subset(n)):
if callable(getattr(module, "inputs", None)):
xs[f"s{i:03d}"] = module.inputs(t1, it, input_spec, model)
continue
img = t1.image(it)
if task == "cls":
xs[f"s{i:03d}"] = preprocess_cls(img, size=size, resize=resize or round(size * 256 / 224), model=model)
elif task == "det" and model:
from ..adapters import get
transform = getattr(get(model), "preprocess", None)
xs[f"s{i:03d}"] = transform(img, input_spec) if transform else letterbox(img, size)[0]
else:
xs[f"s{i:03d}"] = letterbox(img, size)[0]
return xs
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--mode", choices=["fp32", "quant"], required=True)
ap.add_argument("--task", required=True)
ap.add_argument("--model", required=True)
ap.add_argument("--source")
ap.add_argument("--weights")
ap.add_argument("--hbm")
ap.add_argument("--onnx")
ap.add_argument("--backend")
ap.add_argument("--input-spec", required=True)
ap.add_argument("--resize", type=int, default=0, help="cls:resize 短边(torchvision 默认权重各不同);0 = size*256/224")
ap.add_argument("--t1", required=True)
ap.add_argument("--t1-sha")
ap.add_argument("--baseline")
ap.add_argument("--recipe", help="protected task recipe snapshot, never a workspace import")
ap.add_argument("--g2-samples", type=int, default=32)
ap.add_argument("--calib-source", default="t0")
ap.add_argument("--runtime-python", help="模拟器所在 venv(仅 sim 后端需要)")
ap.add_argument("--runner", choices=["sim", "board"], default="sim")
ap.add_argument("--runtime-dir", default="/userdata/bench")
ap.add_argument("--evaluator-version", default=EVALUATOR_VERSION)
ap.add_argument("--out", required=True)
a = ap.parse_args()
if a.mode == "quant" and a.runner == "board":
from ..tools.board_perf import not_started
not_started("BOARD_LOCAL_VALIDATION")
if a.evaluator_version not in COMPATIBLE_BASELINE_VERSIONS:
print(f"evaluator version mismatch: job wants {a.evaluator_version}, code is {EVALUATOR_VERSION}", file=sys.stderr)
return 66
if a.task not in TASK_MODULES:
print(f"task {a.task} evaluator not implemented (P4+)", file=sys.stderr)
return 65
mod = task_module(a.task)
recipe = None
if a.recipe:
if a.task != "seg":
raise ValueError("task recipes currently support semantic segmentation only")
from ..recipes import load
from .common import RecipeRunner
recipe = load(Path(a.recipe))
t1 = T1.open(a.t1, a.t1_sha)
out: dict = {"task": a.task, "model": a.model, "evaluator_version": EVALUATOR_VERSION,
"t1_manifest_sha": t1.manifest.get("sha256"), "n": len(t1.items)}
if a.mode == "fp32":
od = Path(a.out); od.mkdir(parents=True, exist_ok=True)
runner = (RecipeRunner(recipe, sys.executable, od.parent / "model_calls", t1.manifest["dataset"], reference=True)
if recipe else TorchRunner(a.model, a.source, a.weights, a.input_spec))
if recipe:
images = [t1.image(item) for item in t1.g2_subset(a.g2_samples)]
xs = {f"s{i:03d}": x for i, x in enumerate(runner.prepare(images))}
np.savez(od / "g2_images.npz", **{f"s{i:03d}": np.asarray(im) for i, im in enumerate(images)})
else:
xs = g2_inputs(a.task, t1, a.input_spec, a.g2_samples, a.resize, a.model)
np.savez(od / "g2_inputs.npz", **xs)
values = run_batched(runner, list(xs.values()))
if len(values) != len(xs):
raise ValueError("reference runner changed sample count")
np.savez(od / "g2_fp32.npz", **{f"{k}/out{i}": o for k, outputs in zip(xs, values) for i, o in enumerate(outputs)})
mod.run_fp32(runner, t1, runner.input_spec if recipe else a.input_spec, out, resize=a.resize)
(od / "metrics.json").write_text(json.dumps(out, ensure_ascii=False, indent=2))
print(json.dumps(out["metrics"]))
return 0
# quant:后端由受信任的阶段明确选择;板卡出错不回退模拟器。
bdir = Path(a.baseline)
baseline = json.loads((bdir / "metrics.json").read_text())
if baseline.get("t1_manifest_sha") != t1.manifest.get("sha256"):
print("baseline T1 sha != current T1 sha:基线与实测子集不一致,delta 不可比", file=sys.stderr)
return 67
if baseline.get("evaluator_version") not in COMPATIBLE_BASELINE_VERSIONS:
print("baseline evaluator version incompatible", file=sys.stderr)
return 66
if a.runner == "board":
from .board import BoardRunner
runner = BoardRunner(a.hbm, runtime_dir=a.runtime_dir)
else:
if not a.runtime_python:
print("--runtime-python required for sim runner", file=sys.stderr)
return 2
runner = HbmRunner(a.hbm, a.runtime_python)
if recipe:
runner = RecipeRunner(recipe, sys.executable, Path(a.out).parent / "_calls",
t1.manifest["dataset"], backend=runner)
with runner:
out["G2"] = numeric_diagnostic(runner, bdir, a.calib_source)
mod.run_quant(runner, t1, a.input_spec, baseline, out, resize=a.resize)
execution = {"source": a.runner, **getattr(runner, "metadata", {})}
if recipe:
execution["host_processing_seconds"] = dict(runner.host_seconds)
execution["host_timing_scope"] = "observed calls including process startup; not board-only latency"
out["execution"] = execution
if out.get("G3"):
out["G3"].update(execution)
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
if '_sample_predictions' in out:
import hashlib
prediction_path = Path(a.out).with_name('predictions.jsonl')
raw = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in out.pop('_sample_predictions')).encode()
prediction_path.write_bytes(raw)
out['G3']['predictions'] = {'schema': 'bench.classification-predictions/1',
'path': str(prediction_path), 'sha256': hashlib.sha256(raw).hexdigest(),
'n': out['n']}
Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=2))
print(json.dumps({"G2": out["G2"], "G3": out.get("G3")}))
return 0
def numeric_diagnostic(runner, bdir: Path, calib_source: str) -> dict:
"""缺少诊断样本或输出形状不同不拦任务指标;推理失败仍由调用方报告。"""
if not all((bdir / name).exists() for name in ("g2_inputs.npz", "g2_fp32.npz")):
return {"cos_min": None, "reason": "G2 reference samples missing"}
with np.load(bdir / "g2_inputs.npz", allow_pickle=False) as xs, np.load(bdir / "g2_fp32.npz", allow_pickle=False) as fp:
per_out: dict[str, list[float]] = {}
keys = list(xs.files)
if hasattr(runner, "recipe_adapter"):
if not (bdir / "g2_images.npz").is_file():
return {"cos_min": None, "reason": "G2 original images missing for deployment preprocessing"}
with np.load(bdir / "g2_images.npz", allow_pickle=False) as images:
if set(images.files) != set(keys):
return {"cos_min": None, "reason": "G2 original image identities differ"}
prepared = runner.prepare([images[key] for key in keys])
else:
prepared = [xs[k] for k in keys]
results = run_batched(runner, prepared)
if len(results) != len(keys):
raise ValueError("G2 runner changed sample count")
for k, outs in zip(keys, results):
for i, o in enumerate(outs):
ref_key = f"{k}/out{i}"
if ref_key not in fp or fp[ref_key].shape != o.shape:
return {"cos_min": None, "reason": "G2 output shapes differ"}
per_out.setdefault(f"out{i}", []).append(cosine(fp[ref_key], o))
cos_per_output = {k: float(min(v)) for k, v in per_out.items()}
return {"cos_min": min(cos_per_output.values()) if cos_per_output else None,
"cos_per_output": cos_per_output, "n": len(keys), "source": calib_source}
if __name__ == "__main__":
sys.exit(main())