data-gen-bpu/harness/eval/__init__.py
bench 6ab088d7ee 评测分批不建新批次也能上:冻结规则增加逐对批准的等价改写,任务文件换成分批版本
- harness/eval/__init__.py:EQUIVALENT_SOURCES 列出精确的「冻结指纹 → 现指纹」对,contract_holds 判断现文件
  仍实现冻结合同(完全相同,或同一模块的已批准改写)。阶段检查、基线检查、expand 比对三处都走它。
- 批准的只有 f56ac2f 的 5 个任务文件:上板由逐张改为按实测字节分批,板上仍逐个样本推理
  (board_bench.infer_inputs),每张的预处理、后处理和指标一字未改。
- 批次的冻结记录不动,基线缓存按冻结指纹寻址,照常命中,不用重算。
- 用户 10-15:不建新批次。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 15:10:04 +08:00

63 lines
3.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""逐任务评测器。agent 不可写;verdict 的唯一依据。FP32 基线与量化实测走同一份代码、同一子集、同一预处理。
EVALUATOR_VERSION 任何口径变化都要升;job.json 钉死它,基线缓存按它失效。
"""
EVALUATOR_VERSION = "0.5.0"
# 0.5 新增真实板卡后端;FP32 的预处理、预测及指标公式与 0.4 完全相同。
COMPATIBLE_BASELINE_VERSIONS = {"0.4.0", EVALUATOR_VERSION}
TASK_MODULES = {
"cls": "harness.eval.cls",
"det": "harness.eval.det",
# Additive task support; cls/det preprocessing and metrics remain 0.5-compatible.
"iseg": "harness.eval.coco_extra",
"pose": "harness.eval.coco_extra",
"flow": "harness.eval.flow",
"seg": "harness.eval.seg",
"mde": "harness.eval.mde",
"feat": "harness.eval.feat",
# P4+:stereo occ
}
# A frozen task module may be replaced inside a running run only by a rewrite approved here, pair by pair, as giving
# identical per-sample outputs and metrics; anything else still needs a new run. 10-15 (f56ac2f): board calls batched
# by measured bytes instead of one image per call; the board still infers sample by sample (board_bench.infer_inputs).
EQUIVALENT_SOURCES = {
("95f5b4a5682feaf08920c7beb3a0455931f7d7cab270f6ba2d298a3989025c94", "f348981b3be85712b7c3874036e1ac5b1a8f4ab625f500467f7ea62586653969"), # seg
("3feef7cd074a9e09a0ef98c68bd8d5db09c44b0fb3f0588cfd9185c84237788a", "dca658bfc25d906d5a155d51efdfe9300d9ff7fd1430c715574d64bd61177dd3"), # mde
("5bdd6fd229cd3cf4accb0190e74920d71fc4a5d6a0d0aced5872ca84f7983bb3", "ca570f710ecd4428d02b6b81765de51a4d3b692754c6630d1462c5bc5d1073cd"), # flow
("210f0ea503018b17ed4cb98fe79f82a0a761a80c6308821c020ca7fdc5aab32e", "9a321c31f29a6fd73d4072ac281fd6a5d00dbf5f88f0fc15bc96c6fe092fde32"), # feat
("599cf728e77f3d6ffc8e678457a5b703d4e0de615ead12498275f37142db24d7", "505d37ffc245dc095e7d0d4ec215ed28e4f0105d542e660c5e3ff9ba32bdd9da"), # coco_extra
}
def contract_holds(frozen: dict, current: dict) -> bool:
"""The current module still implements the frozen contract: identical, or an approved rewrite of the same module."""
return current == frozen or (
{**current, "source_sha256": None} == {**frozen, "source_sha256": None}
and (frozen.get("source_sha256"), current.get("source_sha256")) in EQUIVALENT_SOURCES)
def get(task: str):
"""One dispatch point. A task module owns inputs and both metric paths."""
import importlib
if task not in TASK_MODULES:
raise ValueError(f"task {task!r} evaluator not implemented")
module = importlib.import_module(TASK_MODULES[task])
for name in ("run_fp32", "run_quant"):
if not callable(getattr(module, name, None)):
raise ValueError(f"task {task!r} missing {name}")
return module
def task_contract(task: str) -> dict:
"""Freeze implementation identity without importing torch or a toolchain."""
import hashlib
from pathlib import Path
name = TASK_MODULES.get(task)
source = Path(__file__).resolve().parents[2] / (name.replace(".", "/") + ".py") if name else None
return {"task": task, "protocol": "bench.eval/1", "module": name,
"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source and source.is_file() else None,
"implemented": bool(name and source and source.is_file())}