- harness/eval/__init__.py:EQUIVALENT_SOURCES 列出精确的「冻结指纹 → 现指纹」对,contract_holds 判断现文件 仍实现冻结合同(完全相同,或同一模块的已批准改写)。阶段检查、基线检查、expand 比对三处都走它。 - 批准的只有 f56ac2f 的 5 个任务文件:上板由逐张改为按实测字节分批,板上仍逐个样本推理 (board_bench.infer_inputs),每张的预处理、后处理和指标一字未改。 - 批次的冻结记录不动,基线缓存按冻结指纹寻址,照常命中,不用重算。 - 用户 10-15:不建新批次。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
63 lines
3.3 KiB
Python
63 lines
3.3 KiB
Python
"""逐任务评测器。agent 不可写;verdict 的唯一依据。FP32 基线与量化实测走同一份代码、同一子集、同一预处理。
|
||
|
||
EVALUATOR_VERSION 任何口径变化都要升;job.json 钉死它,基线缓存按它失效。
|
||
"""
|
||
EVALUATOR_VERSION = "0.5.0"
|
||
# 0.5 新增真实板卡后端;FP32 的预处理、预测及指标公式与 0.4 完全相同。
|
||
COMPATIBLE_BASELINE_VERSIONS = {"0.4.0", EVALUATOR_VERSION}
|
||
|
||
TASK_MODULES = {
|
||
"cls": "harness.eval.cls",
|
||
"det": "harness.eval.det",
|
||
# Additive task support; cls/det preprocessing and metrics remain 0.5-compatible.
|
||
"iseg": "harness.eval.coco_extra",
|
||
"pose": "harness.eval.coco_extra",
|
||
"flow": "harness.eval.flow",
|
||
"seg": "harness.eval.seg",
|
||
"mde": "harness.eval.mde",
|
||
"feat": "harness.eval.feat",
|
||
# P4+:stereo occ
|
||
}
|
||
|
||
|
||
# A frozen task module may be replaced inside a running run only by a rewrite approved here, pair by pair, as giving
|
||
# identical per-sample outputs and metrics; anything else still needs a new run. 10-15 (f56ac2f): board calls batched
|
||
# by measured bytes instead of one image per call; the board still infers sample by sample (board_bench.infer_inputs).
|
||
EQUIVALENT_SOURCES = {
|
||
("95f5b4a5682feaf08920c7beb3a0455931f7d7cab270f6ba2d298a3989025c94", "f348981b3be85712b7c3874036e1ac5b1a8f4ab625f500467f7ea62586653969"), # seg
|
||
("3feef7cd074a9e09a0ef98c68bd8d5db09c44b0fb3f0588cfd9185c84237788a", "dca658bfc25d906d5a155d51efdfe9300d9ff7fd1430c715574d64bd61177dd3"), # mde
|
||
("5bdd6fd229cd3cf4accb0190e74920d71fc4a5d6a0d0aced5872ca84f7983bb3", "ca570f710ecd4428d02b6b81765de51a4d3b692754c6630d1462c5bc5d1073cd"), # flow
|
||
("210f0ea503018b17ed4cb98fe79f82a0a761a80c6308821c020ca7fdc5aab32e", "9a321c31f29a6fd73d4072ac281fd6a5d00dbf5f88f0fc15bc96c6fe092fde32"), # feat
|
||
("599cf728e77f3d6ffc8e678457a5b703d4e0de615ead12498275f37142db24d7", "505d37ffc245dc095e7d0d4ec215ed28e4f0105d542e660c5e3ff9ba32bdd9da"), # coco_extra
|
||
}
|
||
|
||
|
||
def contract_holds(frozen: dict, current: dict) -> bool:
|
||
"""The current module still implements the frozen contract: identical, or an approved rewrite of the same module."""
|
||
return current == frozen or (
|
||
{**current, "source_sha256": None} == {**frozen, "source_sha256": None}
|
||
and (frozen.get("source_sha256"), current.get("source_sha256")) in EQUIVALENT_SOURCES)
|
||
|
||
|
||
def get(task: str):
|
||
"""One dispatch point. A task module owns inputs and both metric paths."""
|
||
import importlib
|
||
if task not in TASK_MODULES:
|
||
raise ValueError(f"task {task!r} evaluator not implemented")
|
||
module = importlib.import_module(TASK_MODULES[task])
|
||
for name in ("run_fp32", "run_quant"):
|
||
if not callable(getattr(module, name, None)):
|
||
raise ValueError(f"task {task!r} missing {name}")
|
||
return module
|
||
|
||
|
||
def task_contract(task: str) -> dict:
|
||
"""Freeze implementation identity without importing torch or a toolchain."""
|
||
import hashlib
|
||
from pathlib import Path
|
||
name = TASK_MODULES.get(task)
|
||
source = Path(__file__).resolve().parents[2] / (name.replace(".", "/") + ".py") if name else None
|
||
return {"task": task, "protocol": "bench.eval/1", "module": name,
|
||
"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source and source.is_file() else None,
|
||
"implemented": bool(name and source and source.is_file())}
|