44 lines
1.8 KiB
Python
44 lines
1.8 KiB
Python
"""逐任务评测器。agent 不可写;verdict 的唯一依据。FP32 基线与量化实测走同一份代码、同一子集、同一预处理。
|
||
|
||
EVALUATOR_VERSION 任何口径变化都要升;job.json 钉死它,基线缓存按它失效。
|
||
"""
|
||
EVALUATOR_VERSION = "0.5.0"
|
||
# 0.5 新增真实板卡后端;FP32 的预处理、预测及指标公式与 0.4 完全相同。
|
||
COMPATIBLE_BASELINE_VERSIONS = {"0.4.0", EVALUATOR_VERSION}
|
||
|
||
TASK_MODULES = {
|
||
"cls": "harness.eval.cls",
|
||
"det": "harness.eval.det",
|
||
# Additive task support; cls/det preprocessing and metrics remain 0.5-compatible.
|
||
"iseg": "harness.eval.coco_extra",
|
||
"pose": "harness.eval.coco_extra",
|
||
"flow": "harness.eval.flow",
|
||
"seg": "harness.eval.seg",
|
||
"mde": "harness.eval.mde",
|
||
"feat": "harness.eval.feat",
|
||
# P4+:stereo occ
|
||
}
|
||
|
||
|
||
def get(task: str):
|
||
"""One dispatch point. A task module owns inputs and both metric paths."""
|
||
import importlib
|
||
if task not in TASK_MODULES:
|
||
raise ValueError(f"task {task!r} evaluator not implemented")
|
||
module = importlib.import_module(TASK_MODULES[task])
|
||
for name in ("run_fp32", "run_quant"):
|
||
if not callable(getattr(module, name, None)):
|
||
raise ValueError(f"task {task!r} missing {name}")
|
||
return module
|
||
|
||
|
||
def task_contract(task: str) -> dict:
|
||
"""Freeze implementation identity without importing torch or a toolchain."""
|
||
import hashlib
|
||
from pathlib import Path
|
||
name = TASK_MODULES.get(task)
|
||
source = Path(__file__).resolve().parents[2] / (name.replace(".", "/") + ".py") if name else None
|
||
return {"task": task, "protocol": "bench.eval/1", "module": name,
|
||
"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest() if source and source.is_file() else None,
|
||
"implemented": bool(name and source and source.is_file())}
|