data-gen-bpu/export/reward.py
bench 41eb1d2193 agent v17:提示词删掉「评测数据只读可见」;判分 v0.8:板上磁盘满和板子连不上一样免罚
- dev 环境 10-14 起只挂校准集,VM 上 bench/data 为空,agent 已看不到评测数据,提示词与实际不符。
- 用户 10-15 定:板上磁盘满(disk_full)与 board_offline 同等处理,不扣 G2/G3、不扣超时分。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 13:53:34 +08:00

68 lines
3.2 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""reward v0.8:只按本次真实门证据、成本、内容审核和超时计算。
不读取 Oracle 对照或 BLOCKED 正确性标签。BLOCKED 的已测阶段仍保留分数,
但导出器不将未证实的 BLOCKED 纳入奖励训练。板卡不可用(连不上或板上磁盘满)仍按既有规则免超时罚分。
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
RECIPE_VERSION = "v0.8"
BOARD_STAGES = ("07-board", "08-eval")
BOARD_UNAVAILABLE = ("board_offline", "disk_full") # v0.8: 10-15 S600 /userdata filled by kept sessions
PASSING = {"pass", "pass-screen", "pass-weak"}
W = {"G1": 1.0, "G2": 1.0, "G3": 2.0, "cost_turns": 0.5, "cost_tokens": 0.5,
"timeout": 1.0}
def board_unavailable(verdict: dict) -> bool:
"""板卡不可用导致 G2/G3 没法测:G3 unverified,且 07-board 或 08-eval 的最后一次尝试以 board_offline / disk_full 失败。
verdict.stages 只记最后一次 attempt(verify._latest_meta),所以"最后一次"就是这条轨迹结束时板子的状态。"""
g3 = (verdict.get("gates") or {}).get("G3") or {}
if g3.get("verdict") != "unverified":
return False
stages = verdict.get("stages") or {}
return any((stages.get(s) or {}).get("error_class") in BOARD_UNAVAILABLE for s in BOARD_STAGES)
def compute(verdict: dict, max_turns: int = 40, max_tokens: int = 200000) -> dict:
terms: dict[str, float] = {}
g = verdict.get("gates") or {}
terminal = verdict.get("terminal")
cost = verdict.get("cost") or {}
unverifiable = None
g3_na = (g.get("G3") or {}).get("verdict") == "na"
if board_unavailable(verdict):
unverifiable = "board_offline"
w1, w2, w3 = W["G1"], 0.0, 0.0
else:
w1, w2, w3 = (2.0, 2.0, 0.0) if g3_na else (W["G1"], W["G2"], W["G3"])
terms["G1"] = w1 * float((g.get("G1") or {}).get("verdict") in PASSING)
terms["G2"] = w2 * float((g.get("G2") or {}).get("verdict") in PASSING)
terms["G3"] = w3 * float((g.get("G3") or {}).get("verdict") in PASSING)
if (verdict.get("execution_terminal") == "TIMEOUT" or terminal == "TIMEOUT") and not unverifiable:
terms["timeout"] = -W["timeout"] # 板子不通时把预算耗在重试上,不算 agent 超时
turns, tokens = cost.get("turns") or 0, cost.get("tokens") or 0
if cost.get("tokens_reported") is False or tokens == 0:
# 端点不回 usage(dr-gateway 实测全 0):代价项只按 turns 算,权重合并,别把"没报"当成"很省"
terms["cost"] = -((W["cost_turns"] + W["cost_tokens"]) * min(1.0, turns / max_turns))
terms_note = "tokens unavailable"
else:
terms["cost"] = -(W["cost_turns"] * min(1.0, turns / max_turns) + W["cost_tokens"] * min(1.0, tokens / max_tokens))
terms_note = None
r = sum(terms.values())
return {"reward": round(r, 4), "terms": {k: round(v, 4) for k, v in terms.items()}, "recipe": RECIPE_VERSION,
"terminal": terminal, "note": terms_note,
"unverifiable": unverifiable}
if __name__ == "__main__":
for p in sys.argv[1:]:
v = json.loads(Path(p).read_text())
print(p, json.dumps(compute(v), ensure_ascii=False))