67 lines
3.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""reward v0.7:只按本次真实门证据、成本、内容审核和超时计算。
不读取 Oracle 对照或 BLOCKED 正确性标签。BLOCKED 的已测阶段仍保留分数,
但导出器不将未证实的 BLOCKED 纳入奖励训练。板卡不可用仍按既有规则免超时罚分。
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
RECIPE_VERSION = "v0.7"
BOARD_STAGES = ("07-board", "08-eval")
PASSING = {"pass", "pass-screen", "pass-weak"}
W = {"G1": 1.0, "G2": 1.0, "G3": 2.0, "cost_turns": 0.5, "cost_tokens": 0.5,
"timeout": 1.0}
def board_unavailable(verdict: dict) -> bool:
"""板卡不可用导致 G2/G3 没法测:G3 unverified,且 07-board 或 08-eval 的最后一次尝试以 board_offline 失败。
verdict.stages 只记最后一次 attempt(verify._latest_meta),所以"最后一次"就是这条轨迹结束时板子的状态。"""
g3 = (verdict.get("gates") or {}).get("G3") or {}
if g3.get("verdict") != "unverified":
return False
stages = verdict.get("stages") or {}
return any((stages.get(s) or {}).get("error_class") == "board_offline" for s in BOARD_STAGES)
def compute(verdict: dict, max_turns: int = 40, max_tokens: int = 200000) -> dict:
terms: dict[str, float] = {}
g = verdict.get("gates") or {}
terminal = verdict.get("terminal")
cost = verdict.get("cost") or {}
unverifiable = None
g3_na = (g.get("G3") or {}).get("verdict") == "na"
if board_unavailable(verdict):
unverifiable = "board_offline"
w1, w2, w3 = W["G1"], 0.0, 0.0
else:
w1, w2, w3 = (2.0, 2.0, 0.0) if g3_na else (W["G1"], W["G2"], W["G3"])
terms["G1"] = w1 * float((g.get("G1") or {}).get("verdict") in PASSING)
terms["G2"] = w2 * float((g.get("G2") or {}).get("verdict") in PASSING)
terms["G3"] = w3 * float((g.get("G3") or {}).get("verdict") in PASSING)
if (verdict.get("execution_terminal") == "TIMEOUT" or terminal == "TIMEOUT") and not unverifiable:
terms["timeout"] = -W["timeout"] # 板子不通时把预算耗在重试上,不算 agent 超时
turns, tokens = cost.get("turns") or 0, cost.get("tokens") or 0
if cost.get("tokens_reported") is False or tokens == 0:
# 端点不回 usage(dr-gateway 实测全 0):代价项只按 turns 算,权重合并,别把"没报"当成"很省"
terms["cost"] = -((W["cost_turns"] + W["cost_tokens"]) * min(1.0, turns / max_turns))
terms_note = "tokens unavailable"
else:
terms["cost"] = -(W["cost_turns"] * min(1.0, turns / max_turns) + W["cost_tokens"] * min(1.0, tokens / max_tokens))
terms_note = None
r = sum(terms.values())
return {"reward": round(r, 4), "terms": {k: round(v, 4) for k, v in terms.items()}, "recipe": RECIPE_VERSION,
"terminal": terminal, "note": terms_note,
"unverifiable": unverifiable}
if __name__ == "__main__":
for p in sys.argv[1:]:
v = json.loads(Path(p).read_text())
print(p, json.dumps(compute(v), ensure_ascii=False))