Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
395 lines
20 KiB
Python
395 lines
20 KiB
Python
"""L3:对一次 rollout 独立核验 → verdict.json。reward 的唯一来源。
|
||
|
||
输入:rollout/stages/*/meta.json(harness 自己跑的,agent 改不了)、claim.json(agent 自报,不可信)、
|
||
job.json(冻结的任务门槛)、agent/events.jsonl(成本)。
|
||
不读 workspace 里 agent 写的任何"结果"。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import stat
|
||
import time
|
||
from collections import Counter
|
||
from contextlib import nullcontext
|
||
from pathlib import Path
|
||
|
||
from . import db, gates
|
||
VERDICT_SCHEMA = "bench.verdict/4"
|
||
VERIFICATION_RULE = "independent-rollout-fixed-tolerance-v3"
|
||
|
||
|
||
def eval_set(job: dict) -> dict | None:
|
||
"""评测口径来自本次任务快照。"""
|
||
data = job.get("data") or {}
|
||
t1 = data.get("t1") or {}
|
||
if not t1:
|
||
return None
|
||
return {"dataset": data.get("dataset"), "n": t1.get("n"),
|
||
"content_sha": t1.get("content_sha"), "manifest_sha": t1.get("manifest_sha"),
|
||
"evaluator_version": job.get("evaluator_version"), "eval_set_id": t1.get("eval_set_id")}
|
||
|
||
|
||
def _latest_meta(stage_root: Path, name: str) -> dict | None:
|
||
p = stage_root / name / "meta.json"
|
||
return json.loads(p.read_text()) if p.exists() else None
|
||
|
||
|
||
def _current_recipe(stage_root: Path) -> dict | None:
|
||
"""The latest Worker-accepted submission, including attempts that failed early."""
|
||
records = []
|
||
for file in stage_root.glob("[0-9][0-9]-*/a*/meta.json"):
|
||
meta = json.loads(file.read_text())
|
||
if meta.get("recipe"):
|
||
records.append(meta)
|
||
queued = []
|
||
for file in stage_root.glob("_queue/*/request.json"):
|
||
request = json.loads(file.read_text())
|
||
if request.get("recipe"):
|
||
queued.append(request)
|
||
if queued:
|
||
request = max(queued, key=lambda item: item.get("received_at", 0))
|
||
frozen = json.loads(Path(request["recipe"]).read_text())
|
||
submission = frozen["submission_sha256"]
|
||
records = [meta for meta in records if meta["recipe"].get("submission") == submission]
|
||
if not records:
|
||
return {"submission": submission, "reference": None, "runtime": None}
|
||
return max(records, key=lambda item: item.get("started_at", 0))["recipe"] if records else None
|
||
|
||
|
||
def protected_predictions_path(root: Path, ref: dict) -> Path:
|
||
"""Only regular files below a protected result root supply prediction bytes."""
|
||
if not isinstance(ref, dict) or not isinstance(ref.get("path"), str) or not re.fullmatch("[0-9a-f]{64}", ref.get("sha256", "")):
|
||
raise ValueError("INVALID_PREDICTION_REFERENCE")
|
||
path = Path(ref["path"])
|
||
root = root.absolute()
|
||
if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root):
|
||
raise ValueError("PREDICTION_PATH_OUTSIDE_PROTECTED_ROOT")
|
||
cursor = path
|
||
while cursor != root.parent:
|
||
if cursor.is_symlink():
|
||
raise ValueError("SYMLINK_PREDICTION_SOURCE")
|
||
cursor = cursor.parent
|
||
if not stat.S_ISREG(path.stat().st_mode):
|
||
raise ValueError("NONREGULAR_PREDICTION_SOURCE")
|
||
return path
|
||
|
||
|
||
def terminal_predictions_path(rd: Path, record: dict) -> Path:
|
||
return protected_predictions_path(rd.resolve() / "stages" / "_queue", record.get("predictions") or {})
|
||
|
||
|
||
def custom_terminal_result(rd: Path) -> dict:
|
||
"""Keep scores visible, without promoting unobserved model use to PASS."""
|
||
try:
|
||
source = rd / "terminal_result.json"
|
||
if source.is_symlink() or not stat.S_ISREG(source.stat().st_mode):
|
||
raise ValueError("INVALID_TERMINAL_RESULT_SOURCE")
|
||
record = json.loads(source.read_text())
|
||
if not isinstance(record, dict) or record.get("schema") != "bench.terminal/1":
|
||
raise ValueError("INVALID_TERMINAL_RESULT")
|
||
record = dict(record)
|
||
if record.get("predictions"):
|
||
path = terminal_predictions_path(rd, record)
|
||
if hashlib.sha256(path.read_bytes()).hexdigest() != record["predictions"]["sha256"]:
|
||
raise ValueError("PREDICTION_DIGEST_MISMATCH")
|
||
record.update(measurement_terminal="UNVERIFIED", model_execution_source="UNVERIFIED")
|
||
return record
|
||
except (OSError, ValueError, TypeError):
|
||
return {"measurement_terminal": "UNVERIFIED", "model_execution_source": "UNVERIFIED",
|
||
"reason": "INVALID_CUSTOM_EXECUTION_EVIDENCE"}
|
||
|
||
|
||
def collect_measurements(rd: Path, stage_root: Path | None = None) -> tuple[dict, dict]:
|
||
"""从 stages/<NN>/meta.json 汇总测量值;返回 (measurements, stage_summary)。
|
||
rollout 传 rollout 目录;oracle 传 stage_root=…/oracle/stages。06-hbm 的 G1 若 meta 里没有,回退读它的 stdout.log(就是那份 JSON)。"""
|
||
root = stage_root or (rd / "stages")
|
||
if stage_root is None and ((rd / "terminal_result.json").exists() or (rd / "terminal_result.json").is_symlink()):
|
||
# A formal custom submission supersedes prior stage evidence. The
|
||
# prototype observes command/output bytes, not arbitrary HBM use.
|
||
return {"CUSTOM": custom_terminal_result(rd)}, {"terminal": {"state": "UNVERIFIED"}}
|
||
meas, summ = {}, {}
|
||
identity = _current_recipe(root)
|
||
receipt_memo = {}
|
||
for name in ("01-fetch", "02-baseline", "03-export", "04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"):
|
||
m = _latest_meta(root, name)
|
||
if not m:
|
||
continue
|
||
summ[name] = {"state": m.get("state"), "error_class": m.get("error_class"), "attempts": m.get("attempt", 0) + 1}
|
||
if identity:
|
||
from .stages.base import _recipe_receipt
|
||
try:
|
||
m, _ = _recipe_receipt(root, name, identity, reference=(name in ("01-fetch", "02-baseline")), memo=receipt_memo)
|
||
summ[name] = {"state": m.get("state"), "error_class": m.get("error_class"), "attempts": m["attempt"] + 1}
|
||
except (OSError, ValueError, KeyError) as exc:
|
||
matching = m.get("recipe") == identity
|
||
summ[name]["state"] = m.get("state") if matching and m.get("state") in ("FAIL", "STALLED") else "UNVERIFIED"
|
||
summ[name]["recipe_error"] = str(exc)
|
||
continue
|
||
if m.get("state") in ("OK", "SKIPPED"):
|
||
for k, v in (m.get("measurements") or {}).items():
|
||
meas[k] = v
|
||
if name == "06-hbm" and "G1" not in meas and not identity:
|
||
so = root / name / f"a{m.get('attempt', 0):02d}" / "stdout.log"
|
||
try:
|
||
meas["G1"] = json.loads(so.read_text())
|
||
except Exception:
|
||
pass
|
||
prediction = (meas.get("G3") or {}).get("predictions")
|
||
if prediction:
|
||
try:
|
||
source = protected_predictions_path(root, prediction)
|
||
if hashlib.sha256(source.read_bytes()).hexdigest() != prediction["sha256"]:
|
||
raise ValueError("PREDICTION_DIGEST_MISMATCH")
|
||
except (OSError, ValueError, TypeError):
|
||
meas["G3"] = None
|
||
summ.setdefault("08-eval", {})["prediction_error"] = "INVALID_PREDICTION_SOURCE"
|
||
if "06-hbm" not in summ or summ["06-hbm"]["state"] != "OK":
|
||
meas.setdefault("G1", {"hbm_exists": False})
|
||
return meas, summ
|
||
|
||
|
||
def rollout_hbm_path(rd: Path, job_id: str) -> Path:
|
||
"""Prefer the protected rollout artifact; support the early workspace layout."""
|
||
identity = _current_recipe(rd / "stages")
|
||
if identity:
|
||
from .stages.base import _recipe_receipt
|
||
try:
|
||
meta, _ = _recipe_receipt(rd / "stages", "06-hbm", identity)
|
||
return Path(meta["recipe_outputs"]["hbm"]["path"])
|
||
except (OSError, ValueError, KeyError):
|
||
# Deliberately absent: never let a legacy public artifact repair a broken recipe chain.
|
||
return rd / "stages" / "_missing_recipe_hbm"
|
||
protected = rd / "artifacts" / f"{job_id}.hbm"
|
||
legacy = rd / "workspace" / "artifacts" / f"{job_id}.hbm"
|
||
return protected if protected.exists() or not legacy.exists() else legacy
|
||
|
||
|
||
def validate_hbm_measurements(measurements: dict, hbm: Path) -> tuple[dict, dict]:
|
||
"""Return accepted evidence and identity reasons without changing raw records.
|
||
|
||
06 attests the artifact checked by the compiler tools. 07/08 must refer to
|
||
that same artifact and to the bytes present now. Missing historical hashes
|
||
require revalidation; hashing today's file cannot repair yesterday's record.
|
||
"""
|
||
accepted = dict(measurements)
|
||
current, file_error = None, None
|
||
try:
|
||
digest = hashlib.sha256()
|
||
with hbm.open("rb") as stream:
|
||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||
digest.update(chunk)
|
||
current = digest.hexdigest()
|
||
except OSError as exc:
|
||
file_error = "current HBM missing" if isinstance(exc, FileNotFoundError) else "current HBM unreadable"
|
||
|
||
def recorded_sha(gate):
|
||
value = (measurements.get(gate) or {}).get("hbm_sha256")
|
||
return value.lower() if isinstance(value, str) and re.fullmatch(r"[0-9a-fA-F]{64}", value) else None
|
||
|
||
g1_sha = recorded_sha("G1")
|
||
g1_error = file_error or ("06-hbm hbm_sha256 missing or invalid" if not g1_sha else
|
||
"06-hbm hbm_sha256 differs from current HBM" if g1_sha != current else None)
|
||
identity = {"path": str(hbm), "sha256": current, "gates": {}}
|
||
for gate in ("G1", "G3", "G4"):
|
||
recorded = recorded_sha(gate)
|
||
reason = g1_error
|
||
if not reason and not recorded:
|
||
reason = f"{gate} hbm_sha256 missing or invalid"
|
||
if not reason and recorded != current:
|
||
reason = f"{gate} hbm_sha256 differs from current HBM / 06-hbm"
|
||
identity["gates"][gate] = {"status": "unverified" if reason else "verified",
|
||
"recorded_sha256": recorded, "reason": reason}
|
||
if reason:
|
||
# Keep an explicit stage failure; do not retain stale positive
|
||
# checks, accuracy or latency as accepted measurements.
|
||
accepted[gate] = ({"hbm_exists": False} if gate == "G1" and
|
||
(measurements.get(gate) or {}).get("hbm_exists") is False else None)
|
||
return accepted, identity
|
||
|
||
|
||
def _cost(rd: Path) -> dict:
|
||
"""成本只从 events.jsonl 数:turns = turn_end 事件数(别数 message_end,那会重复计三次)。
|
||
|
||
实测(2026-09-15,dr-gateway / glm-5.3-flash):网关回的 usage 全是 0。
|
||
所以 tokens 可能整体不可用 → 记 tokens_reported=false,reward 的代价项据此退化为只按 turns 算,
|
||
否则"没报 token"会被误读成"这条轨迹很省"。
|
||
"""
|
||
meta = rd / "rollout.json"
|
||
runner = json.loads(meta.read_text()).get("runner", "pi") if meta.exists() else "pi"
|
||
if runner != "pi":
|
||
from .runners import get
|
||
return get(runner).cost(rd)
|
||
ev = rd / "agent" / "events.jsonl"
|
||
turns = tokens = tool_calls = 0
|
||
t0 = t1 = None
|
||
if ev.exists():
|
||
with ev.open(errors="replace") as f:
|
||
for line in f:
|
||
try:
|
||
e = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
ts = e.get("timestamp") or e.get("ts") or (e.get("message") or {}).get("timestamp")
|
||
if isinstance(ts, (int, float)):
|
||
t0 = t0 or ts; t1 = ts
|
||
typ = e.get("type", "")
|
||
if typ == "turn_end":
|
||
turns += 1
|
||
u = ((e.get("message") or {}).get("usage")) or e.get("usage")
|
||
if isinstance(u, dict):
|
||
tokens += int(u.get("input", 0) or u.get("input_tokens", 0) or 0)
|
||
tokens += int(u.get("output", 0) or u.get("output_tokens", 0) or 0)
|
||
elif typ == "tool_execution_start":
|
||
tool_calls += 1
|
||
wall = None
|
||
if t0 and t1:
|
||
try:
|
||
wall = (float(t1) - float(t0)) / 1000.0 if float(t1) > 1e11 else float(t1) - float(t0)
|
||
except (TypeError, ValueError):
|
||
wall = None
|
||
return {"turns": turns, "tokens": tokens, "tokens_reported": tokens > 0,
|
||
"tool_calls": tool_calls, "wall_s": round(wall, 1) if wall else None}
|
||
|
||
|
||
def completion_summary(gate_results: dict, stages: dict) -> dict:
|
||
"""实际完成了什么,与质量门是否通过、Agent 是否超预算分开统计。"""
|
||
g1 = gate_results.get("G1") or {}
|
||
g3 = gate_results.get("G3") or {}
|
||
g4 = gate_results.get("G4") or {}
|
||
hbm = g1.get("hbm_exists") is True
|
||
measured = (g3.get("source") == "board" and g3.get("quant") is not None
|
||
and g3.get("fp32") is not None and g3.get("verdict") in gates.PASSING | {"fail"})
|
||
board = measured or ((stages.get("07-board") or {}).get("state") == "OK"
|
||
and g4.get("latency_ms_mean") is not None)
|
||
compiled = (stages.get("05-compile") or {}).get("state") == "OK" or hbm
|
||
return {"compiled": compiled, "hbm_generated": hbm, "board_completed": board,
|
||
"accuracy_evaluated": measured, "accuracy_passed": measured and g3.get("verdict") in gates.PASSING,
|
||
"workflow_completed": compiled and hbm and board and measured}
|
||
|
||
|
||
def summarize_results(results) -> dict:
|
||
"""读取旧、新结果都按已完成阶段计数,不把超预算当成部署失败。"""
|
||
counts = Counter()
|
||
for result in results:
|
||
if isinstance(result, str):
|
||
result = json.loads(result)
|
||
if not result:
|
||
continue
|
||
counts["recorded"] += 1
|
||
completion = result.get("completion") or completion_summary(result.get("gates") or {}, result.get("stages") or {})
|
||
counts.update({k: int(v) for k, v in completion.items()})
|
||
counts["agent_timeout"] += (result.get("execution_terminal") or result.get("terminal")) == "TIMEOUT"
|
||
return dict(counts)
|
||
|
||
|
||
def _execution_control(owner, metadata, measured_terminal):
|
||
"""Called under execution.lock. Legacy untracked rollouts do not enter here."""
|
||
try:
|
||
state = owner.snapshot()
|
||
except (OSError, ValueError):
|
||
return {'db_state': 'INFRA_ERROR', 'reason': 'EXECUTION_RECORD_UNAVAILABLE'}
|
||
if not isinstance(state, dict) or not state.get('execution_id') or state['execution_id'] != metadata.get('execution_id'):
|
||
return {'db_state': 'INFRA_ERROR', 'reason': 'EXECUTION_IDENTITY_MISMATCH'}
|
||
status, outcome, cleanup = (state.get(key) for key in ('execution_state', 'execution_outcome', 'cleanup_state'))
|
||
held = {'CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST'}
|
||
if cleanup == 'QUARANTINED' or (status != 'TERMINAL' and outcome is not None):
|
||
db_state = 'INFRA_ERROR'
|
||
elif status == 'TERMINAL':
|
||
if outcome in held:
|
||
db_state = outcome
|
||
elif (outcome in ('COMPLETED', 'AGENT_ERROR') and cleanup == 'VERIFIED'
|
||
and (outcome != 'COMPLETED' or state.get('protocol_complete') is True)):
|
||
db_state = measured_terminal
|
||
else:
|
||
db_state = 'INFRA_ERROR'
|
||
else:
|
||
db_state = {'ACTIVE': 'RUNNING', 'PAUSED_INFRA': 'PAUSED_INFRA', 'CLEANING': 'CLEANING',
|
||
'CANCEL_REQUESTED': 'CLEANING', 'STOP_REQUESTED': 'CLEANING'}.get(status, 'INFRA_ERROR')
|
||
return {'execution_id': state['execution_id'], 'execution_state': status, 'execution_outcome': outcome,
|
||
'cleanup_state': cleanup, 'db_state': db_state,
|
||
'reason': 'EXECUTION_HOLD' if db_state != measured_terminal else 'MEASURED_TERMINAL'}
|
||
|
||
|
||
def verify_rollout(rd: Path, write: bool = True, *, timeout_s=None) -> dict:
|
||
rd = rd.resolve()
|
||
jd = rd.parent.parent
|
||
job = json.loads((jd / "job.json").read_text())
|
||
kk = int(rd.name)
|
||
claim, claim_read_error = None, None
|
||
try:
|
||
if (rd / "claim.json").exists():
|
||
claim = json.loads((rd / "claim.json").read_text())
|
||
if not isinstance(claim, dict):
|
||
claim, claim_read_error = None, "INVALID_OPTIONAL_SELF_REPORT"
|
||
except (OSError, ValueError):
|
||
claim_read_error = "INVALID_OPTIONAL_SELF_REPORT"
|
||
rmeta = json.loads((rd / "rollout.json").read_text()) if (rd / "rollout.json").exists() else {}
|
||
|
||
meas, summ = collect_measurements(rd)
|
||
# 没有执行 HBM 检查不等于检查失败(例如 Agent 在编译后耗尽预算)。
|
||
if "06-hbm" not in summ:
|
||
meas.pop("G1", None)
|
||
meas, provenance = validate_hbm_measurements(meas, rollout_hbm_path(rd, job["job_id"]))
|
||
ev = gates.evaluate(meas, job["gates"], provenance=provenance)
|
||
terminal = ev["terminal"]
|
||
failed_stage = next((s for name, s in summ.items() if name in ("04-calibrate", "05-compile", "06-hbm")
|
||
and s.get("state") in ("FAIL", "STALLED")), None)
|
||
if terminal == "UNVERIFIED" and failed_stage:
|
||
terminal = "FAIL"
|
||
|
||
# TIMEOUT 是执行终止原因,不能抹掉独立测量的 PASS/FAIL/UNVERIFIED。
|
||
measured_terminal = terminal
|
||
claim_consistent = None
|
||
if claim:
|
||
claim_consistent = (claim.get("terminal") == terminal) if terminal in ("PASS", "FAIL", "BLOCKED") else None
|
||
|
||
v_ec = gates.classify_gate_failure(ev) if terminal == "FAIL" else None
|
||
for st in summ.values():
|
||
if st.get("state") not in ("OK", "SKIPPED") and st.get("error_class"):
|
||
v_ec = st["error_class"]; break
|
||
verdict = {
|
||
"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE,
|
||
"scoring_rule": gates.SCORING_RULE,
|
||
"error_class": v_ec,
|
||
"job_id": job["job_id"], "run_id": job["run_id"], "rollout": kk,
|
||
"eval_set": eval_set(job),
|
||
"continuation": job.get("continuation"),
|
||
"mode": rmeta.get("mode"), "policy": rmeta.get("policy"),
|
||
"gates": ev["gates"], "required": ev["required"],
|
||
"hbm_provenance": provenance,
|
||
"custom_execution": meas.get("CUSTOM"),
|
||
"terminal": terminal, "measured_terminal": measured_terminal,
|
||
"execution_terminal": rmeta.get("state"),
|
||
"timeout_reason": rmeta.get("timeout_reason"),
|
||
"completion": completion_summary(ev["gates"], summ),
|
||
"failed_gate": ev["failed_gate"],
|
||
"stages": summ,
|
||
"claim": claim, "claim_consistent": claim_consistent, "claim_read_error": claim_read_error,
|
||
# BLOCKED is a self-report, not proof that the task is impossible.
|
||
"blocked_correct": None,
|
||
"review": False,
|
||
"cost": _cost(rd),
|
||
"verified_at": time.time(),
|
||
}
|
||
if write:
|
||
from .execution import Execution
|
||
owner = Execution(rd) if (rd / 'execution.json').exists() or rmeta.get('execution_managed') else None
|
||
# The same lock serializes execution stop/cancel and this DB write. A gate
|
||
# result cannot reopen admission by overwriting the control-plane state.
|
||
with owner.locked() if owner else nullcontext():
|
||
control = _execution_control(owner, rmeta, terminal) if owner else None
|
||
if control:
|
||
verdict['execution_control'] = control
|
||
(rd / "verdict.json").write_text(json.dumps(verdict, indent=2, ensure_ascii=False, default=str))
|
||
with db.connect(**({'timeout_s': timeout_s} if timeout_s is not None else {})) as con:
|
||
db.rollout_upsert(con, job["run_id"], job["job_id"], kk, rmeta.get("mode"), rmeta.get("policy"), runner=rmeta.get("runner", "pi"))
|
||
db.rollout_set(con, job["run_id"], job["job_id"], kk, state=control['db_state'] if control else terminal,
|
||
claim_terminal=(claim or {}).get("terminal"), verdict_terminal=terminal,
|
||
claim_consistent=None if claim_consistent is None else int(claim_consistent),
|
||
review=0, turns=verdict["cost"]["turns"], tokens=verdict["cost"]["tokens"],
|
||
wall_s=verdict["cost"]["wall_s"], verdict=json.dumps(verdict, default=str),
|
||
finished_at=rmeta.get("finished_at") or time.time())
|
||
return verdict
|