ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

395 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""L3:对一次 rollout 独立核验 → verdict.json。reward 的唯一来源。
输入:rollout/stages/*/meta.json(harness 自己跑的,agent 改不了)、claim.json(agent 自报,不可信)、
job.json(冻结的任务门槛)、agent/events.jsonl(成本)。
不读 workspace 里 agent 写的任何"结果"。
"""
from __future__ import annotations
import hashlib
import json
import re
import stat
import time
from collections import Counter
from contextlib import nullcontext
from pathlib import Path
from . import db, gates
VERDICT_SCHEMA = "bench.verdict/4"
VERIFICATION_RULE = "independent-rollout-fixed-tolerance-v3"
def eval_set(job: dict) -> dict | None:
"""评测口径来自本次任务快照。"""
data = job.get("data") or {}
t1 = data.get("t1") or {}
if not t1:
return None
return {"dataset": data.get("dataset"), "n": t1.get("n"),
"content_sha": t1.get("content_sha"), "manifest_sha": t1.get("manifest_sha"),
"evaluator_version": job.get("evaluator_version"), "eval_set_id": t1.get("eval_set_id")}
def _latest_meta(stage_root: Path, name: str) -> dict | None:
p = stage_root / name / "meta.json"
return json.loads(p.read_text()) if p.exists() else None
def _current_recipe(stage_root: Path) -> dict | None:
"""The latest Worker-accepted submission, including attempts that failed early."""
records = []
for file in stage_root.glob("[0-9][0-9]-*/a*/meta.json"):
meta = json.loads(file.read_text())
if meta.get("recipe"):
records.append(meta)
queued = []
for file in stage_root.glob("_queue/*/request.json"):
request = json.loads(file.read_text())
if request.get("recipe"):
queued.append(request)
if queued:
request = max(queued, key=lambda item: item.get("received_at", 0))
frozen = json.loads(Path(request["recipe"]).read_text())
submission = frozen["submission_sha256"]
records = [meta for meta in records if meta["recipe"].get("submission") == submission]
if not records:
return {"submission": submission, "reference": None, "runtime": None}
return max(records, key=lambda item: item.get("started_at", 0))["recipe"] if records else None
def protected_predictions_path(root: Path, ref: dict) -> Path:
"""Only regular files below a protected result root supply prediction bytes."""
if not isinstance(ref, dict) or not isinstance(ref.get("path"), str) or not re.fullmatch("[0-9a-f]{64}", ref.get("sha256", "")):
raise ValueError("INVALID_PREDICTION_REFERENCE")
path = Path(ref["path"])
root = root.absolute()
if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root):
raise ValueError("PREDICTION_PATH_OUTSIDE_PROTECTED_ROOT")
cursor = path
while cursor != root.parent:
if cursor.is_symlink():
raise ValueError("SYMLINK_PREDICTION_SOURCE")
cursor = cursor.parent
if not stat.S_ISREG(path.stat().st_mode):
raise ValueError("NONREGULAR_PREDICTION_SOURCE")
return path
def terminal_predictions_path(rd: Path, record: dict) -> Path:
return protected_predictions_path(rd.resolve() / "stages" / "_queue", record.get("predictions") or {})
def custom_terminal_result(rd: Path) -> dict:
"""Keep scores visible, without promoting unobserved model use to PASS."""
try:
source = rd / "terminal_result.json"
if source.is_symlink() or not stat.S_ISREG(source.stat().st_mode):
raise ValueError("INVALID_TERMINAL_RESULT_SOURCE")
record = json.loads(source.read_text())
if not isinstance(record, dict) or record.get("schema") != "bench.terminal/1":
raise ValueError("INVALID_TERMINAL_RESULT")
record = dict(record)
if record.get("predictions"):
path = terminal_predictions_path(rd, record)
if hashlib.sha256(path.read_bytes()).hexdigest() != record["predictions"]["sha256"]:
raise ValueError("PREDICTION_DIGEST_MISMATCH")
record.update(measurement_terminal="UNVERIFIED", model_execution_source="UNVERIFIED")
return record
except (OSError, ValueError, TypeError):
return {"measurement_terminal": "UNVERIFIED", "model_execution_source": "UNVERIFIED",
"reason": "INVALID_CUSTOM_EXECUTION_EVIDENCE"}
def collect_measurements(rd: Path, stage_root: Path | None = None) -> tuple[dict, dict]:
"""从 stages/<NN>/meta.json 汇总测量值;返回 (measurements, stage_summary)。
rollout 传 rollout 目录;oracle 传 stage_root=…/oracle/stages。06-hbm 的 G1 若 meta 里没有,回退读它的 stdout.log(就是那份 JSON)。"""
root = stage_root or (rd / "stages")
if stage_root is None and ((rd / "terminal_result.json").exists() or (rd / "terminal_result.json").is_symlink()):
# A formal custom submission supersedes prior stage evidence. The
# prototype observes command/output bytes, not arbitrary HBM use.
return {"CUSTOM": custom_terminal_result(rd)}, {"terminal": {"state": "UNVERIFIED"}}
meas, summ = {}, {}
identity = _current_recipe(root)
receipt_memo = {}
for name in ("01-fetch", "02-baseline", "03-export", "04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"):
m = _latest_meta(root, name)
if not m:
continue
summ[name] = {"state": m.get("state"), "error_class": m.get("error_class"), "attempts": m.get("attempt", 0) + 1}
if identity:
from .stages.base import _recipe_receipt
try:
m, _ = _recipe_receipt(root, name, identity, reference=(name in ("01-fetch", "02-baseline")), memo=receipt_memo)
summ[name] = {"state": m.get("state"), "error_class": m.get("error_class"), "attempts": m["attempt"] + 1}
except (OSError, ValueError, KeyError) as exc:
matching = m.get("recipe") == identity
summ[name]["state"] = m.get("state") if matching and m.get("state") in ("FAIL", "STALLED") else "UNVERIFIED"
summ[name]["recipe_error"] = str(exc)
continue
if m.get("state") in ("OK", "SKIPPED"):
for k, v in (m.get("measurements") or {}).items():
meas[k] = v
if name == "06-hbm" and "G1" not in meas and not identity:
so = root / name / f"a{m.get('attempt', 0):02d}" / "stdout.log"
try:
meas["G1"] = json.loads(so.read_text())
except Exception:
pass
prediction = (meas.get("G3") or {}).get("predictions")
if prediction:
try:
source = protected_predictions_path(root, prediction)
if hashlib.sha256(source.read_bytes()).hexdigest() != prediction["sha256"]:
raise ValueError("PREDICTION_DIGEST_MISMATCH")
except (OSError, ValueError, TypeError):
meas["G3"] = None
summ.setdefault("08-eval", {})["prediction_error"] = "INVALID_PREDICTION_SOURCE"
if "06-hbm" not in summ or summ["06-hbm"]["state"] != "OK":
meas.setdefault("G1", {"hbm_exists": False})
return meas, summ
def rollout_hbm_path(rd: Path, job_id: str) -> Path:
"""Prefer the protected rollout artifact; support the early workspace layout."""
identity = _current_recipe(rd / "stages")
if identity:
from .stages.base import _recipe_receipt
try:
meta, _ = _recipe_receipt(rd / "stages", "06-hbm", identity)
return Path(meta["recipe_outputs"]["hbm"]["path"])
except (OSError, ValueError, KeyError):
# Deliberately absent: never let a legacy public artifact repair a broken recipe chain.
return rd / "stages" / "_missing_recipe_hbm"
protected = rd / "artifacts" / f"{job_id}.hbm"
legacy = rd / "workspace" / "artifacts" / f"{job_id}.hbm"
return protected if protected.exists() or not legacy.exists() else legacy
def validate_hbm_measurements(measurements: dict, hbm: Path) -> tuple[dict, dict]:
"""Return accepted evidence and identity reasons without changing raw records.
06 attests the artifact checked by the compiler tools. 07/08 must refer to
that same artifact and to the bytes present now. Missing historical hashes
require revalidation; hashing today's file cannot repair yesterday's record.
"""
accepted = dict(measurements)
current, file_error = None, None
try:
digest = hashlib.sha256()
with hbm.open("rb") as stream:
for chunk in iter(lambda: stream.read(1 << 20), b""):
digest.update(chunk)
current = digest.hexdigest()
except OSError as exc:
file_error = "current HBM missing" if isinstance(exc, FileNotFoundError) else "current HBM unreadable"
def recorded_sha(gate):
value = (measurements.get(gate) or {}).get("hbm_sha256")
return value.lower() if isinstance(value, str) and re.fullmatch(r"[0-9a-fA-F]{64}", value) else None
g1_sha = recorded_sha("G1")
g1_error = file_error or ("06-hbm hbm_sha256 missing or invalid" if not g1_sha else
"06-hbm hbm_sha256 differs from current HBM" if g1_sha != current else None)
identity = {"path": str(hbm), "sha256": current, "gates": {}}
for gate in ("G1", "G3", "G4"):
recorded = recorded_sha(gate)
reason = g1_error
if not reason and not recorded:
reason = f"{gate} hbm_sha256 missing or invalid"
if not reason and recorded != current:
reason = f"{gate} hbm_sha256 differs from current HBM / 06-hbm"
identity["gates"][gate] = {"status": "unverified" if reason else "verified",
"recorded_sha256": recorded, "reason": reason}
if reason:
# Keep an explicit stage failure; do not retain stale positive
# checks, accuracy or latency as accepted measurements.
accepted[gate] = ({"hbm_exists": False} if gate == "G1" and
(measurements.get(gate) or {}).get("hbm_exists") is False else None)
return accepted, identity
def _cost(rd: Path) -> dict:
"""成本只从 events.jsonl 数:turns = turn_end 事件数(别数 message_end,那会重复计三次)。
实测(2026-09-15,dr-gateway / glm-5.3-flash):网关回的 usage 全是 0。
所以 tokens 可能整体不可用 → 记 tokens_reported=false,reward 的代价项据此退化为只按 turns 算,
否则"没报 token"会被误读成"这条轨迹很省"。
"""
meta = rd / "rollout.json"
runner = json.loads(meta.read_text()).get("runner", "pi") if meta.exists() else "pi"
if runner != "pi":
from .runners import get
return get(runner).cost(rd)
ev = rd / "agent" / "events.jsonl"
turns = tokens = tool_calls = 0
t0 = t1 = None
if ev.exists():
with ev.open(errors="replace") as f:
for line in f:
try:
e = json.loads(line)
except json.JSONDecodeError:
continue
ts = e.get("timestamp") or e.get("ts") or (e.get("message") or {}).get("timestamp")
if isinstance(ts, (int, float)):
t0 = t0 or ts; t1 = ts
typ = e.get("type", "")
if typ == "turn_end":
turns += 1
u = ((e.get("message") or {}).get("usage")) or e.get("usage")
if isinstance(u, dict):
tokens += int(u.get("input", 0) or u.get("input_tokens", 0) or 0)
tokens += int(u.get("output", 0) or u.get("output_tokens", 0) or 0)
elif typ == "tool_execution_start":
tool_calls += 1
wall = None
if t0 and t1:
try:
wall = (float(t1) - float(t0)) / 1000.0 if float(t1) > 1e11 else float(t1) - float(t0)
except (TypeError, ValueError):
wall = None
return {"turns": turns, "tokens": tokens, "tokens_reported": tokens > 0,
"tool_calls": tool_calls, "wall_s": round(wall, 1) if wall else None}
def completion_summary(gate_results: dict, stages: dict) -> dict:
"""实际完成了什么,与质量门是否通过、Agent 是否超预算分开统计。"""
g1 = gate_results.get("G1") or {}
g3 = gate_results.get("G3") or {}
g4 = gate_results.get("G4") or {}
hbm = g1.get("hbm_exists") is True
measured = (g3.get("source") == "board" and g3.get("quant") is not None
and g3.get("fp32") is not None and g3.get("verdict") in gates.PASSING | {"fail"})
board = measured or ((stages.get("07-board") or {}).get("state") == "OK"
and g4.get("latency_ms_mean") is not None)
compiled = (stages.get("05-compile") or {}).get("state") == "OK" or hbm
return {"compiled": compiled, "hbm_generated": hbm, "board_completed": board,
"accuracy_evaluated": measured, "accuracy_passed": measured and g3.get("verdict") in gates.PASSING,
"workflow_completed": compiled and hbm and board and measured}
def summarize_results(results) -> dict:
"""读取旧、新结果都按已完成阶段计数,不把超预算当成部署失败。"""
counts = Counter()
for result in results:
if isinstance(result, str):
result = json.loads(result)
if not result:
continue
counts["recorded"] += 1
completion = result.get("completion") or completion_summary(result.get("gates") or {}, result.get("stages") or {})
counts.update({k: int(v) for k, v in completion.items()})
counts["agent_timeout"] += (result.get("execution_terminal") or result.get("terminal")) == "TIMEOUT"
return dict(counts)
def _execution_control(owner, metadata, measured_terminal):
"""Called under execution.lock. Legacy untracked rollouts do not enter here."""
try:
state = owner.snapshot()
except (OSError, ValueError):
return {'db_state': 'INFRA_ERROR', 'reason': 'EXECUTION_RECORD_UNAVAILABLE'}
if not isinstance(state, dict) or not state.get('execution_id') or state['execution_id'] != metadata.get('execution_id'):
return {'db_state': 'INFRA_ERROR', 'reason': 'EXECUTION_IDENTITY_MISMATCH'}
status, outcome, cleanup = (state.get(key) for key in ('execution_state', 'execution_outcome', 'cleanup_state'))
held = {'CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST'}
if cleanup == 'QUARANTINED' or (status != 'TERMINAL' and outcome is not None):
db_state = 'INFRA_ERROR'
elif status == 'TERMINAL':
if outcome in held:
db_state = outcome
elif (outcome in ('COMPLETED', 'AGENT_ERROR') and cleanup == 'VERIFIED'
and (outcome != 'COMPLETED' or state.get('protocol_complete') is True)):
db_state = measured_terminal
else:
db_state = 'INFRA_ERROR'
else:
db_state = {'ACTIVE': 'RUNNING', 'PAUSED_INFRA': 'PAUSED_INFRA', 'CLEANING': 'CLEANING',
'CANCEL_REQUESTED': 'CLEANING', 'STOP_REQUESTED': 'CLEANING'}.get(status, 'INFRA_ERROR')
return {'execution_id': state['execution_id'], 'execution_state': status, 'execution_outcome': outcome,
'cleanup_state': cleanup, 'db_state': db_state,
'reason': 'EXECUTION_HOLD' if db_state != measured_terminal else 'MEASURED_TERMINAL'}
def verify_rollout(rd: Path, write: bool = True, *, timeout_s=None) -> dict:
rd = rd.resolve()
jd = rd.parent.parent
job = json.loads((jd / "job.json").read_text())
kk = int(rd.name)
claim, claim_read_error = None, None
try:
if (rd / "claim.json").exists():
claim = json.loads((rd / "claim.json").read_text())
if not isinstance(claim, dict):
claim, claim_read_error = None, "INVALID_OPTIONAL_SELF_REPORT"
except (OSError, ValueError):
claim_read_error = "INVALID_OPTIONAL_SELF_REPORT"
rmeta = json.loads((rd / "rollout.json").read_text()) if (rd / "rollout.json").exists() else {}
meas, summ = collect_measurements(rd)
# 没有执行 HBM 检查不等于检查失败(例如 Agent 在编译后耗尽预算)。
if "06-hbm" not in summ:
meas.pop("G1", None)
meas, provenance = validate_hbm_measurements(meas, rollout_hbm_path(rd, job["job_id"]))
ev = gates.evaluate(meas, job["gates"], provenance=provenance)
terminal = ev["terminal"]
failed_stage = next((s for name, s in summ.items() if name in ("04-calibrate", "05-compile", "06-hbm")
and s.get("state") in ("FAIL", "STALLED")), None)
if terminal == "UNVERIFIED" and failed_stage:
terminal = "FAIL"
# TIMEOUT 是执行终止原因,不能抹掉独立测量的 PASS/FAIL/UNVERIFIED。
measured_terminal = terminal
claim_consistent = None
if claim:
claim_consistent = (claim.get("terminal") == terminal) if terminal in ("PASS", "FAIL", "BLOCKED") else None
v_ec = gates.classify_gate_failure(ev) if terminal == "FAIL" else None
for st in summ.values():
if st.get("state") not in ("OK", "SKIPPED") and st.get("error_class"):
v_ec = st["error_class"]; break
verdict = {
"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE,
"scoring_rule": gates.SCORING_RULE,
"error_class": v_ec,
"job_id": job["job_id"], "run_id": job["run_id"], "rollout": kk,
"eval_set": eval_set(job),
"continuation": job.get("continuation"),
"mode": rmeta.get("mode"), "policy": rmeta.get("policy"),
"gates": ev["gates"], "required": ev["required"],
"hbm_provenance": provenance,
"custom_execution": meas.get("CUSTOM"),
"terminal": terminal, "measured_terminal": measured_terminal,
"execution_terminal": rmeta.get("state"),
"timeout_reason": rmeta.get("timeout_reason"),
"completion": completion_summary(ev["gates"], summ),
"failed_gate": ev["failed_gate"],
"stages": summ,
"claim": claim, "claim_consistent": claim_consistent, "claim_read_error": claim_read_error,
# BLOCKED is a self-report, not proof that the task is impossible.
"blocked_correct": None,
"review": False,
"cost": _cost(rd),
"verified_at": time.time(),
}
if write:
from .execution import Execution
owner = Execution(rd) if (rd / 'execution.json').exists() or rmeta.get('execution_managed') else None
# The same lock serializes execution stop/cancel and this DB write. A gate
# result cannot reopen admission by overwriting the control-plane state.
with owner.locked() if owner else nullcontext():
control = _execution_control(owner, rmeta, terminal) if owner else None
if control:
verdict['execution_control'] = control
(rd / "verdict.json").write_text(json.dumps(verdict, indent=2, ensure_ascii=False, default=str))
with db.connect(**({'timeout_s': timeout_s} if timeout_s is not None else {})) as con:
db.rollout_upsert(con, job["run_id"], job["job_id"], kk, rmeta.get("mode"), rmeta.get("policy"), runner=rmeta.get("runner", "pi"))
db.rollout_set(con, job["run_id"], job["job_id"], kk, state=control['db_state'] if control else terminal,
claim_terminal=(claim or {}).get("terminal"), verdict_terminal=terminal,
claim_consistent=None if claim_consistent is None else int(claim_consistent),
review=0, turns=verdict["cost"]["turns"], tokens=verdict["cost"]["tokens"],
wall_s=verdict["cost"]["wall_s"], verdict=json.dumps(verdict, default=str),
finished_at=rmeta.get("finished_at") or time.time())
return verdict