#!/usr/bin/env python3 """audit_rollouts —— 一条 rollout 到底能不能当训练数据?每条一行,给出结论和原因。 python3 scripts/audit_rollouts.py --run r20260915-03 # 表格 python3 scripts/audit_rollouts.py --run r20260915-03 --json # 机器可读 python3 scripts/audit_rollouts.py --run r20260915-03 --annotate # 把"污染"结论写进 annotations 表(幂等) 判据(全部来自轨迹本身 events.jsonl + verdict.json + guard.log,不看 claim 的自述): oracle_read agent 成功读到 /oracle/…(e2e 抄答案) → RL/SFT 都不能用 sibling_read 成功读到别的 rollout → 同上(GRPO 组内独立性没了) harness_read 成功读到 harness/ 或工具链 site-packages → 不算污染,但是坏习惯;记数 stalled_killed 05-compile 被看门狗按 900 s 误杀(error_class=stalled)→ 这条 TIMEOUT 不是 agent 的错,不能当负样本 old_version agent_config_version < --min-version(默认 agent/VERSION)→ 旧口径,导出默认过滤 claim_absent / claim_mismatch → 可以当负样本(reward 已扣),但 SFT 不收 "成功读到"= 该次工具调用的结果里没有 path-guard 拒绝语;被 guard 拦下的尝试单独计入 blocked_attempts(不算污染)。 结论列: RL ✓ = 初步候选(独立执行 + 结果可核验 + 版本合格 + 不是被误杀) SFT ✓ = RL ✓ 且 verdict=PASS 且 claim 一致;完整性以导出器为准 """ from __future__ import annotations import argparse import json import os import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from harness import paths, db # noqa: E402 ORACLE_RX = re.compile(r"(^|[\s/'\"=(])oracle/|/oracle(/|$|[\s'\")])") HARNESS_RX = re.compile(r"/bench/(harness|export)/|site-packages/(hmct|horizon_tc_ui|hbdk)") TOOLCHAIN_RX = re.compile(r"toolchains/") def _text(x) -> str: if isinstance(x, str): return x if isinstance(x, list): return " ".join(_text(c) for c in x) if isinstance(x, dict): return " ".join(_text(v) for k, v in x.items() if k in ("text", "content", "command", "path", "file_path", "output")) return "" DECIMAL = re.compile(r'(? 24): needles.add(text) return {n for n in needles if n not in public} def scan_events(rd: Path, kk: int) -> dict: ev = rd / "agent" / "events.jsonl" out = {"turns": 0, "tool_calls": 0, "oracle_read": 0, "sibling_read": 0, "harness_read": 0, "toolchain_read": 0, "oracle_content_match": 0, "state_access": 0, "other_job_read": 0, "blocked_attempts": 0, "claim_calls": 0, "has_agent_end": False} if not ev.exists(): out["no_events"] = True return out sib_rx = re.compile(r"/rollouts/(?!%02d(/|$))\d\d(/|$)" % kk) needles = oracle_needles(rd) run_name = rd.parents[2].name job_name = rd.parents[1].name other_rx = re.compile(r'runs/' + re.escape(run_name) + r'/([^/\s\x22\x27]+)') independently_seen = set() pending_names = {} pending: dict[str, str] = {} # toolCallId → 目标字符串 with ev.open(errors="replace") as stream: lines = stream.readlines() for line in lines: try: e = json.loads(line) except json.JSONDecodeError: continue t = e.get("type") if t == "turn_end": out["turns"] += 1 elif t == "tool_execution_start": out["tool_calls"] += 1 name = e.get("toolName") or "" args = e.get("args") or {} if name == "claim": out["claim_calls"] += 1 pending_names[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = name target = " ".join(str(args.get(k, "")) for k in ("command", "path", "file_path", "pattern", "query")) pending[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = target elif t == "tool_execution_end": cid = str(e.get("toolCallId") or e.get("id") or "") target = pending.pop(cid, "") res = _text(e.get("result")) blocked = "path-guard" in res if blocked: out["blocked_attempts"] += 1 continue # Only actual tool results, excluding guard denials / failed filesystem probes. full_result = json.dumps(e.get('result'), ensure_ascii=False) tool_name = pending_names.pop(cid, '') # Independently executed stages can reproduce the oracle exactly. # Exempt only a single wait/status command for an existing own handle; # claim returns the agent's own input, so it is not an external read. wait_cmd = re.fullmatch(r'bench stage (?:wait|status) ([a-zA-Z0-9_-]+)(?: --poll \d+)?(?: 2>&1)?(?: \| tail -\d+)?', target.strip()) own_stage = bool(wait_cmd and (rd / 'handles' / wait_cmd[1] / 'request.json').is_file()) if own_stage: independently_seen.update(n for n in needles if n in full_result) if tool_name != 'claim' and not own_stage and any(n in full_result for n in needles - independently_seen): out['oracle_content_match'] += 1 if STATE_RX.search(target): out['state_access'] += 1 if any(m[1] != job_name for m in other_rx.finditer(target)): out['other_job_read'] += 1 if ORACLE_RX.search(target): out["oracle_read"] += 1 if sib_rx.search(target): out["sibling_read"] += 1 if HARNESS_RX.search(target): out["harness_read"] += 1 if TOOLCHAIN_RX.search(target): out["toolchain_read"] += 1 elif t == "agent_end": out["has_agent_end"] = True return out def load(p: Path) -> dict: try: return json.loads(p.read_text()) except Exception: return {} def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict: rj, vj, cj = load(rd / "rollout.json"), load(rd / "verdict.json"), load(rd / "claim.json") s = scan_events(rd, kk) ver = int(str(rj.get("agent_config_version") or 1)) reasons = [] if mode != "repair" and s["oracle_read"]: reasons.append(f"oracle_read×{s['oracle_read']}") for reason in ('oracle_content_match', 'state_access', 'other_job_read'): if s[reason] and (reason != 'oracle_content_match' or mode != 'repair'): reasons.append(f'{reason}×{s[reason]}') if s["sibling_read"]: reasons.append(f"sibling_read×{s['sibling_read']}") if s["harness_read"] or s["toolchain_read"]: reasons.append(f"harness/toolchain_read×{s['harness_read'] + s['toolchain_read']}") stalled = any((st or {}).get("error_class") == "stalled" for st in (vj.get("stages") or {}).values()) if stalled: reasons.append("stalled_killed") if ver < min_version: reasons.append(f"old_version(v{ver}<{min_version})") if mode == "repair": reasons.append("historical-oracle-repair-analysis-only") terminal = vj.get("terminal") or rj.get("state") claim_t = (cj or {}).get("terminal") or (vj.get("claim") or {}).get("terminal") cc = vj.get("claim_consistent") if claim_t in (None, "ABSENT"): reasons.append("claim_absent") elif cc is False: reasons.append("claim_mismatch") contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"] or s["state_access"] or s["other_job_read"] or (mode != "repair" and s["oracle_content_match"])) running = rj.get("state") in ("RUNNING", "VERIFYING") or not s["has_agent_end"] and rj.get("state") == "RUNNING" rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running \ and mode != "repair" and terminal in ("PASS", "FAIL") sft_ok = rl_ok and terminal == "PASS" and cc is True return {"kk": kk, "version": ver, "state": rj.get("state"), "terminal": terminal, "claim": claim_t, "claim_consistent": cc, "turns": s["turns"], "tool_calls": s["tool_calls"], "blocked": s["blocked_attempts"], "timeout_reason": rj.get("timeout_reason"), "contaminated": contaminated, "stalled_killed": stalled, "rl_ok": rl_ok, "sft_ok": sft_ok, "reasons": reasons, "running": running, "dir": str(rd)} def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--run", required=True) ap.add_argument("--job", help="只看一个 job") ap.add_argument("--min-version", type=int, help="默认读 agent/VERSION") ap.add_argument("--json", action="store_true") ap.add_argument("--annotate", action="store_true", help="把 contaminated / stalled_killed 写进 annotations 表(同一条只写一次)") a = ap.parse_args() vp = paths.AGENT / "VERSION" min_version = a.min_version or (int(vp.read_text().strip()) if vp.exists() else 1) import sqlite3 with sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True) as con: rows = con.execute("SELECT job_id, kk, mode, state FROM rollouts WHERE run_id=? AND state!='QUEUED' ORDER BY job_id, kk", (a.run,)).fetchall() existing = {r[0] for r in con.execute("SELECT note FROM annotations WHERE run_id=?", (a.run,)).fetchall()} results = [] for job_id, kk, mode, state in rows: if a.job and job_id != a.job: continue rd = paths.RUNS / a.run / job_id / "rollouts" / f"{kk:02d}" if not rd.exists(): continue r = audit_one(rd, kk, min_version, mode or "e2e") r.update({"job_id": job_id, "mode": mode}) results.append(r) if a.json: print(json.dumps(results, ensure_ascii=False, indent=1)) else: print(f"run={a.run} min_version={min_version} 已完成/进行中 rollout={len(results)}") print(f"{'job':<40} {'kk':>2} {'v':>2} {'state':<8} {'verdict':<8} {'claim':<7} {'turns':>5} {'calls':>5} {'blk':>3} RL SFT 原因") for r in results: print(f"{r['job_id'][:40]:<40} {r['kk']:>2} {r['version']:>2} {str(r['state'])[:8]:<8} {str(r['terminal'])[:8]:<8} " f"{str(r['claim'])[:7]:<7} {r['turns']:>5} {r['tool_calls']:>5} {r['blocked']:>3} " f"{'✓' if r['rl_ok'] else '✗'} {'✓' if r['sft_ok'] else '✗'} {', '.join(r['reasons']) or ('运行中' if r['running'] else '干净')}") n_rl = sum(r["rl_ok"] for r in results); n_sft = sum(r["sft_ok"] for r in results) n_cont = sum(r["contaminated"] for r in results); n_stall = sum(r["stalled_killed"] for r in results) print(f"\n可进 RL:{n_rl} 可进 SFT:{n_sft} 污染(读了 oracle/兄弟):{n_cont} 被看门狗误杀:{n_stall}") print("导出时默认只收 agent_config_version ≥ 当前 VERSION 的:export/to_agentrl.py --min-agent-version", min_version) if a.annotate: n = 0 with db.connect() as con: for r in results: tags = [] if r["contaminated"]: tags.append("contaminated:" + "+".join(x.split("×")[0] for x in r["reasons"] if x.startswith(("oracle_read", "sibling_read", "oracle_content_match", "state_access", "other_job_read")))) if r["stalled_killed"]: tags.append("stalled_killed") for tag in tags: note = f"rollout#{r['kk']:02d} {tag} (audit_rollouts, v{r['version']})" if note in existing: continue db.annotate(con, a.run, r["job_id"], note); n += 1 print(f"annotations 新写入 {n} 条(已有的不重复)") return 0 if __name__ == "__main__": sys.exit(main())