data-gen-bpu/scripts/audit_rollouts.py

258 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""audit_rollouts —— 一条 rollout 到底能不能当训练数据?每条一行,给出结论和原因。
python3 scripts/audit_rollouts.py --run r20260915-03 # 表格
python3 scripts/audit_rollouts.py --run r20260915-03 --json # 机器可读
python3 scripts/audit_rollouts.py --run r20260915-03 --annotate # 把"污染"结论写进 annotations 表(幂等)
判据(全部来自轨迹本身 events.jsonl + verdict.json + guard.log,不看 claim 的自述):
oracle_read agent 成功读到 <job>/oracle/…(e2e 抄答案) → RL/SFT 都不能用
sibling_read 成功读到别的 rollout → 同上(GRPO 组内独立性没了)
harness_read 成功读到 harness/ 或工具链 site-packages → 不算污染,但是坏习惯;记数
stalled_killed 05-compile 被看门狗按 900 s 误杀(error_class=stalled)→ 这条 TIMEOUT 不是 agent 的错,不能当负样本
old_version agent_config_version < --min-version(默认 agent/VERSION)→ 旧口径,导出默认过滤
claim_absent / claim_mismatch → 可以当负样本(reward 已扣),但 SFT 不收
"成功读到"= 该次工具调用的结果里没有 path-guard 拒绝语;被 guard 拦下的尝试单独计入 blocked_attempts(不算污染)。
结论列:
RL ✓ = 初步候选(独立执行 + 结果可核验 + 版本合格 + 不是被误杀)
SFT ✓ = RL ✓ 且 verdict=PASS 且 claim 一致;完整性以导出器为准
"""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness import paths, db # noqa: E402
ORACLE_RX = re.compile(r"(^|[\s/'\"=(])oracle/|/oracle(/|$|[\s'\")])")
HARNESS_RX = re.compile(r"/bench/(harness|export)/|site-packages/(hmct|horizon_tc_ui|hbdk)")
TOOLCHAIN_RX = re.compile(r"toolchains/")
def _text(x) -> str:
if isinstance(x, str):
return x
if isinstance(x, list):
return " ".join(_text(c) for c in x)
if isinstance(x, dict):
return " ".join(_text(v) for k, v in x.items() if k in ("text", "content", "command", "path", "file_path", "output"))
return ""
DECIMAL = re.compile(r'(?<![\w.])-?\d+\.\d{4,}(?:[eE][+-]?\d+)?(?![\w.])')
STATE_RX = re.compile(r'jobs\.sqlite|bench-state|(?:^|[/\s\x22\x27])state/|\bsqlite3\b')
def oracle_needles(rd):
jd = rd.parent.parent
public = (jd / 'job.json').read_text(errors='replace') if (jd / 'job.json').exists() else ''
# Small metadata text only, never scan data pools or read binary datasets.
data = jd / 'data'
if data.is_dir():
for base, dirs, files in os.walk(data, followlinks=False):
for name in files:
p = Path(base) / name
if p.suffix in ('.json', '.yaml', '.yml', '.txt', '.md', '.csv') and p.is_file() and p.stat().st_size < 1024*1024:
public += '\n' + p.read_text(errors='replace')
result = jd / 'oracle/result.json'
needles = set(DECIMAL.findall(result.read_text())) if result.exists() else set()
yp = jd / 'oracle/stages/04-calibrate/compile.yaml'
# Only non-default documented knobs. Generic paths / defaults match independently produced configs.
defaults = {'calibration_type': {'default', 'max'}, 'compile_mode': {'latency'},
'optimize_level': {'O2'}, 'max_percentile': {'1.0'}, 'per_channel': {'False', 'false'}}
if yp.exists():
for line in yp.read_text().splitlines():
text = line.strip()
if ':' not in text:
continue
key, value = text.split(':', 1)
value = value.strip().strip("'\"")
if key in defaults and value not in defaults[key] and (DECIMAL.search(value) or len(value) > 24):
needles.add(text)
return {n for n in needles if n not in public}
def scan_events(rd: Path, kk: int) -> dict:
ev = rd / "agent" / "events.jsonl"
out = {"turns": 0, "tool_calls": 0, "oracle_read": 0, "sibling_read": 0, "harness_read": 0, "toolchain_read": 0,
"oracle_content_match": 0, "state_access": 0, "other_job_read": 0, "blocked_attempts": 0, "claim_calls": 0, "has_agent_end": False}
if not ev.exists():
out["no_events"] = True
return out
sib_rx = re.compile(r"/rollouts/(?!%02d(/|$))\d\d(/|$)" % kk)
needles = oracle_needles(rd)
run_name = rd.parents[2].name
job_name = rd.parents[1].name
other_rx = re.compile(r'runs/' + re.escape(run_name) + r'/([^/\s\x22\x27]+)')
independently_seen = set()
pending_names = {}
pending: dict[str, str] = {} # toolCallId → 目标字符串
with ev.open(errors="replace") as stream:
lines = stream.readlines()
for line in lines:
try:
e = json.loads(line)
except json.JSONDecodeError:
continue
t = e.get("type")
if t == "turn_end":
out["turns"] += 1
elif t == "tool_execution_start":
out["tool_calls"] += 1
name = e.get("toolName") or ""
args = e.get("args") or {}
if name == "claim":
out["claim_calls"] += 1
pending_names[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = name
target = " ".join(str(args.get(k, "")) for k in ("command", "path", "file_path", "pattern", "query"))
pending[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = target
elif t == "tool_execution_end":
cid = str(e.get("toolCallId") or e.get("id") or "")
target = pending.pop(cid, "")
res = _text(e.get("result"))
blocked = "path-guard" in res
if blocked:
out["blocked_attempts"] += 1
continue
# Only actual tool results, excluding guard denials / failed filesystem probes.
full_result = json.dumps(e.get('result'), ensure_ascii=False)
tool_name = pending_names.pop(cid, '')
# Independently executed stages can reproduce the oracle exactly.
# Exempt only a single wait/status command for an existing own handle;
# claim returns the agent's own input, so it is not an external read.
wait_cmd = re.fullmatch(r'bench stage (?:wait|status) ([a-zA-Z0-9_-]+)(?: --poll \d+)?(?: 2>&1)?(?: \| tail -\d+)?', target.strip())
own_stage = bool(wait_cmd and (rd / 'handles' / wait_cmd[1] / 'request.json').is_file())
if own_stage:
independently_seen.update(n for n in needles if n in full_result)
if tool_name != 'claim' and not own_stage and any(n in full_result for n in needles - independently_seen):
out['oracle_content_match'] += 1
if STATE_RX.search(target):
out['state_access'] += 1
if any(m[1] != job_name for m in other_rx.finditer(target)):
out['other_job_read'] += 1
if ORACLE_RX.search(target):
out["oracle_read"] += 1
if sib_rx.search(target):
out["sibling_read"] += 1
if HARNESS_RX.search(target):
out["harness_read"] += 1
if TOOLCHAIN_RX.search(target):
out["toolchain_read"] += 1
elif t == "agent_end":
out["has_agent_end"] = True
return out
def load(p: Path) -> dict:
try:
return json.loads(p.read_text())
except Exception:
return {}
def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict:
rj, vj, cj = load(rd / "rollout.json"), load(rd / "verdict.json"), load(rd / "claim.json")
s = scan_events(rd, kk)
ver = int(str(rj.get("agent_config_version") or 1))
reasons = []
if mode != "repair" and s["oracle_read"]:
reasons.append(f"oracle_read×{s['oracle_read']}")
for reason in ('oracle_content_match', 'state_access', 'other_job_read'):
if s[reason] and (reason != 'oracle_content_match' or mode != 'repair'):
reasons.append(f'{reason}×{s[reason]}')
if s["sibling_read"]:
reasons.append(f"sibling_read×{s['sibling_read']}")
if s["harness_read"] or s["toolchain_read"]:
reasons.append(f"harness/toolchain_read×{s['harness_read'] + s['toolchain_read']}")
stalled = any((st or {}).get("error_class") == "stalled" for st in (vj.get("stages") or {}).values())
if stalled:
reasons.append("stalled_killed")
if ver < min_version:
reasons.append(f"old_version(v{ver}<{min_version})")
if mode == "repair":
reasons.append("historical-oracle-repair-analysis-only")
terminal = vj.get("terminal") or rj.get("state")
claim_t = (cj or {}).get("terminal") or (vj.get("claim") or {}).get("terminal")
cc = vj.get("claim_consistent")
if claim_t in (None, "ABSENT"):
reasons.append("claim_absent")
elif cc is False:
reasons.append("claim_mismatch")
contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"] or s["state_access"] or s["other_job_read"] or (mode != "repair" and s["oracle_content_match"]))
running = rj.get("state") in ("RUNNING", "VERIFYING") or not s["has_agent_end"] and rj.get("state") == "RUNNING"
rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running \
and mode != "repair" and terminal in ("PASS", "FAIL")
sft_ok = rl_ok and terminal == "PASS" and cc is True
return {"kk": kk, "version": ver, "state": rj.get("state"), "terminal": terminal, "claim": claim_t,
"claim_consistent": cc, "turns": s["turns"], "tool_calls": s["tool_calls"], "blocked": s["blocked_attempts"],
"timeout_reason": rj.get("timeout_reason"), "contaminated": contaminated, "stalled_killed": stalled,
"rl_ok": rl_ok, "sft_ok": sft_ok, "reasons": reasons, "running": running, "dir": str(rd)}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--run", required=True)
ap.add_argument("--job", help="只看一个 job")
ap.add_argument("--min-version", type=int, help="默认读 agent/VERSION")
ap.add_argument("--json", action="store_true")
ap.add_argument("--annotate", action="store_true", help="把 contaminated / stalled_killed 写进 annotations 表(同一条只写一次)")
a = ap.parse_args()
vp = paths.AGENT / "VERSION"
min_version = a.min_version or (int(vp.read_text().strip()) if vp.exists() else 1)
import sqlite3
with sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True) as con:
rows = con.execute("SELECT job_id, kk, mode, state FROM rollouts WHERE run_id=? AND state!='QUEUED' ORDER BY job_id, kk",
(a.run,)).fetchall()
existing = {r[0] for r in con.execute("SELECT note FROM annotations WHERE run_id=?", (a.run,)).fetchall()}
results = []
for job_id, kk, mode, state in rows:
if a.job and job_id != a.job:
continue
rd = paths.RUNS / a.run / job_id / "rollouts" / f"{kk:02d}"
if not rd.exists():
continue
r = audit_one(rd, kk, min_version, mode or "e2e")
r.update({"job_id": job_id, "mode": mode})
results.append(r)
if a.json:
print(json.dumps(results, ensure_ascii=False, indent=1))
else:
print(f"run={a.run} min_version={min_version} 已完成/进行中 rollout={len(results)}")
print(f"{'job':<40} {'kk':>2} {'v':>2} {'state':<8} {'verdict':<8} {'claim':<7} {'turns':>5} {'calls':>5} {'blk':>3} RL SFT 原因")
for r in results:
print(f"{r['job_id'][:40]:<40} {r['kk']:>2} {r['version']:>2} {str(r['state'])[:8]:<8} {str(r['terminal'])[:8]:<8} "
f"{str(r['claim'])[:7]:<7} {r['turns']:>5} {r['tool_calls']:>5} {r['blocked']:>3} "
f"{'✓' if r['rl_ok'] else '✗'} {'✓' if r['sft_ok'] else '✗'} {', '.join(r['reasons']) or ('运行中' if r['running'] else '干净')}")
n_rl = sum(r["rl_ok"] for r in results); n_sft = sum(r["sft_ok"] for r in results)
n_cont = sum(r["contaminated"] for r in results); n_stall = sum(r["stalled_killed"] for r in results)
print(f"\n可进 RL:{n_rl} 可进 SFT:{n_sft} 污染(读了 oracle/兄弟):{n_cont} 被看门狗误杀:{n_stall}")
print("导出时默认只收 agent_config_version ≥ 当前 VERSION 的:export/to_agentrl.py --min-agent-version", min_version)
if a.annotate:
n = 0
with db.connect() as con:
for r in results:
tags = []
if r["contaminated"]:
tags.append("contaminated:" + "+".join(x.split("×")[0] for x in r["reasons"] if x.startswith(("oracle_read", "sibling_read", "oracle_content_match", "state_access", "other_job_read"))))
if r["stalled_killed"]:
tags.append("stalled_killed")
for tag in tags:
note = f"rollout#{r['kk']:02d} {tag} (audit_rollouts, v{r['version']})"
if note in existing:
continue
db.annotate(con, a.run, r["job_id"], note); n += 1
print(f"annotations 新写入 {n} 条(已有的不重复)")
return 0
if __name__ == "__main__":
sys.exit(main())