255 lines
13 KiB
Python
255 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""audit_rollouts —— 一条 rollout 到底能不能当训练数据?每条一行,给出结论和原因。
|
||
|
||
python3 scripts/audit_rollouts.py --run r20260915-03 # 表格
|
||
python3 scripts/audit_rollouts.py --run r20260915-03 --json # 机器可读
|
||
python3 scripts/audit_rollouts.py --run r20260915-03 --annotate # 把"污染"结论写进 annotations 表(幂等)
|
||
|
||
判据(全部来自轨迹本身 events.jsonl + verdict.json + guard.log,不看 claim 的自述):
|
||
oracle_read agent 成功读到 <job>/oracle/…(e2e 抄答案) → RL/SFT 都不能用
|
||
sibling_read 成功读到别的 rollout → 同上(GRPO 组内独立性没了)
|
||
harness_read 成功读到 harness/ 或工具链 site-packages → 不算污染,但是坏习惯;记数
|
||
stalled_killed 05-compile 被看门狗按 900 s 误杀(error_class=stalled)→ 这条 TIMEOUT 不是 agent 的错,不能当负样本
|
||
old_version agent_config_version < --min-version(默认 agent/VERSION)→ 旧口径,导出默认过滤
|
||
claim_absent / claim_mismatch → 可以当负样本(reward 已扣),但 SFT 不收
|
||
"成功读到"= 该次工具调用的结果里没有 path-guard 拒绝语;被 guard 拦下的尝试单独计入 blocked_attempts(不算污染)。
|
||
|
||
结论列:
|
||
RL ✓ = 可进 GRPO 组(干净 + 版本合格 + 不是被误杀)
|
||
SFT ✓ = RL ✓ 且 verdict=PASS 且 claim 一致
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
from harness import paths, db # noqa: E402
|
||
|
||
ORACLE_RX = re.compile(r"(^|[\s/'\"=(])oracle/|/oracle(/|$|[\s'\")])")
|
||
HARNESS_RX = re.compile(r"/bench/(harness|export)/|site-packages/(hmct|horizon_tc_ui|hbdk)")
|
||
TOOLCHAIN_RX = re.compile(r"toolchains/")
|
||
|
||
|
||
def _text(x) -> str:
|
||
if isinstance(x, str):
|
||
return x
|
||
if isinstance(x, list):
|
||
return " ".join(_text(c) for c in x)
|
||
if isinstance(x, dict):
|
||
return " ".join(_text(v) for k, v in x.items() if k in ("text", "content", "command", "path", "file_path", "output"))
|
||
return ""
|
||
|
||
|
||
DECIMAL = re.compile(r'(?<![\w.])-?\d+\.\d{4,}(?:[eE][+-]?\d+)?(?![\w.])')
|
||
STATE_RX = re.compile(r'jobs\.sqlite|bench-state|(?:^|[/\s\x22\x27])state/|\bsqlite3\b')
|
||
|
||
def oracle_needles(rd):
|
||
jd = rd.parent.parent
|
||
public = (jd / 'job.json').read_text(errors='replace') if (jd / 'job.json').exists() else ''
|
||
# Small metadata text only, never scan data pools or read binary datasets.
|
||
data = jd / 'data'
|
||
if data.is_dir():
|
||
for base, dirs, files in os.walk(data, followlinks=False):
|
||
for name in files:
|
||
p = Path(base) / name
|
||
if p.suffix in ('.json', '.yaml', '.yml', '.txt', '.md', '.csv') and p.is_file() and p.stat().st_size < 1024*1024:
|
||
public += '\n' + p.read_text(errors='replace')
|
||
result = jd / 'oracle/result.json'
|
||
needles = set(DECIMAL.findall(result.read_text())) if result.exists() else set()
|
||
yp = jd / 'oracle/stages/04-calibrate/compile.yaml'
|
||
# Only non-default documented knobs. Generic paths / defaults match independently produced configs.
|
||
defaults = {'calibration_type': {'default', 'max'}, 'compile_mode': {'latency'},
|
||
'optimize_level': {'O2'}, 'max_percentile': {'1.0'}, 'per_channel': {'False', 'false'}}
|
||
if yp.exists():
|
||
for line in yp.read_text().splitlines():
|
||
text = line.strip()
|
||
if ':' not in text:
|
||
continue
|
||
key, value = text.split(':', 1)
|
||
value = value.strip().strip("'\"")
|
||
if key in defaults and value not in defaults[key] and (DECIMAL.search(value) or len(value) > 24):
|
||
needles.add(text)
|
||
return {n for n in needles if n not in public}
|
||
|
||
|
||
def scan_events(rd: Path, kk: int) -> dict:
|
||
ev = rd / "agent" / "events.jsonl"
|
||
out = {"turns": 0, "tool_calls": 0, "oracle_read": 0, "sibling_read": 0, "harness_read": 0, "toolchain_read": 0,
|
||
"oracle_content_match": 0, "state_access": 0, "other_job_read": 0, "blocked_attempts": 0, "claim_calls": 0, "has_agent_end": False}
|
||
if not ev.exists():
|
||
out["no_events"] = True
|
||
return out
|
||
sib_rx = re.compile(r"/rollouts/(?!%02d(/|$))\d\d(/|$)" % kk)
|
||
needles = oracle_needles(rd)
|
||
run_name = rd.parents[2].name
|
||
job_name = rd.parents[1].name
|
||
other_rx = re.compile(r'runs/' + re.escape(run_name) + r'/([^/\s\x22\x27]+)')
|
||
independently_seen = set()
|
||
pending_names = {}
|
||
pending: dict[str, str] = {} # toolCallId → 目标字符串
|
||
with ev.open(errors="replace") as stream:
|
||
lines = stream.readlines()
|
||
for line in lines:
|
||
try:
|
||
e = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
t = e.get("type")
|
||
if t == "turn_end":
|
||
out["turns"] += 1
|
||
elif t == "tool_execution_start":
|
||
out["tool_calls"] += 1
|
||
name = e.get("toolName") or ""
|
||
args = e.get("args") or {}
|
||
if name == "claim":
|
||
out["claim_calls"] += 1
|
||
pending_names[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = name
|
||
target = " ".join(str(args.get(k, "")) for k in ("command", "path", "file_path", "pattern", "query"))
|
||
pending[str(e.get("toolCallId") or e.get("id") or out["tool_calls"])] = target
|
||
elif t == "tool_execution_end":
|
||
cid = str(e.get("toolCallId") or e.get("id") or "")
|
||
target = pending.pop(cid, "")
|
||
res = _text(e.get("result"))
|
||
blocked = "path-guard" in res
|
||
if blocked:
|
||
out["blocked_attempts"] += 1
|
||
continue
|
||
# Only actual tool results, excluding guard denials / failed filesystem probes.
|
||
full_result = json.dumps(e.get('result'), ensure_ascii=False)
|
||
tool_name = pending_names.pop(cid, '')
|
||
# Independently executed stages can reproduce the oracle exactly.
|
||
# Exempt only a single wait/status command for an existing own handle;
|
||
# claim returns the agent's own input, so it is not an external read.
|
||
wait_cmd = re.fullmatch(r'bench stage (?:wait|status) ([a-zA-Z0-9_-]+)(?: --poll \d+)?(?: 2>&1)?(?: \| tail -\d+)?', target.strip())
|
||
own_stage = bool(wait_cmd and (rd / 'handles' / wait_cmd[1] / 'request.json').is_file())
|
||
if own_stage:
|
||
independently_seen.update(n for n in needles if n in full_result)
|
||
if tool_name != 'claim' and not own_stage and any(n in full_result for n in needles - independently_seen):
|
||
out['oracle_content_match'] += 1
|
||
if STATE_RX.search(target):
|
||
out['state_access'] += 1
|
||
if any(m[1] != job_name for m in other_rx.finditer(target)):
|
||
out['other_job_read'] += 1
|
||
if ORACLE_RX.search(target):
|
||
out["oracle_read"] += 1
|
||
if sib_rx.search(target):
|
||
out["sibling_read"] += 1
|
||
if HARNESS_RX.search(target):
|
||
out["harness_read"] += 1
|
||
if TOOLCHAIN_RX.search(target):
|
||
out["toolchain_read"] += 1
|
||
elif t == "agent_end":
|
||
out["has_agent_end"] = True
|
||
return out
|
||
|
||
|
||
def load(p: Path) -> dict:
|
||
try:
|
||
return json.loads(p.read_text())
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict:
|
||
rj, vj, cj = load(rd / "rollout.json"), load(rd / "verdict.json"), load(rd / "claim.json")
|
||
s = scan_events(rd, kk)
|
||
ver = int(str(rj.get("agent_config_version") or 1))
|
||
reasons = []
|
||
if mode != "repair" and s["oracle_read"]:
|
||
reasons.append(f"oracle_read×{s['oracle_read']}")
|
||
for reason in ('oracle_content_match', 'state_access', 'other_job_read'):
|
||
if s[reason] and (reason != 'oracle_content_match' or mode != 'repair'):
|
||
reasons.append(f'{reason}×{s[reason]}')
|
||
if s["sibling_read"]:
|
||
reasons.append(f"sibling_read×{s['sibling_read']}")
|
||
if s["harness_read"] or s["toolchain_read"]:
|
||
reasons.append(f"harness/toolchain_read×{s['harness_read'] + s['toolchain_read']}")
|
||
stalled = any((st or {}).get("error_class") == "stalled" for st in (vj.get("stages") or {}).values())
|
||
if stalled:
|
||
reasons.append("stalled_killed")
|
||
if ver < min_version:
|
||
reasons.append(f"old_version(v{ver}<{min_version})")
|
||
terminal = vj.get("terminal") or rj.get("state")
|
||
claim_t = (cj or {}).get("terminal") or (vj.get("claim") or {}).get("terminal")
|
||
cc = vj.get("claim_consistent")
|
||
if claim_t in (None, "ABSENT"):
|
||
reasons.append("claim_absent")
|
||
elif cc is False:
|
||
reasons.append("claim_mismatch")
|
||
contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"] or s["state_access"] or s["other_job_read"] or (mode != "repair" and s["oracle_content_match"]))
|
||
running = rj.get("state") in ("RUNNING", "VERIFYING") or not s["has_agent_end"] and rj.get("state") == "RUNNING"
|
||
rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running
|
||
sft_ok = rl_ok and terminal == "PASS" and cc is True
|
||
return {"kk": kk, "version": ver, "state": rj.get("state"), "terminal": terminal, "claim": claim_t,
|
||
"claim_consistent": cc, "turns": s["turns"], "tool_calls": s["tool_calls"], "blocked": s["blocked_attempts"],
|
||
"timeout_reason": rj.get("timeout_reason"), "contaminated": contaminated, "stalled_killed": stalled,
|
||
"rl_ok": rl_ok, "sft_ok": sft_ok, "reasons": reasons, "running": running, "dir": str(rd)}
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("--run", required=True)
|
||
ap.add_argument("--job", help="只看一个 job")
|
||
ap.add_argument("--min-version", type=int, help="默认读 agent/VERSION")
|
||
ap.add_argument("--json", action="store_true")
|
||
ap.add_argument("--annotate", action="store_true", help="把 contaminated / stalled_killed 写进 annotations 表(同一条只写一次)")
|
||
a = ap.parse_args()
|
||
vp = paths.AGENT / "VERSION"
|
||
min_version = a.min_version or (int(vp.read_text().strip()) if vp.exists() else 1)
|
||
|
||
import sqlite3
|
||
with sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True) as con:
|
||
rows = con.execute("SELECT job_id, kk, mode, state FROM rollouts WHERE run_id=? AND state!='QUEUED' ORDER BY job_id, kk",
|
||
(a.run,)).fetchall()
|
||
existing = {r[0] for r in con.execute("SELECT note FROM annotations WHERE run_id=?", (a.run,)).fetchall()}
|
||
results = []
|
||
for job_id, kk, mode, state in rows:
|
||
if a.job and job_id != a.job:
|
||
continue
|
||
rd = paths.RUNS / a.run / job_id / "rollouts" / f"{kk:02d}"
|
||
if not rd.exists():
|
||
continue
|
||
r = audit_one(rd, kk, min_version, mode or "e2e")
|
||
r.update({"job_id": job_id, "mode": mode})
|
||
results.append(r)
|
||
|
||
if a.json:
|
||
print(json.dumps(results, ensure_ascii=False, indent=1))
|
||
else:
|
||
print(f"run={a.run} min_version={min_version} 已完成/进行中 rollout={len(results)}")
|
||
print(f"{'job':<40} {'kk':>2} {'v':>2} {'state':<8} {'verdict':<8} {'claim':<7} {'turns':>5} {'calls':>5} {'blk':>3} RL SFT 原因")
|
||
for r in results:
|
||
print(f"{r['job_id'][:40]:<40} {r['kk']:>2} {r['version']:>2} {str(r['state'])[:8]:<8} {str(r['terminal'])[:8]:<8} "
|
||
f"{str(r['claim'])[:7]:<7} {r['turns']:>5} {r['tool_calls']:>5} {r['blocked']:>3} "
|
||
f"{'✓' if r['rl_ok'] else '✗'} {'✓' if r['sft_ok'] else '✗'} {', '.join(r['reasons']) or ('运行中' if r['running'] else '干净')}")
|
||
n_rl = sum(r["rl_ok"] for r in results); n_sft = sum(r["sft_ok"] for r in results)
|
||
n_cont = sum(r["contaminated"] for r in results); n_stall = sum(r["stalled_killed"] for r in results)
|
||
print(f"\n可进 RL:{n_rl} 可进 SFT:{n_sft} 污染(读了 oracle/兄弟):{n_cont} 被看门狗误杀:{n_stall}")
|
||
print("导出时默认只收 agent_config_version ≥ 当前 VERSION 的:export/to_agentrl.py --min-agent-version", min_version)
|
||
|
||
if a.annotate:
|
||
n = 0
|
||
with db.connect() as con:
|
||
for r in results:
|
||
tags = []
|
||
if r["contaminated"]:
|
||
tags.append("contaminated:" + "+".join(x.split("×")[0] for x in r["reasons"] if x.startswith(("oracle_read", "sibling_read", "oracle_content_match", "state_access", "other_job_read"))))
|
||
if r["stalled_killed"]:
|
||
tags.append("stalled_killed")
|
||
for tag in tags:
|
||
note = f"rollout#{r['kk']:02d} {tag} (audit_rollouts, v{r['version']})"
|
||
if note in existing:
|
||
continue
|
||
db.annotate(con, a.run, r["job_id"], note); n += 1
|
||
print(f"annotations 新写入 {n} 条(已有的不重复)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|