A checkout whose state is elsewhere (BENCH_STATE_DIR, staging) was read as <checkout>/state, so content review and export found no database and called a PASS rollout unfinished. One helper, paths.state_dir, used by every lookup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
994 lines
55 KiB
Python
994 lines
55 KiB
Python
"""L4 data materialization: preserve Pi source events, verify, then publish.
|
||
|
||
python export/to_agentrl.py --runs r20260914-01 [r…] [--policy teacher] [--include-timeout]
|
||
|
||
Legacy analysis-only cleaning (never used for strict Pi SFT):
|
||
R1 双方言归一:OpenAI / Anthropic 形状 → {role, content, tool_calls?, tool_call_id?}
|
||
R2 剥离 thinking / reasoning 块
|
||
R3 system 只在首条且只有一条(多条合并)
|
||
R4 保留完整工具调用/结果;缺失配对的轨迹标记 partial,不进入成功 SFT
|
||
R5 去空消息
|
||
R6 合并相邻同 role 的纯文本消息
|
||
R7 tool 结果截断(头 6k + 尾 2k 字符)
|
||
R8 隔离:无 assistant 消息,或 messages < 2(排除理由落 exclusions.jsonl)
|
||
轨迹源:agent_end.messages[],中断时恢复已完成的 message_end;不会捏造 assistant 回答。
|
||
默认保存全部终态;SFT 候选另行标记。pipeline.jsonl 保存真实确定性流程记录,和 Agent 消息分开。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import contextlib
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import sqlite3
|
||
import os
|
||
import re
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
BENCH = Path(os.environ.get("BENCH_ROOT", HERE.parent))
|
||
sys.path.insert(0, str(HERE.parent))
|
||
sys.path.insert(0, str(HERE))
|
||
from redact import redact_messages, redact_preserving # noqa: E402
|
||
from export import trajectory as preserved # noqa: E402
|
||
from reward import compute as compute_reward, RECIPE_VERSION # noqa: E402
|
||
|
||
TOOL_HEAD, TOOL_TAIL = 6000, 2000
|
||
_TOOLS_CACHE = {}
|
||
_UNSET = object()
|
||
|
||
|
||
# ---------------------------------------------------------------- 轨迹提取
|
||
|
||
def system_prompt(rd: Path) -> tuple[str, str]:
|
||
"""取这条轨迹的系统提示 → (text, source)。
|
||
|
||
pi 的 --append-system-prompt 内容**不出现在** agent_end.messages[] 里(实测 0.73.1):
|
||
导出时若不补回来,SFT 样本就只有 user 任务 + assistant 动作,没有"只能写 workspace 等"这些规则,
|
||
等于训练一个看不见规则的策略。优先读 rollout 自己的快照,其次按 cmd.json 里的路径读当前文件(会漂移,故记 source)。
|
||
"""
|
||
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
|
||
segments = _read_json(rd / "agent/raw-system.json", default=[])
|
||
if isinstance(segments, list) and segments and all(isinstance(v, str) for v in segments):
|
||
return "\n\n".join(segments), "runner-raw-system"
|
||
return "", "missing"
|
||
snap = rd / "agent" / "system.md"
|
||
if snap.exists():
|
||
return snap.read_text(errors="replace"), "snapshot"
|
||
cmd = rd / "agent" / "cmd.json"
|
||
if cmd.exists():
|
||
try:
|
||
argv = json.loads(cmd.read_text())["cmd"]
|
||
i = argv.index("--append-system-prompt")
|
||
f = Path(argv[i + 1])
|
||
if f.exists():
|
||
return f.read_text(errors="replace"), "live-file"
|
||
except (ValueError, KeyError, IndexError, OSError):
|
||
pass
|
||
return "", "missing"
|
||
|
||
|
||
def tool_defs(rd: Path) -> tuple[list, str]:
|
||
"""这条轨迹的工具定义 → (list, source)。快照没有就按 cmd.json 的 --tools 白名单现 dump 一次。"""
|
||
snap = rd / "agent" / "tools.json"
|
||
if snap.exists():
|
||
try:
|
||
definitions = json.loads(snap.read_text())
|
||
if isinstance(definitions, list):
|
||
return definitions, "snapshot"
|
||
except json.JSONDecodeError:
|
||
pass
|
||
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
|
||
return [], "missing"
|
||
cmd = rd / "agent" / "cmd.json"
|
||
if cmd.exists():
|
||
try:
|
||
argv = json.loads(cmd.read_text())["cmd"]
|
||
wl = argv[argv.index("--tools") + 1]
|
||
import subprocess
|
||
from harness import paths
|
||
node = os.path.expanduser(str(paths.config()["paths"].get("node_bin") or "node"))
|
||
if "/" in node and not Path(node).is_absolute():
|
||
node = str(BENCH / node)
|
||
env = os.environ.copy()
|
||
env["PATH"] = str(Path(node).parent) + os.pathsep + env.get("PATH", "/usr/bin:/bin")
|
||
script = BENCH / "agent" / "dump_tools.js"
|
||
key = (node, str(script), script.stat().st_mtime_ns, wl)
|
||
if key in _TOOLS_CACHE:
|
||
return _TOOLS_CACHE[key], "redumped"
|
||
r = subprocess.run([node, str(script), wl],
|
||
capture_output=True, text=True, timeout=60, env=env)
|
||
if r.returncode == 0 and r.stdout.strip():
|
||
definitions = json.loads(r.stdout)
|
||
if isinstance(definitions, list) and definitions:
|
||
_TOOLS_CACHE[key] = definitions
|
||
return definitions, "redumped"
|
||
except Exception:
|
||
pass
|
||
return [], "missing"
|
||
|
||
|
||
def load_trajectory(rd: Path) -> tuple[list[dict], dict]:
|
||
"""Read every Pi segment; container mirrors cannot replace earlier history."""
|
||
runner = _read_json(rd / "rollout.json").get("runner", "pi")
|
||
if runner != "pi":
|
||
from harness.runners import get
|
||
return get(runner).trajectory(rd)
|
||
scanned = preserved.inventory(preserved.freeze(rd))
|
||
return [item["payload"] for item in scanned["messages"] if item["event_type"] != "system_snapshot"], {
|
||
"source": "pi-segment-events" if scanned["agent_end"] else "message_end",
|
||
"partial": bool(scanned["errors"]), "agent_end": scanned["agent_end"], "errors": scanned["errors"]}
|
||
|
||
|
||
def load_messages(rd: Path) -> list[dict] | None:
|
||
return load_trajectory(rd)[0] or None
|
||
|
||
|
||
# ---------------------------------------------------------------- R1–R8
|
||
|
||
def _text_of(content) -> str:
|
||
if isinstance(content, str):
|
||
return content
|
||
if isinstance(content, list):
|
||
parts = []
|
||
for c in content:
|
||
if isinstance(c, dict):
|
||
t = c.get("type")
|
||
if t in ("text", "output_text") and c.get("text"):
|
||
parts.append(c["text"])
|
||
elif t in ("thinking", "reasoning", "redacted_thinking"):
|
||
continue # R2
|
||
elif isinstance(c, str):
|
||
parts.append(c)
|
||
return "\n".join(parts)
|
||
return "" if content is None else str(content)
|
||
|
||
|
||
def r1_normalize(m: dict) -> list[dict]:
|
||
"""一条原始消息 → 0..n 条归一消息(Anthropic 的 tool_use/tool_result 混在 content 里要拆)。"""
|
||
role = m.get("role")
|
||
content = m.get("content")
|
||
out: list[dict] = []
|
||
if role in ("toolResult", "tool_result", "tool"):
|
||
return [{"role": "tool", "tool_call_id": m.get("toolCallId") or m.get("tool_call_id"),
|
||
"content": _text_of(content) or json.dumps(m.get("output") or m.get("result") or "", ensure_ascii=False)}]
|
||
if role == "assistant":
|
||
tool_calls = []
|
||
if isinstance(content, list):
|
||
for c in content:
|
||
if isinstance(c, dict) and c.get("type") in ("toolCall", "tool_use", "function_call"):
|
||
tool_calls.append({"id": c.get("id"), "name": c.get("name"),
|
||
"arguments": c.get("arguments") if "arguments" in c else c.get("input")})
|
||
for tc in m.get("tool_calls") or []:
|
||
fn = tc.get("function") or {}
|
||
tool_calls.append({"id": tc.get("id"), "name": fn.get("name") or tc.get("name"),
|
||
"arguments": fn.get("arguments") if fn else tc.get("arguments")})
|
||
msg = {"role": "assistant", "content": _text_of(content)}
|
||
if tool_calls:
|
||
msg["tool_calls"] = tool_calls
|
||
return [msg]
|
||
if role == "custom":
|
||
# pi 扩展塞进会话的消息(如 budget-kill 的预算提醒):模型是当 user 输入看到的,训练样本也要有
|
||
t = _text_of(content)
|
||
return [{"role": "user", "content": f"[{m.get('customType', 'extension')}] {t}"}] if t else []
|
||
if role in ("user", "system"):
|
||
if isinstance(content, list) and any(isinstance(c, dict) and c.get("type") == "tool_result" for c in content):
|
||
for c in content:
|
||
if isinstance(c, dict) and c.get("type") == "tool_result":
|
||
out.append({"role": "tool", "tool_call_id": c.get("tool_use_id"), "content": _text_of(c.get("content"))})
|
||
else:
|
||
t = _text_of([c])
|
||
if t:
|
||
out.append({"role": "user", "content": t})
|
||
return out
|
||
return [{"role": role, "content": _text_of(content)}]
|
||
return []
|
||
|
||
|
||
def clean(messages: list[dict]) -> tuple[list[dict], dict]:
|
||
stats = {"in": len(messages)}
|
||
msgs: list[dict] = []
|
||
for m in messages:
|
||
msgs.extend(r1_normalize(m))
|
||
# R3 system 合并到首条
|
||
systems = [m["content"] for m in msgs if m["role"] == "system"]
|
||
msgs = [m for m in msgs if m["role"] != "system"]
|
||
if systems:
|
||
msgs.insert(0, {"role": "system", "content": "\n\n".join(s for s in systems if s)})
|
||
# R5 去空
|
||
msgs = [m for m in msgs if m.get("content") or m.get("tool_calls") or m["role"] == "tool"]
|
||
# R7 tool 结果截断
|
||
for m in msgs:
|
||
if m["role"] == "tool" and len(m["content"]) > TOOL_HEAD + TOOL_TAIL:
|
||
m["content"] = m["content"][:TOOL_HEAD] + f"\n…[truncated {len(m['content']) - TOOL_HEAD - TOOL_TAIL} chars]…\n" + m["content"][-TOOL_TAIL:]
|
||
stats["r7_truncated"] = stats.get("r7_truncated", 0) + 1
|
||
# R6 合并相邻同 role 纯文本
|
||
merged: list[dict] = []
|
||
for m in msgs:
|
||
if merged and m["role"] == merged[-1]["role"] and m["role"] in ("user", "assistant") \
|
||
and not m.get("tool_calls") and not merged[-1].get("tool_calls"):
|
||
merged[-1]["content"] += "\n" + m["content"]
|
||
stats["r6_merged"] = stats.get("r6_merged", 0) + 1
|
||
else:
|
||
merged.append(m)
|
||
msgs = merged
|
||
# R4 Preserve completed tool exchanges, including a trailing tool result.
|
||
# Dropping that result used to manufacture a dangling assistant call.
|
||
# Incomplete/interior-invalid exchanges remain useful evidence, but cannot
|
||
# enter SFT; never fabricate tool results or an assistant final answer.
|
||
pending, seen = set(), set()
|
||
missing, orphan, duplicates = [], [], []
|
||
for m in msgs:
|
||
if m["role"] != "tool" and pending:
|
||
missing.extend(sorted(pending)); pending.clear()
|
||
if m["role"] == "assistant":
|
||
for tc in m.get("tool_calls") or []:
|
||
tid = tc.get("id")
|
||
if not tid or tid in seen:
|
||
duplicates.append(tid)
|
||
if tid:
|
||
seen.add(tid); pending.add(tid)
|
||
elif m["role"] == "tool":
|
||
tid = m.get("tool_call_id")
|
||
if tid not in pending:
|
||
orphan.append(tid)
|
||
else:
|
||
pending.remove(tid)
|
||
missing.extend(sorted(pending))
|
||
stats["tool_pairing"] = {"valid": not (missing or orphan or duplicates),
|
||
"missing_results": len(missing), "orphan_results": len(orphan),
|
||
"duplicate_or_missing_call_ids": len(duplicates)}
|
||
stats["trailing_user"] = bool(msgs and msgs[-1]["role"] == "user")
|
||
# R8
|
||
n_assist = sum(m["role"] == "assistant" for m in msgs)
|
||
stats.update({"out": len(msgs), "assistant_turns": n_assist, "drop": n_assist == 0 or len(msgs) < 2})
|
||
return msgs, stats
|
||
|
||
|
||
# ---------------------------------------------------------------- 主流程
|
||
|
||
def sft_eligibility(verdict: dict, reward: float) -> tuple[bool, str]:
|
||
"""Outcome eligibility only; the caller also checks transcript integrity.
|
||
|
||
Execution TIMEOUT is separate from measured terminal PASS/FAIL. A completed,
|
||
measured success is not rejected solely for exhausting its budget.
|
||
Legacy TIMEOUT verdicts stay evidence until reverified, never assumed PASS.
|
||
"""
|
||
if verdict.get("mode") == "repair":
|
||
return False, "historical-repair-context"
|
||
t = verdict.get("terminal")
|
||
if t == "PASS":
|
||
return True, "PASS"
|
||
if t == "BLOCKED":
|
||
return False, "BLOCKED-unproven"
|
||
if t == "TIMEOUT":
|
||
return False, "legacy-TIMEOUT-needs-reverification"
|
||
return False, f"{t}-not-success-demonstration"
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def readonly_db():
|
||
from harness import paths
|
||
state = paths.state_dir(BENCH)
|
||
con = sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True, timeout=60)
|
||
con.row_factory = sqlite3.Row
|
||
try:
|
||
yield con
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
DB_TERMINAL_STATES = ('PASS', 'FAIL', 'BLOCKED', 'UNVERIFIED', 'TIMEOUT', 'DONE', 'ERROR', 'CANCELLED', 'DEADLINE_EXCEEDED', 'BUDGET_EXHAUSTED', 'INFRA_ERROR', 'LOST')
|
||
|
||
|
||
def _database_terminal(rd, source_root=None):
|
||
root = Path(source_root or BENCH)
|
||
from harness import paths
|
||
state = paths.state_dir(root)
|
||
try:
|
||
with contextlib.closing(sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True)) as con:
|
||
row = con.execute('SELECT state FROM rollouts WHERE run_id=? AND job_id=? AND kk=?',
|
||
(rd.parents[2].name, rd.parents[1].name, int(rd.name))).fetchone()
|
||
return row is not None and row[0] in DB_TERMINAL_STATES
|
||
except (OSError, sqlite3.Error):
|
||
return False
|
||
|
||
|
||
def scan(run_ids: list[str]) -> list[Path]:
|
||
# DB index only: never enumerate the JuiceFS run tree.
|
||
with readonly_db() as con:
|
||
rows = [row for rid in run_ids for row in con.execute(
|
||
"SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state IN (" +
|
||
','.join('?' for _ in DB_TERMINAL_STATES) + ')', (rid, *DB_TERMINAL_STATES))]
|
||
return sorted(BENCH / 'runs' / rid / jid / 'rollouts' / f'{kk:02d}' for rid, jid, kk in rows)
|
||
|
||
|
||
def annotation_reasons(rows, rid, jid, kk):
|
||
reasons = set()
|
||
for arun, ajob, note in rows:
|
||
if arun != rid or ajob not in (None, jid):
|
||
continue
|
||
match = re.search(r'rollout#(\d+)\b', note or '')
|
||
if match and int(match[1]) != kk:
|
||
continue
|
||
for needle, reason in [('contaminated:', 'contaminated'), ('stalled_killed', 'stalled_killed')]:
|
||
if needle in (note or ''):
|
||
reasons.add(reason)
|
||
return reasons
|
||
|
||
|
||
def next_dataset_id() -> str:
|
||
d = time.strftime("%Y%m%d")
|
||
existing = sorted((BENCH / "datasets").glob(f"ds{d}-*"))
|
||
return f"ds{d}-{len(existing) + 1:02d}"
|
||
|
||
|
||
def _read_json(path: Path, default=None):
|
||
try:
|
||
return json.loads(path.read_text())
|
||
except (OSError, ValueError):
|
||
return {} if default is None else default
|
||
|
||
|
||
def _gate_sources(rd: Path) -> dict:
|
||
"""Fingerprint the verifier's bounded inputs; never copy model artifacts."""
|
||
from harness.verify import rollout_hbm_path
|
||
files = {"job.json": rd.parent.parent / "job.json", "claim.json": rd / "claim.json",
|
||
"rollout.json": rd / "rollout.json",
|
||
"terminal_result.json": rd / "terminal_result.json"}
|
||
for pattern in ("[0-9][0-9]-*/meta.json", "[0-9][0-9]-*/a*/meta.json", "06-hbm/a*/stdout.log", "_queue/*/request.json"):
|
||
for path in (rd / "stages").glob(pattern):
|
||
files[str(path.relative_to(rd))] = path
|
||
if path.name == "request.json":
|
||
recipe = _read_json(path).get("recipe")
|
||
if recipe:
|
||
files["recipe:" + hashlib.sha256(str(recipe).encode()).hexdigest()] = Path(recipe)
|
||
job = _read_json(files["job.json"])
|
||
files["hbm"] = rollout_hbm_path(rd, job.get("job_id", rd.parent.parent.name))
|
||
stage_predictions = {}
|
||
from harness.verify import protected_predictions_path
|
||
for source in tuple(files.values()):
|
||
if source.name != "meta.json":
|
||
continue
|
||
meta = _read_json(source)
|
||
ref = ((meta.get("measurements") or {}).get("G3") or {}).get("predictions")
|
||
if ref:
|
||
key = "predictions:" + str(source.relative_to(rd))
|
||
try:
|
||
files[key] = protected_predictions_path(rd / "stages", ref)
|
||
except (OSError, ValueError, TypeError):
|
||
stage_predictions[key] = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
|
||
terminal = _read_json(rd / "terminal_result.json")
|
||
terminal = terminal if isinstance(terminal, dict) else {}
|
||
invalid_prediction = None
|
||
if terminal.get("predictions"):
|
||
from harness.verify import terminal_predictions_path
|
||
try:
|
||
files["terminal:predictions"] = terminal_predictions_path(rd, terminal)
|
||
except (OSError, ValueError, TypeError):
|
||
invalid_prediction = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
|
||
manifest = dict(stage_predictions)
|
||
if invalid_prediction:
|
||
manifest["terminal:predictions"] = invalid_prediction
|
||
for name, path in sorted(files.items()):
|
||
try:
|
||
digest, size = hashlib.sha256(), 0
|
||
with path.open("rb") as stream:
|
||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||
digest.update(chunk); size += len(chunk)
|
||
manifest[name] = {"sha256": digest.hexdigest(), "bytes": size}
|
||
except FileNotFoundError:
|
||
manifest[name] = {"missing": True}
|
||
return manifest
|
||
|
||
|
||
def _hard_gate_check(job, verdict, manifest, current, measurements=None):
|
||
from harness.verify import eval_set
|
||
required = verdict.get("required") or job.get("gates", {}).get("required", [])
|
||
failures = [gate for gate in required if (verdict.get("gates", {}).get(gate) or {}).get("verdict") not in ("pass", "pass-screen")]
|
||
g3 = (verdict.get("gates") or {}).get("G3") or {}
|
||
expected_n = ((job.get("data") or {}).get("t1") or {}).get("n")
|
||
if "G3" in required and (g3.get("source") != "board" or expected_n is not None and g3.get("n") != expected_n):
|
||
failures.append("G3_SOURCE_OR_SAMPLE_COUNT")
|
||
if "G3" in required:
|
||
expected_set = eval_set(job)
|
||
measured_g3 = (measurements or {}).get("G3") or {}
|
||
measured_set = measured_g3.get("eval_set")
|
||
fields = ("dataset", "n", "content_sha", "manifest_sha", "eval_set_id", "evaluator_version")
|
||
def valid_identity(value):
|
||
return (isinstance(value, dict) and type(value.get('n')) is int and value['n'] > 0
|
||
and all(isinstance(value.get(k), str) and value[k].strip() for k in ('dataset', 'evaluator_version'))
|
||
and all(isinstance(value.get(k), str) and re.fullmatch('[0-9a-fA-F]{64}', value[k])
|
||
for k in ('content_sha', 'manifest_sha')))
|
||
if (not expected_set or not isinstance(measured_set, dict)
|
||
or not valid_identity(expected_set) or not valid_identity(measured_set)
|
||
or any((key != "eval_set_id" and expected_set.get(key) is None)
|
||
or measured_set.get(key) != expected_set[key] for key in fields)):
|
||
failures.append("G3_EVAL_SET_IDENTITY")
|
||
if expected_set and any(key in measured_g3 and measured_g3[key] != expected_set.get(key) for key in fields):
|
||
failures.append("G3_EVAL_SET_IDENTITY_CONFLICT")
|
||
# Current task and training consume the same versioned scorer result.
|
||
from harness.gates import SCORING_RULE
|
||
if verdict.get("scoring_rule") != SCORING_RULE:
|
||
failures.append("SCORING_RULE_MISMATCH")
|
||
if not required:
|
||
failures.append("REQUIRED_GATES_MISSING")
|
||
if manifest != current:
|
||
failures.append("GATE_SOURCE_CHANGED")
|
||
if verdict.get("terminal") != "PASS":
|
||
failures.append("OUTCOME")
|
||
return {"pass": not failures, "required": required, "failures": failures}
|
||
|
||
|
||
def normal_final_response(messages):
|
||
"""An agent_end marker cannot turn a tool tail or failed generation into a final."""
|
||
last = messages[-1] if messages else {}
|
||
content = last.get("content")
|
||
text = content if isinstance(content, str) else "".join(
|
||
item.get("text", "") for item in content or [] if isinstance(item, dict) and item.get("type") == "text")
|
||
ok = (last.get("role") == "assistant" and bool(text.strip()) and not preserved._calls(last)
|
||
and last.get("stopReason") not in ("error", "aborted", "length", "toolUse", "tool_calls"))
|
||
return {"pass": ok, "reason": "PASS" if ok else "MISSING_NORMAL_FINAL_RESPONSE"}
|
||
|
||
|
||
def validate_exported_record(source_root: Path, record: dict, lineage=None, excluded=None) -> dict:
|
||
"""Independent publication check against unchanged source and hard gates.
|
||
|
||
Optional sidecar arguments contain only this attempt's rows. Omitting them
|
||
checks the record payload; publication must additionally supply sidecars.
|
||
"""
|
||
from harness.verify import verify_rollout, collect_measurements
|
||
root = Path(source_root).resolve()
|
||
rd = root / "runs" / record["run_id"] / record["job_id"] / "rollouts" / f"{int(record['rollout']):02d}"
|
||
if not rd.resolve().is_relative_to(root):
|
||
raise ValueError("source path leaves bench root")
|
||
frozen = preserved.freeze(rd)
|
||
expected = preserved.reconstruct(frozen)
|
||
trajectory = record.get("trajectory", {})
|
||
actual = {**expected, "messages": record.get("messages"), "tools": record.get("tools"),
|
||
"source_ids": trajectory.get("source_ids"), "source_manifest": trajectory.get("source_manifest"),
|
||
"redaction": record.get("redact_stats"), "context": trajectory.get("context")}
|
||
for key, rows in (("lineage", lineage), ("excluded", excluded)):
|
||
if rows is not None:
|
||
fields = ("message_id", "observations", "event_type") + (("reason",) if key == "excluded" else ())
|
||
actual[key] = [{field: row[field] for field in fields} for row in rows]
|
||
checks = preserved.validate(frozen, actual)
|
||
checks["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(actual["messages"])
|
||
gate_before = _gate_sources(rd)
|
||
verdict = verify_rollout(rd, write=False)
|
||
measured = collect_measurements(rd)[0]
|
||
job = _read_json(rd.parent.parent / "job.json")
|
||
gate_check = _hard_gate_check(job, verdict, trajectory.get("hard_gate_manifest"), gate_before, measured)
|
||
from harness.verify import eval_set
|
||
if record.get("eval_set") != (measured.get("G3") or {}).get("eval_set") or record.get("required_eval_set") != eval_set(job):
|
||
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["EXPORTED_EVAL_SET_CHANGED"]}
|
||
if gate_before != _gate_sources(rd):
|
||
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["GATE_SOURCE_CHANGED"]}
|
||
checks["checks"]["HARD_GATES"] = gate_check
|
||
from export.content_review import check as check_content_review
|
||
review = check_content_review(rd, verdict, frozen, gate_before)
|
||
if record.get("content_review") != review:
|
||
review = {**review, "pass": False, "reason": "EXPORTED_REVIEW_CHANGED"}
|
||
checks["checks"]["CONTENT_REVIEW"] = review
|
||
checks['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd, root)}
|
||
try:
|
||
# Publication reads the supplied source's own state (only this installation's honours BENCH_STATE_DIR).
|
||
from harness import paths
|
||
uri = (paths.state_dir(root) / 'jobs.sqlite').as_uri() + '?mode=ro'
|
||
with contextlib.closing(sqlite3.connect(uri, uri=True)) as con:
|
||
con.execute('PRAGMA query_only=ON')
|
||
rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?',
|
||
(record['run_id'],)).fetchall()
|
||
reasons = sorted(annotation_reasons(rows, record['run_id'], record['job_id'], int(record['rollout'])))
|
||
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': not reasons, 'reasons': reasons}
|
||
except (OSError, sqlite3.Error, TypeError, ValueError):
|
||
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': False, 'reasons': ['DB_ANNOTATIONS_UNAVAILABLE']}
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
checks["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
checks["pass"] = all(check["pass"] for check in checks["checks"].values())
|
||
return checks
|
||
|
||
|
||
def _inc(counts: dict, key, n=1):
|
||
counts[str(key)] = counts.get(str(key), 0) + n
|
||
|
||
|
||
def _log_tail(path: Path, limit=4096):
|
||
try:
|
||
with path.open("rb") as stream:
|
||
stream.seek(0, os.SEEK_END)
|
||
size = stream.tell()
|
||
stream.seek(max(0, size - limit))
|
||
return {"text": stream.read(limit).decode("utf-8", errors="replace"), "truncated": size > limit}
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def _safe_record(record: dict) -> tuple[dict, dict]:
|
||
# Stage argv can contain secrets split across list elements. Preserve the
|
||
# command structure while masking explicit secret keys/flags before regexes.
|
||
secret_keys = {"password", "passwd", "api_key", "apikey", "authorization", "secret", "access_token"}
|
||
def mask(value):
|
||
if isinstance(value, dict):
|
||
return {k: "<REDACTED>" if k.lower() in secret_keys else mask(v) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
out = [mask(v) for v in value]
|
||
for i, v in enumerate(out[:-1]):
|
||
if isinstance(v, str) and (v.lstrip("-").lower().replace("-", "_") in secret_keys
|
||
or (v == "-p" and "sshpass" in out)):
|
||
out[i + 1] = "<REDACTED>"
|
||
return out
|
||
return value
|
||
return redact_preserving(mask(record))
|
||
|
||
|
||
def pipeline_records(run_ids: list[str]):
|
||
"""One actual execution record per finished oracle job, including failures.
|
||
|
||
DB-indexed only. No recursive traversal of model/data trees and no simulated
|
||
LLM dialogue. Missing evidence is recorded rather than hiding a finished job.
|
||
"""
|
||
from harness.verify import completion_summary, eval_set
|
||
with readonly_db() as con:
|
||
jobs = [dict(row) for rid in run_ids for row in con.execute(
|
||
"SELECT * FROM jobs WHERE run_id=? AND mode='oracle' AND state IN ('PASS','FAIL','UNVERIFIED')", (rid,))]
|
||
stages = [dict(row) for rid in run_ids for row in con.execute(
|
||
"SELECT * FROM stages WHERE run_id=? AND rollout=-1 ORDER BY job_id,stage,attempt", (rid,))]
|
||
by_job = {}
|
||
for stage in stages:
|
||
by_job.setdefault((stage["run_id"], stage["job_id"]), []).append(stage)
|
||
for row in sorted(jobs, key=lambda r: (r["run_id"], r["job_id"])):
|
||
rid, jid = row["run_id"], row["job_id"]
|
||
jd = BENCH / "runs" / rid / jid
|
||
job = _read_json(jd / "job.json")
|
||
result = _read_json(jd / "oracle" / "result.json")
|
||
result_source = "oracle/result.json"
|
||
if not result and row.get("result"):
|
||
try:
|
||
result = json.loads(row["result"])
|
||
result_source = "state/jobs.sqlite:jobs.result"
|
||
except ValueError:
|
||
pass
|
||
actions = []
|
||
for stage in by_job.get((rid, jid), []):
|
||
sd = jd / "oracle" / "stages" / stage["stage"] / f"a{stage['attempt']:02d}"
|
||
meta = _read_json(sd / "meta.json")
|
||
actions.append({"stage": stage["stage"], "attempt": stage["attempt"],
|
||
"database": stage, "meta": meta, "meta_available": bool(meta),
|
||
"command": meta.get("cmd"),
|
||
"source": str(sd.relative_to(BENCH)),
|
||
"logs": {name: str((sd / name).relative_to(BENCH))
|
||
for name in ("stdout.log", "stderr.log") if (sd / name).exists()}})
|
||
if stage["state"] not in ("OK", "SKIPPED"):
|
||
actions[-1]["diagnostic_tail"] = {name: tail for name in ("stdout.log", "stderr.log")
|
||
if (tail := _log_tail(sd / name)) is not None}
|
||
cp = jd / "oracle" / "stages" / "04-calibrate" / "compile.yaml"
|
||
record = {
|
||
"schema": "bench.pipeline-training/1", "source": "deterministic_pipeline",
|
||
"id": hashlib.sha1(f"{rid}/{jid}/oracle".encode()).hexdigest()[:16],
|
||
"run_id": rid, "job_id": jid, "model": job.get("model", row.get("model")),
|
||
"eval_set": eval_set(job), "continuation": job.get("continuation"),
|
||
"task": job.get("task", row.get("task")), "march": job.get("march", row.get("march")),
|
||
"backend": job.get("backend", row.get("backend")), "terminal": result.get("terminal", row["state"]),
|
||
"completion": completion_summary(result.get("gates") or {}, result.get("stages") or {}),
|
||
"started_at": row.get("started_at"), "finished_at": row.get("finished_at"),
|
||
"job": {k: job[k] for k in ("model", "task", "march", "backend", "qcfg", "model_spec", "data", "gates") if k in job},
|
||
"compile_config": cp.read_text(errors="replace") if cp.exists() else None,
|
||
"stages": actions, "result": result,
|
||
"evidence": {"job_available": bool(job), "result_available": bool(result),
|
||
"result_source": result_source if result else None,
|
||
"job_path": str((jd / "job.json").relative_to(BENCH))},
|
||
"training_uses": ["workflow_analysis", "outcome_learning_candidate"],
|
||
"sft_eligible": False, "sft_reason": "deterministic-pipeline-not-agent-dialogue",
|
||
}
|
||
record, redaction = _safe_record(record)
|
||
record["redact_stats"] = redaction
|
||
yield record
|
||
|
||
|
||
def _write_dataset(a, work: Path, ds_id: str) -> dict:
|
||
with readonly_db() as con:
|
||
annotations = con.execute("SELECT run_id, job_id, note FROM annotations").fetchall()
|
||
coverage = {rid: {
|
||
"oracle": dict(con.execute("SELECT state,count(*) FROM jobs WHERE run_id=? AND mode='oracle' GROUP BY state", (rid,))),
|
||
"agent": dict(con.execute("SELECT state,count(*) FROM rollouts WHERE run_id=? GROUP BY state", (rid,))),
|
||
} for rid in a.runs}
|
||
stats = {"scanned": 0, "written": 0, "agent_written": 0, "teacher_written": 0, "student_written": 0,
|
||
"pipeline_written": 0, "excluded": 0, "dropped": {}, "review": 0, "partial": 0,
|
||
"by_terminal": {}, "by_execution_terminal": {}, "pipeline_by_terminal": {}, "sft_eligible": 0,
|
||
"by_march": {}, "sft_eligible_by_march": {}, "reverified": 0}
|
||
with contextlib.ExitStack() as stack:
|
||
names = ("teacher", "student", "pipeline", "exclusions", "sft", "retained", "lineage",
|
||
"operations", "excluded_observations", "validation", "annotations", "source-manifest", "disposition")
|
||
files = {name: stack.enter_context((work / f"{name}.jsonl").open("w")) for name in names}
|
||
source_seen = set()
|
||
def write(name, rec):
|
||
files[name].write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
|
||
if name == "source-manifest":
|
||
source_seen.add((rec["run_id"], rec["job_id"], rec["rollout"]))
|
||
def exclude(rd, reasons, review_binding=None, review_receipt=None):
|
||
stats["excluded"] += 1
|
||
for reason in reasons:
|
||
_inc(stats["dropped"], reason)
|
||
write("exclusions", {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name,
|
||
"rollout": int(rd.name), "source": str(rd.relative_to(BENCH)),
|
||
"reasons": sorted(reasons)})
|
||
disposition = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name),
|
||
"source": str(rd.relative_to(BENCH)), "reasons": sorted(reasons),
|
||
"disposition": "no-usable" if "no_usable_messages" in reasons else "retained"}
|
||
if review_binding:
|
||
disposition["review_binding_sha256"] = review_binding
|
||
disposition["review_receipt_sha256"] = review_receipt
|
||
write("disposition", disposition)
|
||
write("retained", disposition)
|
||
if a.purpose == "all":
|
||
for record in pipeline_records(a.runs):
|
||
write("pipeline", record)
|
||
stats["pipeline_written"] += 1
|
||
_inc(stats["pipeline_by_terminal"], record["terminal"])
|
||
for rd in scan(a.runs):
|
||
_write_agent(rd, a, stats, annotations, write, exclude)
|
||
ident = (rd.parents[2].name, rd.parents[1].name, int(rd.name))
|
||
if ident not in source_seen:
|
||
source = preserved.freeze(rd)
|
||
try:
|
||
gate_sources = _gate_sources(rd)
|
||
except (OSError, ValueError, TypeError, KeyError) as error:
|
||
# Rejected evidence must not abort the whole run's export.
|
||
gate_sources = {}
|
||
source.errors.append(f"HARD_GATES_UNAVAILABLE:{type(error).__name__}")
|
||
write("source-manifest", {"run_id": ident[0], "job_id": ident[1], "rollout": ident[2],
|
||
"source": str(rd.relative_to(BENCH)), "files": source.manifest,
|
||
"errors": source.errors, "hard_gates": gate_sources})
|
||
file_info = {}
|
||
for name in names:
|
||
path = work / f"{name}.jsonl"
|
||
with path.open("rb") as stream:
|
||
digest, lines = hashlib.sha256(), 0
|
||
for line in stream:
|
||
digest.update(line); lines += 1
|
||
file_info[path.name] = {"sha256": digest.hexdigest(), "records": lines, "bytes": path.stat().st_size}
|
||
manifest = {"schema": "bench.dataset/2", "dataset_id": ds_id, "runs": a.runs,
|
||
"reward_recipe": RECIPE_VERSION, "created_at": time.time(), "coverage": coverage,
|
||
"filters": {"min_agent_version": a.min_agent_version,
|
||
"min_agent_version_runner": a.min_agent_version_runner,
|
||
"max_turns": a.max_turns, "max_tokens": a.max_tokens,
|
||
"agent_version": a.agent_version, "agent_version_mode": "exact" if a.agent_version else "minimum",
|
||
"annotations_read": True, "dropped_counts_overlap": True, "policy": a.policy,
|
||
"include_timeout": True, "purpose": a.purpose, "min_reward": a.min_reward},
|
||
"stats": stats, "files": file_info,
|
||
"notes": ["pipeline.jsonl contains historical deterministic evidence only for purpose=all; SFT/RL export reads Agent evidence only.",
|
||
f"Verdict v4 and reward {RECIPE_VERSION} use only this rollout evidence; Oracle references are history only.",
|
||
"SFT/reward candidates are labels; no train split, framework conversion, or online RL readiness is implied.",
|
||
"Writers hold .export.lock; manifest is published last. Readers can lock or verify file hashes for a coherent refresh."]}
|
||
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||
return manifest
|
||
|
||
|
||
def _write_agent(rd, a, stats, annotations, write, exclude):
|
||
from harness.verify import eval_set, verify_rollout, collect_measurements, VERDICT_SCHEMA, VERIFICATION_RULE
|
||
stats["scanned"] += 1
|
||
original_verdict = _read_json(rd / "verdict.json")
|
||
verdict = original_verdict
|
||
rj = _read_json(rd / "rollout.json")
|
||
if not verdict:
|
||
exclude(rd, {"missing_verdict"}); return
|
||
policy = verdict.get("policy") or rj.get("policy") or "teacher"
|
||
if a.policy and policy != a.policy:
|
||
exclude(rd, {"policy_filter"}); return
|
||
if policy not in ("teacher", "student"):
|
||
exclude(rd, {"unknown_policy"}); return
|
||
reasons = annotation_reasons(annotations, rd.parents[2].name, rd.parents[1].name, int(rd.name))
|
||
runner = rj.get("runner", "pi")
|
||
version = str(rj.get("agent_config_version") or 1)
|
||
if a.agent_version is not None:
|
||
if version != str(a.agent_version):
|
||
reasons.add("old_version")
|
||
elif a.min_agent_version_runner in ("*", runner):
|
||
try:
|
||
if int(version) < a.min_agent_version:
|
||
reasons.add("old_version")
|
||
except ValueError:
|
||
reasons.add("invalid_version")
|
||
if reasons:
|
||
exclude(rd, reasons); return
|
||
original_sha = hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest()
|
||
reverified = verdict.get("schema") != VERDICT_SCHEMA
|
||
try:
|
||
gate_manifest = _gate_sources(rd)
|
||
verdict = verify_rollout(rd, write=False)
|
||
stats["reverified"] += int(reverified)
|
||
except Exception as error:
|
||
exclude(rd, {f"reverification_failed:{type(error).__name__}"}); return
|
||
verification = {"schema": verdict.get("schema"),
|
||
"rule": verdict.get("verification_rule", VERIFICATION_RULE),
|
||
"method": "readonly-reverification"}
|
||
strict = None
|
||
try:
|
||
if runner == "pi":
|
||
frozen = preserved.freeze(rd)
|
||
strict = preserved.reconstruct(frozen)
|
||
validation = preserved.validate(frozen, strict)
|
||
validation["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(strict["messages"])
|
||
validation["checks"]["HARD_GATES"] = _hard_gate_check(_read_json(rd.parent.parent / "job.json"), verdict,
|
||
gate_manifest, _gate_sources(rd), collect_measurements(rd)[0])
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
|
||
msgs, tools = strict["messages"], strict["tools"]
|
||
legacy_messages, _ = clean(msgs)
|
||
strict["annotations"].extend(preserved.legacy_truncation_audit(msgs, strict["source_ids"], legacy_messages))
|
||
raw = msgs
|
||
trajectory = {k: v for k, v in strict.items() if k not in ("messages", "tools", "redaction", "lineage", "operations", "excluded")}
|
||
trajectory.update(source="pi-segment-events" if strict["agent_end"] else "message_end",
|
||
partial=any(not check["pass"] for name, check in validation["checks"].items() if name != "HARD_GATES"),
|
||
validation=validation, hard_gate_manifest=gate_manifest)
|
||
ident = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name)}
|
||
for name, values in (("lineage", strict["lineage"]), ("operations", strict["operations"]),
|
||
("excluded_observations", strict["excluded"]),
|
||
("annotations", strict["annotations"] if validation["checks"]["PRIVACY"]["pass"] else [])):
|
||
for value in values:
|
||
write(name, {**ident, **value})
|
||
write("source-manifest", {**ident, "source": str(rd.relative_to(BENCH)), "files": frozen.manifest,
|
||
"hard_gates": gate_manifest})
|
||
pairing = validation["checks"]["CALL_TOOL_CLOSURE"]
|
||
cst = {"in": len(msgs), "out": len(msgs), "assistant_turns": sum(m.get("role") == "assistant" for m in msgs),
|
||
"drop": not validation["checks"]["USABLE_ASSISTANT"]["pass"], "r7_truncated": 0,
|
||
"tool_pairing": {"valid": pairing["pass"], "missing_results": pairing["missing"],
|
||
"orphan_results": pairing["orphan"], "duplicate_or_missing_call_ids": pairing["call_id_conflict"]}}
|
||
sys_text = frozen.files.get("agent/system.md", b"").decode("utf-8", errors="strict")
|
||
sys_src = strict["context"]["source"] if sys_text else "missing"
|
||
tools_src, rst = ("snapshot" if tools else "missing"), strict["redaction"]
|
||
else:
|
||
raw, trajectory = load_trajectory(rd)
|
||
msgs, cst = clean(raw)
|
||
sys_text, sys_src = system_prompt(rd)
|
||
if sys_text:
|
||
msgs.insert(0, {"role": "system", "content": sys_text})
|
||
tools, tools_src = tool_defs(rd)
|
||
msgs, rst = redact_messages(msgs)
|
||
trajectory["strict_sft_reason"] = "UNSUPPORTED_SOURCE"
|
||
except (OSError, ValueError, KeyError, TypeError) as exc:
|
||
exclude(rd, {f"invalid_trajectory:{type(exc).__name__}"}); return
|
||
if not raw:
|
||
exclude(rd, {"no_trajectory"}); return
|
||
cst["system_source"] = sys_src
|
||
if sys_src == "runner-raw-system":
|
||
trajectory["system_segments"] = _read_json(rd / "agent/raw-system.json", default=[])
|
||
if cst["drop"]:
|
||
if strict is not None:
|
||
write("validation", {**ident, **validation})
|
||
exclude(rd, {"no_usable_messages"}); return
|
||
jd = rd.parent.parent
|
||
job = json.loads((jd / "job.json").read_text())
|
||
mode = verdict.get("mode") or rj.get("mode")
|
||
verdict = {**verdict, "mode": mode}
|
||
rw = compute_reward(verdict, rj.get("max_turns", a.max_turns), rj.get("max_tokens", a.max_tokens))
|
||
sft_ok, sft_reason = sft_eligibility(verdict, rw["reward"])
|
||
required = verdict.get("required") or job.get("gates", {}).get("required", [])
|
||
if sft_ok and (not required or any((verdict.get("gates", {}).get(g) or {}).get("verdict") not in ("pass", "pass-screen") for g in required)):
|
||
sft_ok, sft_reason = False, "required-hard-gate-not-passed"
|
||
if sft_ok and strict is None:
|
||
sft_ok, sft_reason = False, "unsupported-strict-sft-source"
|
||
if sft_ok and strict is not None and not validation["checks"]["HARD_GATES"]["pass"]:
|
||
sft_ok, sft_reason = False, "hard-gate-validation-failed"
|
||
trajectory["partial"] = trajectory["partial"] or not cst["tool_pairing"]["valid"] or cst.get("trailing_user", False)
|
||
if sft_ok and trajectory["partial"]:
|
||
sft_ok, sft_reason = False, "partial-transcript"
|
||
if sft_ok and (not tools or not sys_text):
|
||
sft_ok, sft_reason = False, "missing-tools-or-system"
|
||
known_tools = {t.get("name") or (t.get("function") or {}).get("name") for t in tools if isinstance(t, dict)}
|
||
used_tools = {tc.get("name") for msg in msgs for tc in preserved._calls(msg)}
|
||
missing_tools = sorted(name for name in used_tools - known_tools if name)
|
||
trajectory["missing_tool_definitions"] = missing_tools
|
||
if sft_ok and missing_tools:
|
||
sft_ok, sft_reason = False, "missing-called-tool-definitions"
|
||
if strict is not None:
|
||
validation['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd)}
|
||
if not validation['checks']['DB_SOURCE_TERMINAL']['pass']:
|
||
validation['pass'] = False
|
||
sft_ok, sft_reason = False, 'source-not-terminal'
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
validation["pass"] = False
|
||
trajectory["partial"] = True
|
||
sft_ok, sft_reason = False, "source-changed"
|
||
if a.min_reward is not None and rw["reward"] < a.min_reward:
|
||
exclude(rd, {"min_reward"}); return
|
||
content_review = {"pass": False, "reason": "NOT_SCRIPT_QUALIFIED"}
|
||
if sft_ok:
|
||
from export.content_review import check as check_content_review
|
||
if getattr(a, "review", False) and a.purpose in ("all", "sft"):
|
||
from export.content_review import review_once
|
||
try:
|
||
review_once(rd)
|
||
except (OSError, ValueError):
|
||
sft_ok, sft_reason = False, "content-review-unavailable"
|
||
content_review = check_content_review(rd, verdict, frozen, gate_manifest)
|
||
if not preserved.source_unchanged(rd, frozen) or _gate_sources(rd) != gate_manifest:
|
||
sft_ok, sft_reason = False, "source-changed"
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
if not content_review["pass"]:
|
||
sft_ok, sft_reason = False, "content-review-" + content_review["reason"].lower()
|
||
if strict is not None:
|
||
validation["checks"]["CONTENT_REVIEW"] = content_review
|
||
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
|
||
write("validation", {**ident, **validation})
|
||
if a.purpose == "sft" and not sft_ok:
|
||
exclude(rd, {f"sft:{sft_reason}"}, content_review.get("binding_sha256"),
|
||
content_review.get("receipt_sha256")); return
|
||
reward_candidate = verdict["terminal"] in ("PASS", "FAIL") and mode != "repair" and not rw.get("unverifiable")
|
||
if a.purpose == "rl" and not reward_candidate:
|
||
exclude(rd, {"rl:historical-repair-or-unverified-outcome"}); return
|
||
if a.min_reward is not None and rw["reward"] < a.min_reward:
|
||
exclude(rd, {"min_reward"}); return
|
||
rec = {
|
||
"schema": "bench.agent-training/2", "source": "agent_events",
|
||
"id": hashlib.sha1(f"{verdict['run_id']}/{verdict['job_id']}/{verdict['rollout']}".encode()).hexdigest()[:16],
|
||
"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
|
||
"eval_set": ((collect_measurements(rd)[0].get("G3") or {}).get("eval_set")),
|
||
"required_eval_set": eval_set(job),
|
||
"continuation": {**job["continuation"], "source_rollout": verdict["rollout"]} if job.get("continuation") else None,
|
||
"mode": mode, "policy": policy, "runner": runner, "model": rj.get("model"),
|
||
"agent_config_version": str(rj.get("agent_config_version", "1")),
|
||
"runner_version": rj.get("runner_version"),
|
||
"task": job["task"], "march": job["march"],
|
||
"board_type": (job.get("target") or {}).get("board_type"), "backend": job["backend"],
|
||
"messages": msgs,
|
||
"verdict": {k: verdict[k] for k in ("schema", "verification_rule", "terminal", "measured_terminal", "execution_terminal", "timeout_reason", "completion", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict},
|
||
"gates": {g: (verdict["gates"].get(g) or {}).get("verdict") for g in ("G1", "G2", "G3", "G4")},
|
||
"verification": verification,
|
||
"reward": rw["reward"], "reward_terms": rw["terms"], "reward_recipe": rw["recipe"],
|
||
"reward_unverifiable": rw.get("unverifiable"), # v0.4:board_offline = 板卡故障导致没法测,不是 agent 的结果
|
||
"sft_eligible": sft_ok, "sft_reason": sft_reason,
|
||
"training_uses": {"analysis": True, "sft_candidate": sft_ok, "reward_learning_candidate": reward_candidate,
|
||
"review_required": not content_review["pass"]},
|
||
"content_review": content_review,
|
||
"trajectory": trajectory,
|
||
"execution": {"terminal": verdict.get("execution_terminal") or rj.get("state"),
|
||
"timeout_reason": verdict.get("timeout_reason") or rj.get("timeout_reason"),
|
||
"started_at": rj.get("started_at"), "finished_at": rj.get("finished_at"),
|
||
"max_turns": rj.get("max_turns"), "max_tokens": rj.get("max_tokens")},
|
||
"provenance": {"events": str((rd / "agent" / "events.jsonl").relative_to(BENCH)),
|
||
"verdict": str((rd / "verdict.json").relative_to(BENCH)),
|
||
"original_verdict": {"sha256": original_sha, "schema": original_verdict.get("schema"),
|
||
"terminal": original_verdict.get("terminal"),
|
||
"review": original_verdict.get("review")}},
|
||
"cost": verdict.get("cost"),
|
||
"tools": tools,
|
||
"tools_source": tools_src,
|
||
"system_source": sys_src,
|
||
"system_sha1": hashlib.sha1(sys_text.encode()).hexdigest()[:12] if sys_text else None,
|
||
"clean_stats": cst, "redact_stats": rst,
|
||
}
|
||
# Preserve the already validated context exactly; redact metadata separately
|
||
# so a second traversal cannot silently rewrite call/source identities.
|
||
context = {key: rec.pop(key) for key in ("messages", "tools", "trajectory", "redact_stats")}
|
||
rec, _ = _safe_record(rec)
|
||
rec.update(context)
|
||
if strict is not None and not validation["checks"]["PRIVACY"]["pass"]:
|
||
# Identity cannot be rewritten safely. Keep original bytes only in the
|
||
# controlled source; public evidence contains a hash/reference receipt.
|
||
rec = {key: rec[key] for key in ("schema", "source", "id", "run_id", "job_id", "rollout", "sft_eligible", "sft_reason")}
|
||
rec.update(source_reference=str(rd.relative_to(BENCH)), reason_code="PRIVATE_SOURCE_REFERENCE_ONLY",
|
||
trajectory={"source_manifest": frozen.manifest, "validation": validation})
|
||
write(policy, rec)
|
||
write("sft" if sft_ok else "retained", rec)
|
||
write("disposition", {"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
|
||
"source": str(rd.relative_to(BENCH)), "disposition": "accepted" if sft_ok else "retained",
|
||
"review_binding_sha256": content_review.get("binding_sha256"),
|
||
"review_receipt_sha256": content_review.get("receipt_sha256"),
|
||
"reasons": [] if sft_ok else [sft_reason]})
|
||
stats["written"] += 1
|
||
stats["agent_written"] += 1
|
||
stats[f"{policy}_written"] += 1
|
||
stats["partial"] += int(trajectory["partial"])
|
||
_inc(stats["by_terminal"], verdict["terminal"])
|
||
_inc(stats["by_execution_terminal"], verdict.get("execution_terminal") or rj.get("state"))
|
||
_inc(stats["by_march"], job["march"])
|
||
stats["sft_eligible"] += int(sft_ok)
|
||
if sft_ok:
|
||
_inc(stats["sft_eligible_by_march"], job["march"])
|
||
|
||
def materialize(run_ids: list[str], dataset_id: str, *, refresh=False, purpose=_UNSET, policy=_UNSET,
|
||
min_agent_version=_UNSET, agent_version=_UNSET, min_reward=_UNSET,
|
||
max_turns=_UNSET, max_tokens=_UNSET, review=False) -> dict:
|
||
"""Refresh one stable output directory; locks writers, replaces files atomically.
|
||
|
||
The manifest commits last and supplies hashes/counts. Concurrent consumers
|
||
should share .export.lock or check those hashes; the directory is not swapped.
|
||
Source rollout/run artifacts are never modified.
|
||
"""
|
||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", dataset_id):
|
||
raise ValueError("dataset_id must be a simple directory name")
|
||
if review and not (os.environ.get("BENCH_REVIEW_URL") and os.environ.get("BENCH_REVIEW_MODEL")):
|
||
raise ValueError("--review requires BENCH_REVIEW_URL and BENCH_REVIEW_MODEL")
|
||
run_ids = list(dict.fromkeys(run_ids))
|
||
out_dir = BENCH / "datasets" / dataset_id
|
||
if out_dir.is_symlink():
|
||
raise ValueError("refusing to write a symlink dataset")
|
||
try:
|
||
out_dir.mkdir(parents=True, exist_ok=False)
|
||
except FileExistsError:
|
||
if not refresh:
|
||
raise FileExistsError(f"dataset exists: {out_dir}; use --refresh explicitly")
|
||
if (out_dir / "VOID").exists():
|
||
raise ValueError(f"refusing to refresh VOID dataset: {dataset_id}")
|
||
requested = {"purpose": purpose, "policy": policy, "agent_version": agent_version,
|
||
"min_agent_version": min_agent_version, "min_reward": min_reward,
|
||
"max_turns": max_turns, "max_tokens": max_tokens}
|
||
with (out_dir / ".export.lock").open("a") as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
prior = _read_json(out_dir / "manifest.json")
|
||
if prior and set(prior.get("runs", [])) != set(run_ids):
|
||
raise ValueError("refresh must keep the same run set; choose a new dataset_id")
|
||
defaults = {"purpose": "all", "policy": None, "agent_version": None,
|
||
"min_agent_version": 0, "min_reward": None,
|
||
"max_turns": 40, "max_tokens": 200000}
|
||
previous = prior.get("filters", {})
|
||
if min_agent_version is None or (min_agent_version is _UNSET and "min_agent_version" not in previous):
|
||
version_file = BENCH / "agent/VERSION"
|
||
defaults["min_agent_version"] = int(version_file.read_text()) if version_file.exists() else 1
|
||
resolved = {key: previous.get(key, default) if requested[key] is _UNSET else requested[key]
|
||
for key, default in defaults.items()}
|
||
scope = previous.get("min_agent_version_runner", "*" if prior else "pi")
|
||
if min_agent_version is not _UNSET:
|
||
scope = "pi" if min_agent_version is None else "*"
|
||
if resolved["min_agent_version"] is None:
|
||
resolved["min_agent_version"] = defaults["min_agent_version"] if min_agent_version is None else 0
|
||
if resolved["purpose"] not in ("all", "sft", "rl"):
|
||
raise ValueError("purpose must be all/sft/rl")
|
||
if resolved["policy"] not in (None, "teacher", "student"):
|
||
raise ValueError("policy must be teacher/student or None")
|
||
if not isinstance(resolved["min_agent_version"], int) or resolved["min_agent_version"] < 0:
|
||
raise ValueError("min_agent_version must be a nonnegative integer")
|
||
a = argparse.Namespace(runs=run_ids, min_agent_version_runner=scope, review=review, **resolved)
|
||
with tempfile.TemporaryDirectory(prefix=".refresh-", dir=out_dir) as temp:
|
||
work = Path(temp)
|
||
manifest = _write_dataset(a, work, dataset_id)
|
||
if prior.get("created_at"):
|
||
manifest["created_at"] = prior["created_at"]
|
||
manifest["updated_at"] = time.time()
|
||
manifest["filter_history"] = list(prior.get("filter_history", []))
|
||
if prior and previous != manifest["filters"]:
|
||
explicit = sorted(k for k, v in requested.items() if v is not _UNSET)
|
||
manifest["filter_history"].append({"changed_at": manifest["updated_at"],
|
||
"reason": "explicit_filter_change" if explicit else "legacy_filter_metadata_upgrade",
|
||
"explicit_fields": explicit, "before": previous, "after": manifest["filters"]})
|
||
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||
for name in [*manifest["files"], "manifest.json"]:
|
||
os.replace(work / name, out_dir / name)
|
||
return manifest
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--runs", nargs="+", required=True)
|
||
ap.add_argument("--policy", choices=["teacher", "student"], default=argparse.SUPPRESS)
|
||
ap.add_argument("--include-timeout", action="store_true", help="compatibility flag: timeout evidence is now included by default")
|
||
ap.add_argument("--purpose", choices=["all", "sft", "rl"], default=argparse.SUPPRESS,
|
||
help="all=保留终态轨迹并标注用途;sft=完整诚实成功候选;rl=本次可核验 PASS/FAIL 候选")
|
||
ap.add_argument("--min-reward", type=float, default=argparse.SUPPRESS)
|
||
ap.add_argument("--max-turns", type=int, default=argparse.SUPPRESS)
|
||
ap.add_argument("--max-tokens", type=int, default=argparse.SUPPRESS)
|
||
ap.add_argument("--dataset-id")
|
||
ap.add_argument("--review", action="store_true", help="explicitly review script-qualified candidates once; requires reviewer environment")
|
||
ap.add_argument("--refresh", action="store_true", help="explicitly refresh this dataset from its source runs")
|
||
ap.add_argument("--agent-version", default=argparse.SUPPRESS)
|
||
ap.add_argument("--min-agent-version", type=int, default=argparse.SUPPRESS)
|
||
a = ap.parse_args()
|
||
filters = {key: value for key, value in vars(a).items() if key in (
|
||
"purpose", "policy", "min_agent_version", "agent_version", "min_reward", "max_turns", "max_tokens")}
|
||
manifest = materialize(a.runs, a.dataset_id or next_dataset_id(), refresh=a.refresh, review=a.review, **filters)
|
||
print(json.dumps(manifest, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|