Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
992 lines
55 KiB
Python
992 lines
55 KiB
Python
"""L4 data materialization: preserve Pi source events, verify, then publish.
|
||
|
||
python export/to_agentrl.py --runs r20260914-01 [r…] [--policy teacher] [--include-timeout]
|
||
|
||
Legacy analysis-only cleaning (never used for strict Pi SFT):
|
||
R1 双方言归一:OpenAI / Anthropic 形状 → {role, content, tool_calls?, tool_call_id?}
|
||
R2 剥离 thinking / reasoning 块
|
||
R3 system 只在首条且只有一条(多条合并)
|
||
R4 保留完整工具调用/结果;缺失配对的轨迹标记 partial,不进入成功 SFT
|
||
R5 去空消息
|
||
R6 合并相邻同 role 的纯文本消息
|
||
R7 tool 结果截断(头 6k + 尾 2k 字符)
|
||
R8 隔离:无 assistant 消息,或 messages < 2(排除理由落 exclusions.jsonl)
|
||
轨迹源:agent_end.messages[],中断时恢复已完成的 message_end;不会捏造 assistant 回答。
|
||
默认保存全部终态;SFT 候选另行标记。pipeline.jsonl 保存真实确定性流程记录,和 Agent 消息分开。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import contextlib
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import sqlite3
|
||
import os
|
||
import re
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
BENCH = Path(os.environ.get("BENCH_ROOT", HERE.parent))
|
||
sys.path.insert(0, str(HERE.parent))
|
||
sys.path.insert(0, str(HERE))
|
||
from redact import redact_messages, redact_preserving # noqa: E402
|
||
from export import trajectory as preserved # noqa: E402
|
||
from reward import compute as compute_reward, RECIPE_VERSION # noqa: E402
|
||
|
||
TOOL_HEAD, TOOL_TAIL = 6000, 2000
|
||
_TOOLS_CACHE = {}
|
||
_UNSET = object()
|
||
|
||
|
||
# ---------------------------------------------------------------- 轨迹提取
|
||
|
||
def system_prompt(rd: Path) -> tuple[str, str]:
|
||
"""取这条轨迹的系统提示 → (text, source)。
|
||
|
||
pi 的 --append-system-prompt 内容**不出现在** agent_end.messages[] 里(实测 0.73.1):
|
||
导出时若不补回来,SFT 样本就只有 user 任务 + assistant 动作,没有"只能写 workspace 等"这些规则,
|
||
等于训练一个看不见规则的策略。优先读 rollout 自己的快照,其次按 cmd.json 里的路径读当前文件(会漂移,故记 source)。
|
||
"""
|
||
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
|
||
segments = _read_json(rd / "agent/raw-system.json", default=[])
|
||
if isinstance(segments, list) and segments and all(isinstance(v, str) for v in segments):
|
||
return "\n\n".join(segments), "runner-raw-system"
|
||
return "", "missing"
|
||
snap = rd / "agent" / "system.md"
|
||
if snap.exists():
|
||
return snap.read_text(errors="replace"), "snapshot"
|
||
cmd = rd / "agent" / "cmd.json"
|
||
if cmd.exists():
|
||
try:
|
||
argv = json.loads(cmd.read_text())["cmd"]
|
||
i = argv.index("--append-system-prompt")
|
||
f = Path(argv[i + 1])
|
||
if f.exists():
|
||
return f.read_text(errors="replace"), "live-file"
|
||
except (ValueError, KeyError, IndexError, OSError):
|
||
pass
|
||
return "", "missing"
|
||
|
||
|
||
def tool_defs(rd: Path) -> tuple[list, str]:
|
||
"""这条轨迹的工具定义 → (list, source)。快照没有就按 cmd.json 的 --tools 白名单现 dump 一次。"""
|
||
snap = rd / "agent" / "tools.json"
|
||
if snap.exists():
|
||
try:
|
||
definitions = json.loads(snap.read_text())
|
||
if isinstance(definitions, list):
|
||
return definitions, "snapshot"
|
||
except json.JSONDecodeError:
|
||
pass
|
||
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
|
||
return [], "missing"
|
||
cmd = rd / "agent" / "cmd.json"
|
||
if cmd.exists():
|
||
try:
|
||
argv = json.loads(cmd.read_text())["cmd"]
|
||
wl = argv[argv.index("--tools") + 1]
|
||
import subprocess
|
||
from harness import paths
|
||
node = os.path.expanduser(str(paths.config()["paths"].get("node_bin") or "node"))
|
||
if "/" in node and not Path(node).is_absolute():
|
||
node = str(BENCH / node)
|
||
env = os.environ.copy()
|
||
env["PATH"] = str(Path(node).parent) + os.pathsep + env.get("PATH", "/usr/bin:/bin")
|
||
script = BENCH / "agent" / "dump_tools.js"
|
||
key = (node, str(script), script.stat().st_mtime_ns, wl)
|
||
if key in _TOOLS_CACHE:
|
||
return _TOOLS_CACHE[key], "redumped"
|
||
r = subprocess.run([node, str(script), wl],
|
||
capture_output=True, text=True, timeout=60, env=env)
|
||
if r.returncode == 0 and r.stdout.strip():
|
||
definitions = json.loads(r.stdout)
|
||
if isinstance(definitions, list) and definitions:
|
||
_TOOLS_CACHE[key] = definitions
|
||
return definitions, "redumped"
|
||
except Exception:
|
||
pass
|
||
return [], "missing"
|
||
|
||
|
||
def load_trajectory(rd: Path) -> tuple[list[dict], dict]:
|
||
"""Read every Pi segment; container mirrors cannot replace earlier history."""
|
||
runner = _read_json(rd / "rollout.json").get("runner", "pi")
|
||
if runner != "pi":
|
||
from harness.runners import get
|
||
return get(runner).trajectory(rd)
|
||
scanned = preserved.inventory(preserved.freeze(rd))
|
||
return [item["payload"] for item in scanned["messages"] if item["event_type"] != "system_snapshot"], {
|
||
"source": "pi-segment-events" if scanned["agent_end"] else "message_end",
|
||
"partial": bool(scanned["errors"]), "agent_end": scanned["agent_end"], "errors": scanned["errors"]}
|
||
|
||
|
||
def load_messages(rd: Path) -> list[dict] | None:
|
||
return load_trajectory(rd)[0] or None
|
||
|
||
|
||
# ---------------------------------------------------------------- R1–R8
|
||
|
||
def _text_of(content) -> str:
|
||
if isinstance(content, str):
|
||
return content
|
||
if isinstance(content, list):
|
||
parts = []
|
||
for c in content:
|
||
if isinstance(c, dict):
|
||
t = c.get("type")
|
||
if t in ("text", "output_text") and c.get("text"):
|
||
parts.append(c["text"])
|
||
elif t in ("thinking", "reasoning", "redacted_thinking"):
|
||
continue # R2
|
||
elif isinstance(c, str):
|
||
parts.append(c)
|
||
return "\n".join(parts)
|
||
return "" if content is None else str(content)
|
||
|
||
|
||
def r1_normalize(m: dict) -> list[dict]:
|
||
"""一条原始消息 → 0..n 条归一消息(Anthropic 的 tool_use/tool_result 混在 content 里要拆)。"""
|
||
role = m.get("role")
|
||
content = m.get("content")
|
||
out: list[dict] = []
|
||
if role in ("toolResult", "tool_result", "tool"):
|
||
return [{"role": "tool", "tool_call_id": m.get("toolCallId") or m.get("tool_call_id"),
|
||
"content": _text_of(content) or json.dumps(m.get("output") or m.get("result") or "", ensure_ascii=False)}]
|
||
if role == "assistant":
|
||
tool_calls = []
|
||
if isinstance(content, list):
|
||
for c in content:
|
||
if isinstance(c, dict) and c.get("type") in ("toolCall", "tool_use", "function_call"):
|
||
tool_calls.append({"id": c.get("id"), "name": c.get("name"),
|
||
"arguments": c.get("arguments") if "arguments" in c else c.get("input")})
|
||
for tc in m.get("tool_calls") or []:
|
||
fn = tc.get("function") or {}
|
||
tool_calls.append({"id": tc.get("id"), "name": fn.get("name") or tc.get("name"),
|
||
"arguments": fn.get("arguments") if fn else tc.get("arguments")})
|
||
msg = {"role": "assistant", "content": _text_of(content)}
|
||
if tool_calls:
|
||
msg["tool_calls"] = tool_calls
|
||
return [msg]
|
||
if role == "custom":
|
||
# pi 扩展塞进会话的消息(如 budget-kill 的预算提醒):模型是当 user 输入看到的,训练样本也要有
|
||
t = _text_of(content)
|
||
return [{"role": "user", "content": f"[{m.get('customType', 'extension')}] {t}"}] if t else []
|
||
if role in ("user", "system"):
|
||
if isinstance(content, list) and any(isinstance(c, dict) and c.get("type") == "tool_result" for c in content):
|
||
for c in content:
|
||
if isinstance(c, dict) and c.get("type") == "tool_result":
|
||
out.append({"role": "tool", "tool_call_id": c.get("tool_use_id"), "content": _text_of(c.get("content"))})
|
||
else:
|
||
t = _text_of([c])
|
||
if t:
|
||
out.append({"role": "user", "content": t})
|
||
return out
|
||
return [{"role": role, "content": _text_of(content)}]
|
||
return []
|
||
|
||
|
||
def clean(messages: list[dict]) -> tuple[list[dict], dict]:
|
||
stats = {"in": len(messages)}
|
||
msgs: list[dict] = []
|
||
for m in messages:
|
||
msgs.extend(r1_normalize(m))
|
||
# R3 system 合并到首条
|
||
systems = [m["content"] for m in msgs if m["role"] == "system"]
|
||
msgs = [m for m in msgs if m["role"] != "system"]
|
||
if systems:
|
||
msgs.insert(0, {"role": "system", "content": "\n\n".join(s for s in systems if s)})
|
||
# R5 去空
|
||
msgs = [m for m in msgs if m.get("content") or m.get("tool_calls") or m["role"] == "tool"]
|
||
# R7 tool 结果截断
|
||
for m in msgs:
|
||
if m["role"] == "tool" and len(m["content"]) > TOOL_HEAD + TOOL_TAIL:
|
||
m["content"] = m["content"][:TOOL_HEAD] + f"\n…[truncated {len(m['content']) - TOOL_HEAD - TOOL_TAIL} chars]…\n" + m["content"][-TOOL_TAIL:]
|
||
stats["r7_truncated"] = stats.get("r7_truncated", 0) + 1
|
||
# R6 合并相邻同 role 纯文本
|
||
merged: list[dict] = []
|
||
for m in msgs:
|
||
if merged and m["role"] == merged[-1]["role"] and m["role"] in ("user", "assistant") \
|
||
and not m.get("tool_calls") and not merged[-1].get("tool_calls"):
|
||
merged[-1]["content"] += "\n" + m["content"]
|
||
stats["r6_merged"] = stats.get("r6_merged", 0) + 1
|
||
else:
|
||
merged.append(m)
|
||
msgs = merged
|
||
# R4 Preserve completed tool exchanges, including a trailing tool result.
|
||
# Dropping that result used to manufacture a dangling assistant call.
|
||
# Incomplete/interior-invalid exchanges remain useful evidence, but cannot
|
||
# enter SFT; never fabricate tool results or an assistant final answer.
|
||
pending, seen = set(), set()
|
||
missing, orphan, duplicates = [], [], []
|
||
for m in msgs:
|
||
if m["role"] != "tool" and pending:
|
||
missing.extend(sorted(pending)); pending.clear()
|
||
if m["role"] == "assistant":
|
||
for tc in m.get("tool_calls") or []:
|
||
tid = tc.get("id")
|
||
if not tid or tid in seen:
|
||
duplicates.append(tid)
|
||
if tid:
|
||
seen.add(tid); pending.add(tid)
|
||
elif m["role"] == "tool":
|
||
tid = m.get("tool_call_id")
|
||
if tid not in pending:
|
||
orphan.append(tid)
|
||
else:
|
||
pending.remove(tid)
|
||
missing.extend(sorted(pending))
|
||
stats["tool_pairing"] = {"valid": not (missing or orphan or duplicates),
|
||
"missing_results": len(missing), "orphan_results": len(orphan),
|
||
"duplicate_or_missing_call_ids": len(duplicates)}
|
||
stats["trailing_user"] = bool(msgs and msgs[-1]["role"] == "user")
|
||
# R8
|
||
n_assist = sum(m["role"] == "assistant" for m in msgs)
|
||
stats.update({"out": len(msgs), "assistant_turns": n_assist, "drop": n_assist == 0 or len(msgs) < 2})
|
||
return msgs, stats
|
||
|
||
|
||
# ---------------------------------------------------------------- 主流程
|
||
|
||
def sft_eligibility(verdict: dict, reward: float) -> tuple[bool, str]:
|
||
"""Outcome eligibility only; the caller also checks transcript integrity.
|
||
|
||
Execution TIMEOUT is separate from measured terminal PASS/FAIL. A completed,
|
||
measured success is not rejected solely for exhausting its budget.
|
||
Legacy TIMEOUT verdicts stay evidence until reverified, never assumed PASS.
|
||
"""
|
||
if verdict.get("mode") == "repair":
|
||
return False, "historical-repair-context"
|
||
t = verdict.get("terminal")
|
||
if t == "PASS":
|
||
return True, "PASS"
|
||
if t == "BLOCKED":
|
||
return False, "BLOCKED-unproven"
|
||
if t == "TIMEOUT":
|
||
return False, "legacy-TIMEOUT-needs-reverification"
|
||
return False, f"{t}-not-success-demonstration"
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def readonly_db():
|
||
state = Path(os.environ.get('BENCH_STATE_DIR', BENCH / 'state'))
|
||
con = sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True, timeout=60)
|
||
con.row_factory = sqlite3.Row
|
||
try:
|
||
yield con
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
DB_TERMINAL_STATES = ('PASS', 'FAIL', 'BLOCKED', 'UNVERIFIED', 'TIMEOUT', 'DONE', 'ERROR', 'CANCELLED', 'DEADLINE_EXCEEDED', 'BUDGET_EXHAUSTED', 'INFRA_ERROR', 'LOST')
|
||
|
||
|
||
def _database_terminal(rd, source_root=None):
|
||
root = Path(source_root or BENCH)
|
||
state = root / 'state' if source_root is not None else Path(os.environ.get('BENCH_STATE_DIR', root / 'state'))
|
||
try:
|
||
with contextlib.closing(sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True)) as con:
|
||
row = con.execute('SELECT state FROM rollouts WHERE run_id=? AND job_id=? AND kk=?',
|
||
(rd.parents[2].name, rd.parents[1].name, int(rd.name))).fetchone()
|
||
return row is not None and row[0] in DB_TERMINAL_STATES
|
||
except (OSError, sqlite3.Error):
|
||
return False
|
||
|
||
|
||
def scan(run_ids: list[str]) -> list[Path]:
|
||
# DB index only: never enumerate the JuiceFS run tree.
|
||
with readonly_db() as con:
|
||
rows = [row for rid in run_ids for row in con.execute(
|
||
"SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state IN (" +
|
||
','.join('?' for _ in DB_TERMINAL_STATES) + ')', (rid, *DB_TERMINAL_STATES))]
|
||
return sorted(BENCH / 'runs' / rid / jid / 'rollouts' / f'{kk:02d}' for rid, jid, kk in rows)
|
||
|
||
|
||
def annotation_reasons(rows, rid, jid, kk):
|
||
reasons = set()
|
||
for arun, ajob, note in rows:
|
||
if arun != rid or ajob not in (None, jid):
|
||
continue
|
||
match = re.search(r'rollout#(\d+)\b', note or '')
|
||
if match and int(match[1]) != kk:
|
||
continue
|
||
for needle, reason in [('contaminated:', 'contaminated'), ('stalled_killed', 'stalled_killed')]:
|
||
if needle in (note or ''):
|
||
reasons.add(reason)
|
||
return reasons
|
||
|
||
|
||
def next_dataset_id() -> str:
|
||
d = time.strftime("%Y%m%d")
|
||
existing = sorted((BENCH / "datasets").glob(f"ds{d}-*"))
|
||
return f"ds{d}-{len(existing) + 1:02d}"
|
||
|
||
|
||
def _read_json(path: Path, default=None):
|
||
try:
|
||
return json.loads(path.read_text())
|
||
except (OSError, ValueError):
|
||
return {} if default is None else default
|
||
|
||
|
||
def _gate_sources(rd: Path) -> dict:
|
||
"""Fingerprint the verifier's bounded inputs; never copy model artifacts."""
|
||
from harness.verify import rollout_hbm_path
|
||
files = {"job.json": rd.parent.parent / "job.json", "claim.json": rd / "claim.json",
|
||
"rollout.json": rd / "rollout.json",
|
||
"terminal_result.json": rd / "terminal_result.json"}
|
||
for pattern in ("[0-9][0-9]-*/meta.json", "[0-9][0-9]-*/a*/meta.json", "06-hbm/a*/stdout.log", "_queue/*/request.json"):
|
||
for path in (rd / "stages").glob(pattern):
|
||
files[str(path.relative_to(rd))] = path
|
||
if path.name == "request.json":
|
||
recipe = _read_json(path).get("recipe")
|
||
if recipe:
|
||
files["recipe:" + hashlib.sha256(str(recipe).encode()).hexdigest()] = Path(recipe)
|
||
job = _read_json(files["job.json"])
|
||
files["hbm"] = rollout_hbm_path(rd, job.get("job_id", rd.parent.parent.name))
|
||
stage_predictions = {}
|
||
from harness.verify import protected_predictions_path
|
||
for source in tuple(files.values()):
|
||
if source.name != "meta.json":
|
||
continue
|
||
meta = _read_json(source)
|
||
ref = ((meta.get("measurements") or {}).get("G3") or {}).get("predictions")
|
||
if ref:
|
||
key = "predictions:" + str(source.relative_to(rd))
|
||
try:
|
||
files[key] = protected_predictions_path(rd / "stages", ref)
|
||
except (OSError, ValueError, TypeError):
|
||
stage_predictions[key] = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
|
||
terminal = _read_json(rd / "terminal_result.json")
|
||
terminal = terminal if isinstance(terminal, dict) else {}
|
||
invalid_prediction = None
|
||
if terminal.get("predictions"):
|
||
from harness.verify import terminal_predictions_path
|
||
try:
|
||
files["terminal:predictions"] = terminal_predictions_path(rd, terminal)
|
||
except (OSError, ValueError, TypeError):
|
||
invalid_prediction = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
|
||
manifest = dict(stage_predictions)
|
||
if invalid_prediction:
|
||
manifest["terminal:predictions"] = invalid_prediction
|
||
for name, path in sorted(files.items()):
|
||
try:
|
||
digest, size = hashlib.sha256(), 0
|
||
with path.open("rb") as stream:
|
||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||
digest.update(chunk); size += len(chunk)
|
||
manifest[name] = {"sha256": digest.hexdigest(), "bytes": size}
|
||
except FileNotFoundError:
|
||
manifest[name] = {"missing": True}
|
||
return manifest
|
||
|
||
|
||
def _hard_gate_check(job, verdict, manifest, current, measurements=None):
|
||
from harness.verify import eval_set
|
||
required = verdict.get("required") or job.get("gates", {}).get("required", [])
|
||
failures = [gate for gate in required if (verdict.get("gates", {}).get(gate) or {}).get("verdict") not in ("pass", "pass-screen")]
|
||
g3 = (verdict.get("gates") or {}).get("G3") or {}
|
||
expected_n = ((job.get("data") or {}).get("t1") or {}).get("n")
|
||
if "G3" in required and (g3.get("source") != "board" or expected_n is not None and g3.get("n") != expected_n):
|
||
failures.append("G3_SOURCE_OR_SAMPLE_COUNT")
|
||
if "G3" in required:
|
||
expected_set = eval_set(job)
|
||
measured_g3 = (measurements or {}).get("G3") or {}
|
||
measured_set = measured_g3.get("eval_set")
|
||
fields = ("dataset", "n", "content_sha", "manifest_sha", "eval_set_id", "evaluator_version")
|
||
def valid_identity(value):
|
||
return (isinstance(value, dict) and type(value.get('n')) is int and value['n'] > 0
|
||
and all(isinstance(value.get(k), str) and value[k].strip() for k in ('dataset', 'evaluator_version'))
|
||
and all(isinstance(value.get(k), str) and re.fullmatch('[0-9a-fA-F]{64}', value[k])
|
||
for k in ('content_sha', 'manifest_sha')))
|
||
if (not expected_set or not isinstance(measured_set, dict)
|
||
or not valid_identity(expected_set) or not valid_identity(measured_set)
|
||
or any((key != "eval_set_id" and expected_set.get(key) is None)
|
||
or measured_set.get(key) != expected_set[key] for key in fields)):
|
||
failures.append("G3_EVAL_SET_IDENTITY")
|
||
if expected_set and any(key in measured_g3 and measured_g3[key] != expected_set.get(key) for key in fields):
|
||
failures.append("G3_EVAL_SET_IDENTITY_CONFLICT")
|
||
# Current task and training consume the same versioned scorer result.
|
||
from harness.gates import SCORING_RULE
|
||
if verdict.get("scoring_rule") != SCORING_RULE:
|
||
failures.append("SCORING_RULE_MISMATCH")
|
||
if not required:
|
||
failures.append("REQUIRED_GATES_MISSING")
|
||
if manifest != current:
|
||
failures.append("GATE_SOURCE_CHANGED")
|
||
if verdict.get("terminal") != "PASS":
|
||
failures.append("OUTCOME")
|
||
return {"pass": not failures, "required": required, "failures": failures}
|
||
|
||
|
||
def normal_final_response(messages):
|
||
"""An agent_end marker cannot turn a tool tail or failed generation into a final."""
|
||
last = messages[-1] if messages else {}
|
||
content = last.get("content")
|
||
text = content if isinstance(content, str) else "".join(
|
||
item.get("text", "") for item in content or [] if isinstance(item, dict) and item.get("type") == "text")
|
||
ok = (last.get("role") == "assistant" and bool(text.strip()) and not preserved._calls(last)
|
||
and last.get("stopReason") not in ("error", "aborted", "length", "toolUse", "tool_calls"))
|
||
return {"pass": ok, "reason": "PASS" if ok else "MISSING_NORMAL_FINAL_RESPONSE"}
|
||
|
||
|
||
def validate_exported_record(source_root: Path, record: dict, lineage=None, excluded=None) -> dict:
|
||
"""Independent publication check against unchanged source and hard gates.
|
||
|
||
Optional sidecar arguments contain only this attempt's rows. Omitting them
|
||
checks the record payload; publication must additionally supply sidecars.
|
||
"""
|
||
from harness.verify import verify_rollout, collect_measurements
|
||
root = Path(source_root).resolve()
|
||
rd = root / "runs" / record["run_id"] / record["job_id"] / "rollouts" / f"{int(record['rollout']):02d}"
|
||
if not rd.resolve().is_relative_to(root):
|
||
raise ValueError("source path leaves bench root")
|
||
frozen = preserved.freeze(rd)
|
||
expected = preserved.reconstruct(frozen)
|
||
trajectory = record.get("trajectory", {})
|
||
actual = {**expected, "messages": record.get("messages"), "tools": record.get("tools"),
|
||
"source_ids": trajectory.get("source_ids"), "source_manifest": trajectory.get("source_manifest"),
|
||
"redaction": record.get("redact_stats"), "context": trajectory.get("context")}
|
||
for key, rows in (("lineage", lineage), ("excluded", excluded)):
|
||
if rows is not None:
|
||
fields = ("message_id", "observations", "event_type") + (("reason",) if key == "excluded" else ())
|
||
actual[key] = [{field: row[field] for field in fields} for row in rows]
|
||
checks = preserved.validate(frozen, actual)
|
||
checks["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(actual["messages"])
|
||
gate_before = _gate_sources(rd)
|
||
verdict = verify_rollout(rd, write=False)
|
||
measured = collect_measurements(rd)[0]
|
||
job = _read_json(rd.parent.parent / "job.json")
|
||
gate_check = _hard_gate_check(job, verdict, trajectory.get("hard_gate_manifest"), gate_before, measured)
|
||
from harness.verify import eval_set
|
||
if record.get("eval_set") != (measured.get("G3") or {}).get("eval_set") or record.get("required_eval_set") != eval_set(job):
|
||
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["EXPORTED_EVAL_SET_CHANGED"]}
|
||
if gate_before != _gate_sources(rd):
|
||
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["GATE_SOURCE_CHANGED"]}
|
||
checks["checks"]["HARD_GATES"] = gate_check
|
||
from export.content_review import check as check_content_review
|
||
review = check_content_review(rd, verdict, frozen, gate_before)
|
||
if record.get("content_review") != review:
|
||
review = {**review, "pass": False, "reason": "EXPORTED_REVIEW_CHANGED"}
|
||
checks["checks"]["CONTENT_REVIEW"] = review
|
||
checks['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd, root)}
|
||
try:
|
||
# Publication uses the explicitly supplied source, never the exporting
|
||
# process's current project or BENCH_STATE_DIR. Reuse collection's rule.
|
||
uri = (root / 'state/jobs.sqlite').as_uri() + '?mode=ro'
|
||
with contextlib.closing(sqlite3.connect(uri, uri=True)) as con:
|
||
con.execute('PRAGMA query_only=ON')
|
||
rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?',
|
||
(record['run_id'],)).fetchall()
|
||
reasons = sorted(annotation_reasons(rows, record['run_id'], record['job_id'], int(record['rollout'])))
|
||
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': not reasons, 'reasons': reasons}
|
||
except (OSError, sqlite3.Error, TypeError, ValueError):
|
||
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': False, 'reasons': ['DB_ANNOTATIONS_UNAVAILABLE']}
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
checks["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
checks["pass"] = all(check["pass"] for check in checks["checks"].values())
|
||
return checks
|
||
|
||
|
||
def _inc(counts: dict, key, n=1):
|
||
counts[str(key)] = counts.get(str(key), 0) + n
|
||
|
||
|
||
def _log_tail(path: Path, limit=4096):
|
||
try:
|
||
with path.open("rb") as stream:
|
||
stream.seek(0, os.SEEK_END)
|
||
size = stream.tell()
|
||
stream.seek(max(0, size - limit))
|
||
return {"text": stream.read(limit).decode("utf-8", errors="replace"), "truncated": size > limit}
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def _safe_record(record: dict) -> tuple[dict, dict]:
|
||
# Stage argv can contain secrets split across list elements. Preserve the
|
||
# command structure while masking explicit secret keys/flags before regexes.
|
||
secret_keys = {"password", "passwd", "api_key", "apikey", "authorization", "secret", "access_token"}
|
||
def mask(value):
|
||
if isinstance(value, dict):
|
||
return {k: "<REDACTED>" if k.lower() in secret_keys else mask(v) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
out = [mask(v) for v in value]
|
||
for i, v in enumerate(out[:-1]):
|
||
if isinstance(v, str) and (v.lstrip("-").lower().replace("-", "_") in secret_keys
|
||
or (v == "-p" and "sshpass" in out)):
|
||
out[i + 1] = "<REDACTED>"
|
||
return out
|
||
return value
|
||
return redact_preserving(mask(record))
|
||
|
||
|
||
def pipeline_records(run_ids: list[str]):
|
||
"""One actual execution record per finished oracle job, including failures.
|
||
|
||
DB-indexed only. No recursive traversal of model/data trees and no simulated
|
||
LLM dialogue. Missing evidence is recorded rather than hiding a finished job.
|
||
"""
|
||
from harness.verify import completion_summary, eval_set
|
||
with readonly_db() as con:
|
||
jobs = [dict(row) for rid in run_ids for row in con.execute(
|
||
"SELECT * FROM jobs WHERE run_id=? AND mode='oracle' AND state IN ('PASS','FAIL','UNVERIFIED')", (rid,))]
|
||
stages = [dict(row) for rid in run_ids for row in con.execute(
|
||
"SELECT * FROM stages WHERE run_id=? AND rollout=-1 ORDER BY job_id,stage,attempt", (rid,))]
|
||
by_job = {}
|
||
for stage in stages:
|
||
by_job.setdefault((stage["run_id"], stage["job_id"]), []).append(stage)
|
||
for row in sorted(jobs, key=lambda r: (r["run_id"], r["job_id"])):
|
||
rid, jid = row["run_id"], row["job_id"]
|
||
jd = BENCH / "runs" / rid / jid
|
||
job = _read_json(jd / "job.json")
|
||
result = _read_json(jd / "oracle" / "result.json")
|
||
result_source = "oracle/result.json"
|
||
if not result and row.get("result"):
|
||
try:
|
||
result = json.loads(row["result"])
|
||
result_source = "state/jobs.sqlite:jobs.result"
|
||
except ValueError:
|
||
pass
|
||
actions = []
|
||
for stage in by_job.get((rid, jid), []):
|
||
sd = jd / "oracle" / "stages" / stage["stage"] / f"a{stage['attempt']:02d}"
|
||
meta = _read_json(sd / "meta.json")
|
||
actions.append({"stage": stage["stage"], "attempt": stage["attempt"],
|
||
"database": stage, "meta": meta, "meta_available": bool(meta),
|
||
"command": meta.get("cmd"),
|
||
"source": str(sd.relative_to(BENCH)),
|
||
"logs": {name: str((sd / name).relative_to(BENCH))
|
||
for name in ("stdout.log", "stderr.log") if (sd / name).exists()}})
|
||
if stage["state"] not in ("OK", "SKIPPED"):
|
||
actions[-1]["diagnostic_tail"] = {name: tail for name in ("stdout.log", "stderr.log")
|
||
if (tail := _log_tail(sd / name)) is not None}
|
||
cp = jd / "oracle" / "stages" / "04-calibrate" / "compile.yaml"
|
||
record = {
|
||
"schema": "bench.pipeline-training/1", "source": "deterministic_pipeline",
|
||
"id": hashlib.sha1(f"{rid}/{jid}/oracle".encode()).hexdigest()[:16],
|
||
"run_id": rid, "job_id": jid, "model": job.get("model", row.get("model")),
|
||
"eval_set": eval_set(job), "continuation": job.get("continuation"),
|
||
"task": job.get("task", row.get("task")), "march": job.get("march", row.get("march")),
|
||
"backend": job.get("backend", row.get("backend")), "terminal": result.get("terminal", row["state"]),
|
||
"completion": completion_summary(result.get("gates") or {}, result.get("stages") or {}),
|
||
"started_at": row.get("started_at"), "finished_at": row.get("finished_at"),
|
||
"job": {k: job[k] for k in ("model", "task", "march", "backend", "qcfg", "model_spec", "data", "gates") if k in job},
|
||
"compile_config": cp.read_text(errors="replace") if cp.exists() else None,
|
||
"stages": actions, "result": result,
|
||
"evidence": {"job_available": bool(job), "result_available": bool(result),
|
||
"result_source": result_source if result else None,
|
||
"job_path": str((jd / "job.json").relative_to(BENCH))},
|
||
"training_uses": ["workflow_analysis", "outcome_learning_candidate"],
|
||
"sft_eligible": False, "sft_reason": "deterministic-pipeline-not-agent-dialogue",
|
||
}
|
||
record, redaction = _safe_record(record)
|
||
record["redact_stats"] = redaction
|
||
yield record
|
||
|
||
|
||
def _write_dataset(a, work: Path, ds_id: str) -> dict:
|
||
with readonly_db() as con:
|
||
annotations = con.execute("SELECT run_id, job_id, note FROM annotations").fetchall()
|
||
coverage = {rid: {
|
||
"oracle": dict(con.execute("SELECT state,count(*) FROM jobs WHERE run_id=? AND mode='oracle' GROUP BY state", (rid,))),
|
||
"agent": dict(con.execute("SELECT state,count(*) FROM rollouts WHERE run_id=? GROUP BY state", (rid,))),
|
||
} for rid in a.runs}
|
||
stats = {"scanned": 0, "written": 0, "agent_written": 0, "teacher_written": 0, "student_written": 0,
|
||
"pipeline_written": 0, "excluded": 0, "dropped": {}, "review": 0, "partial": 0,
|
||
"by_terminal": {}, "by_execution_terminal": {}, "pipeline_by_terminal": {}, "sft_eligible": 0,
|
||
"by_march": {}, "sft_eligible_by_march": {}, "reverified": 0}
|
||
with contextlib.ExitStack() as stack:
|
||
names = ("teacher", "student", "pipeline", "exclusions", "sft", "retained", "lineage",
|
||
"operations", "excluded_observations", "validation", "annotations", "source-manifest", "disposition")
|
||
files = {name: stack.enter_context((work / f"{name}.jsonl").open("w")) for name in names}
|
||
source_seen = set()
|
||
def write(name, rec):
|
||
files[name].write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
|
||
if name == "source-manifest":
|
||
source_seen.add((rec["run_id"], rec["job_id"], rec["rollout"]))
|
||
def exclude(rd, reasons, review_binding=None, review_receipt=None):
|
||
stats["excluded"] += 1
|
||
for reason in reasons:
|
||
_inc(stats["dropped"], reason)
|
||
write("exclusions", {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name,
|
||
"rollout": int(rd.name), "source": str(rd.relative_to(BENCH)),
|
||
"reasons": sorted(reasons)})
|
||
disposition = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name),
|
||
"source": str(rd.relative_to(BENCH)), "reasons": sorted(reasons),
|
||
"disposition": "no-usable" if "no_usable_messages" in reasons else "retained"}
|
||
if review_binding:
|
||
disposition["review_binding_sha256"] = review_binding
|
||
disposition["review_receipt_sha256"] = review_receipt
|
||
write("disposition", disposition)
|
||
write("retained", disposition)
|
||
if a.purpose == "all":
|
||
for record in pipeline_records(a.runs):
|
||
write("pipeline", record)
|
||
stats["pipeline_written"] += 1
|
||
_inc(stats["pipeline_by_terminal"], record["terminal"])
|
||
for rd in scan(a.runs):
|
||
_write_agent(rd, a, stats, annotations, write, exclude)
|
||
ident = (rd.parents[2].name, rd.parents[1].name, int(rd.name))
|
||
if ident not in source_seen:
|
||
source = preserved.freeze(rd)
|
||
try:
|
||
gate_sources = _gate_sources(rd)
|
||
except (OSError, ValueError, TypeError, KeyError) as error:
|
||
# Rejected evidence must not abort the whole run's export.
|
||
gate_sources = {}
|
||
source.errors.append(f"HARD_GATES_UNAVAILABLE:{type(error).__name__}")
|
||
write("source-manifest", {"run_id": ident[0], "job_id": ident[1], "rollout": ident[2],
|
||
"source": str(rd.relative_to(BENCH)), "files": source.manifest,
|
||
"errors": source.errors, "hard_gates": gate_sources})
|
||
file_info = {}
|
||
for name in names:
|
||
path = work / f"{name}.jsonl"
|
||
with path.open("rb") as stream:
|
||
digest, lines = hashlib.sha256(), 0
|
||
for line in stream:
|
||
digest.update(line); lines += 1
|
||
file_info[path.name] = {"sha256": digest.hexdigest(), "records": lines, "bytes": path.stat().st_size}
|
||
manifest = {"schema": "bench.dataset/2", "dataset_id": ds_id, "runs": a.runs,
|
||
"reward_recipe": RECIPE_VERSION, "created_at": time.time(), "coverage": coverage,
|
||
"filters": {"min_agent_version": a.min_agent_version,
|
||
"min_agent_version_runner": a.min_agent_version_runner,
|
||
"max_turns": a.max_turns, "max_tokens": a.max_tokens,
|
||
"agent_version": a.agent_version, "agent_version_mode": "exact" if a.agent_version else "minimum",
|
||
"annotations_read": True, "dropped_counts_overlap": True, "policy": a.policy,
|
||
"include_timeout": True, "purpose": a.purpose, "min_reward": a.min_reward},
|
||
"stats": stats, "files": file_info,
|
||
"notes": ["pipeline.jsonl contains historical deterministic evidence only for purpose=all; SFT/RL export reads Agent evidence only.",
|
||
"Verdict v4 and reward v0.7 use only this rollout evidence; Oracle references are history only.",
|
||
"SFT/reward candidates are labels; no train split, framework conversion, or online RL readiness is implied.",
|
||
"Writers hold .export.lock; manifest is published last. Readers can lock or verify file hashes for a coherent refresh."]}
|
||
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||
return manifest
|
||
|
||
|
||
def _write_agent(rd, a, stats, annotations, write, exclude):
|
||
from harness.verify import eval_set, verify_rollout, collect_measurements, VERDICT_SCHEMA, VERIFICATION_RULE
|
||
stats["scanned"] += 1
|
||
original_verdict = _read_json(rd / "verdict.json")
|
||
verdict = original_verdict
|
||
rj = _read_json(rd / "rollout.json")
|
||
if not verdict:
|
||
exclude(rd, {"missing_verdict"}); return
|
||
policy = verdict.get("policy") or rj.get("policy") or "teacher"
|
||
if a.policy and policy != a.policy:
|
||
exclude(rd, {"policy_filter"}); return
|
||
if policy not in ("teacher", "student"):
|
||
exclude(rd, {"unknown_policy"}); return
|
||
reasons = annotation_reasons(annotations, rd.parents[2].name, rd.parents[1].name, int(rd.name))
|
||
runner = rj.get("runner", "pi")
|
||
version = str(rj.get("agent_config_version") or 1)
|
||
if a.agent_version is not None:
|
||
if version != str(a.agent_version):
|
||
reasons.add("old_version")
|
||
elif a.min_agent_version_runner in ("*", runner):
|
||
try:
|
||
if int(version) < a.min_agent_version:
|
||
reasons.add("old_version")
|
||
except ValueError:
|
||
reasons.add("invalid_version")
|
||
if reasons:
|
||
exclude(rd, reasons); return
|
||
original_sha = hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest()
|
||
reverified = verdict.get("schema") != VERDICT_SCHEMA
|
||
try:
|
||
gate_manifest = _gate_sources(rd)
|
||
verdict = verify_rollout(rd, write=False)
|
||
stats["reverified"] += int(reverified)
|
||
except Exception as error:
|
||
exclude(rd, {f"reverification_failed:{type(error).__name__}"}); return
|
||
verification = {"schema": verdict.get("schema"),
|
||
"rule": verdict.get("verification_rule", VERIFICATION_RULE),
|
||
"method": "readonly-reverification"}
|
||
strict = None
|
||
try:
|
||
if runner == "pi":
|
||
frozen = preserved.freeze(rd)
|
||
strict = preserved.reconstruct(frozen)
|
||
validation = preserved.validate(frozen, strict)
|
||
validation["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(strict["messages"])
|
||
validation["checks"]["HARD_GATES"] = _hard_gate_check(_read_json(rd.parent.parent / "job.json"), verdict,
|
||
gate_manifest, _gate_sources(rd), collect_measurements(rd)[0])
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
|
||
msgs, tools = strict["messages"], strict["tools"]
|
||
legacy_messages, _ = clean(msgs)
|
||
strict["annotations"].extend(preserved.legacy_truncation_audit(msgs, strict["source_ids"], legacy_messages))
|
||
raw = msgs
|
||
trajectory = {k: v for k, v in strict.items() if k not in ("messages", "tools", "redaction", "lineage", "operations", "excluded")}
|
||
trajectory.update(source="pi-segment-events" if strict["agent_end"] else "message_end",
|
||
partial=any(not check["pass"] for name, check in validation["checks"].items() if name != "HARD_GATES"),
|
||
validation=validation, hard_gate_manifest=gate_manifest)
|
||
ident = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name)}
|
||
for name, values in (("lineage", strict["lineage"]), ("operations", strict["operations"]),
|
||
("excluded_observations", strict["excluded"]),
|
||
("annotations", strict["annotations"] if validation["checks"]["PRIVACY"]["pass"] else [])):
|
||
for value in values:
|
||
write(name, {**ident, **value})
|
||
write("source-manifest", {**ident, "source": str(rd.relative_to(BENCH)), "files": frozen.manifest,
|
||
"hard_gates": gate_manifest})
|
||
pairing = validation["checks"]["CALL_TOOL_CLOSURE"]
|
||
cst = {"in": len(msgs), "out": len(msgs), "assistant_turns": sum(m.get("role") == "assistant" for m in msgs),
|
||
"drop": not validation["checks"]["USABLE_ASSISTANT"]["pass"], "r7_truncated": 0,
|
||
"tool_pairing": {"valid": pairing["pass"], "missing_results": pairing["missing"],
|
||
"orphan_results": pairing["orphan"], "duplicate_or_missing_call_ids": pairing["call_id_conflict"]}}
|
||
sys_text = frozen.files.get("agent/system.md", b"").decode("utf-8", errors="strict")
|
||
sys_src = strict["context"]["source"] if sys_text else "missing"
|
||
tools_src, rst = ("snapshot" if tools else "missing"), strict["redaction"]
|
||
else:
|
||
raw, trajectory = load_trajectory(rd)
|
||
msgs, cst = clean(raw)
|
||
sys_text, sys_src = system_prompt(rd)
|
||
if sys_text:
|
||
msgs.insert(0, {"role": "system", "content": sys_text})
|
||
tools, tools_src = tool_defs(rd)
|
||
msgs, rst = redact_messages(msgs)
|
||
trajectory["strict_sft_reason"] = "UNSUPPORTED_SOURCE"
|
||
except (OSError, ValueError, KeyError, TypeError) as exc:
|
||
exclude(rd, {f"invalid_trajectory:{type(exc).__name__}"}); return
|
||
if not raw:
|
||
exclude(rd, {"no_trajectory"}); return
|
||
cst["system_source"] = sys_src
|
||
if sys_src == "runner-raw-system":
|
||
trajectory["system_segments"] = _read_json(rd / "agent/raw-system.json", default=[])
|
||
if cst["drop"]:
|
||
if strict is not None:
|
||
write("validation", {**ident, **validation})
|
||
exclude(rd, {"no_usable_messages"}); return
|
||
jd = rd.parent.parent
|
||
job = json.loads((jd / "job.json").read_text())
|
||
mode = verdict.get("mode") or rj.get("mode")
|
||
verdict = {**verdict, "mode": mode}
|
||
rw = compute_reward(verdict, rj.get("max_turns", a.max_turns), rj.get("max_tokens", a.max_tokens))
|
||
sft_ok, sft_reason = sft_eligibility(verdict, rw["reward"])
|
||
required = verdict.get("required") or job.get("gates", {}).get("required", [])
|
||
if sft_ok and (not required or any((verdict.get("gates", {}).get(g) or {}).get("verdict") not in ("pass", "pass-screen") for g in required)):
|
||
sft_ok, sft_reason = False, "required-hard-gate-not-passed"
|
||
if sft_ok and strict is None:
|
||
sft_ok, sft_reason = False, "unsupported-strict-sft-source"
|
||
if sft_ok and strict is not None and not validation["checks"]["HARD_GATES"]["pass"]:
|
||
sft_ok, sft_reason = False, "hard-gate-validation-failed"
|
||
trajectory["partial"] = trajectory["partial"] or not cst["tool_pairing"]["valid"] or cst.get("trailing_user", False)
|
||
if sft_ok and trajectory["partial"]:
|
||
sft_ok, sft_reason = False, "partial-transcript"
|
||
if sft_ok and (not tools or not sys_text):
|
||
sft_ok, sft_reason = False, "missing-tools-or-system"
|
||
known_tools = {t.get("name") or (t.get("function") or {}).get("name") for t in tools if isinstance(t, dict)}
|
||
used_tools = {tc.get("name") for msg in msgs for tc in preserved._calls(msg)}
|
||
missing_tools = sorted(name for name in used_tools - known_tools if name)
|
||
trajectory["missing_tool_definitions"] = missing_tools
|
||
if sft_ok and missing_tools:
|
||
sft_ok, sft_reason = False, "missing-called-tool-definitions"
|
||
if strict is not None:
|
||
validation['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd)}
|
||
if not validation['checks']['DB_SOURCE_TERMINAL']['pass']:
|
||
validation['pass'] = False
|
||
sft_ok, sft_reason = False, 'source-not-terminal'
|
||
if not preserved.source_unchanged(rd, frozen):
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
validation["pass"] = False
|
||
trajectory["partial"] = True
|
||
sft_ok, sft_reason = False, "source-changed"
|
||
if a.min_reward is not None and rw["reward"] < a.min_reward:
|
||
exclude(rd, {"min_reward"}); return
|
||
content_review = {"pass": False, "reason": "NOT_SCRIPT_QUALIFIED"}
|
||
if sft_ok:
|
||
from export.content_review import check as check_content_review
|
||
if getattr(a, "review", False) and a.purpose in ("all", "sft"):
|
||
from export.content_review import review_once
|
||
try:
|
||
review_once(rd)
|
||
except (OSError, ValueError):
|
||
sft_ok, sft_reason = False, "content-review-unavailable"
|
||
content_review = check_content_review(rd, verdict, frozen, gate_manifest)
|
||
if not preserved.source_unchanged(rd, frozen) or _gate_sources(rd) != gate_manifest:
|
||
sft_ok, sft_reason = False, "source-changed"
|
||
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
|
||
if not content_review["pass"]:
|
||
sft_ok, sft_reason = False, "content-review-" + content_review["reason"].lower()
|
||
if strict is not None:
|
||
validation["checks"]["CONTENT_REVIEW"] = content_review
|
||
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
|
||
write("validation", {**ident, **validation})
|
||
if a.purpose == "sft" and not sft_ok:
|
||
exclude(rd, {f"sft:{sft_reason}"}, content_review.get("binding_sha256"),
|
||
content_review.get("receipt_sha256")); return
|
||
reward_candidate = verdict["terminal"] in ("PASS", "FAIL") and mode != "repair" and not rw.get("unverifiable")
|
||
if a.purpose == "rl" and not reward_candidate:
|
||
exclude(rd, {"rl:historical-repair-or-unverified-outcome"}); return
|
||
if a.min_reward is not None and rw["reward"] < a.min_reward:
|
||
exclude(rd, {"min_reward"}); return
|
||
rec = {
|
||
"schema": "bench.agent-training/2", "source": "agent_events",
|
||
"id": hashlib.sha1(f"{verdict['run_id']}/{verdict['job_id']}/{verdict['rollout']}".encode()).hexdigest()[:16],
|
||
"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
|
||
"eval_set": ((collect_measurements(rd)[0].get("G3") or {}).get("eval_set")),
|
||
"required_eval_set": eval_set(job),
|
||
"continuation": {**job["continuation"], "source_rollout": verdict["rollout"]} if job.get("continuation") else None,
|
||
"mode": mode, "policy": policy, "runner": runner, "model": rj.get("model"),
|
||
"agent_config_version": str(rj.get("agent_config_version", "1")),
|
||
"runner_version": rj.get("runner_version"),
|
||
"task": job["task"], "march": job["march"],
|
||
"board_type": (job.get("target") or {}).get("board_type"), "backend": job["backend"],
|
||
"messages": msgs,
|
||
"verdict": {k: verdict[k] for k in ("schema", "verification_rule", "terminal", "measured_terminal", "execution_terminal", "timeout_reason", "completion", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict},
|
||
"gates": {g: (verdict["gates"].get(g) or {}).get("verdict") for g in ("G1", "G2", "G3", "G4")},
|
||
"verification": verification,
|
||
"reward": rw["reward"], "reward_terms": rw["terms"], "reward_recipe": rw["recipe"],
|
||
"reward_unverifiable": rw.get("unverifiable"), # v0.4:board_offline = 板卡故障导致没法测,不是 agent 的结果
|
||
"sft_eligible": sft_ok, "sft_reason": sft_reason,
|
||
"training_uses": {"analysis": True, "sft_candidate": sft_ok, "reward_learning_candidate": reward_candidate,
|
||
"review_required": not content_review["pass"]},
|
||
"content_review": content_review,
|
||
"trajectory": trajectory,
|
||
"execution": {"terminal": verdict.get("execution_terminal") or rj.get("state"),
|
||
"timeout_reason": verdict.get("timeout_reason") or rj.get("timeout_reason"),
|
||
"started_at": rj.get("started_at"), "finished_at": rj.get("finished_at"),
|
||
"max_turns": rj.get("max_turns"), "max_tokens": rj.get("max_tokens")},
|
||
"provenance": {"events": str((rd / "agent" / "events.jsonl").relative_to(BENCH)),
|
||
"verdict": str((rd / "verdict.json").relative_to(BENCH)),
|
||
"original_verdict": {"sha256": original_sha, "schema": original_verdict.get("schema"),
|
||
"terminal": original_verdict.get("terminal"),
|
||
"review": original_verdict.get("review")}},
|
||
"cost": verdict.get("cost"),
|
||
"tools": tools,
|
||
"tools_source": tools_src,
|
||
"system_source": sys_src,
|
||
"system_sha1": hashlib.sha1(sys_text.encode()).hexdigest()[:12] if sys_text else None,
|
||
"clean_stats": cst, "redact_stats": rst,
|
||
}
|
||
# Preserve the already validated context exactly; redact metadata separately
|
||
# so a second traversal cannot silently rewrite call/source identities.
|
||
context = {key: rec.pop(key) for key in ("messages", "tools", "trajectory", "redact_stats")}
|
||
rec, _ = _safe_record(rec)
|
||
rec.update(context)
|
||
if strict is not None and not validation["checks"]["PRIVACY"]["pass"]:
|
||
# Identity cannot be rewritten safely. Keep original bytes only in the
|
||
# controlled source; public evidence contains a hash/reference receipt.
|
||
rec = {key: rec[key] for key in ("schema", "source", "id", "run_id", "job_id", "rollout", "sft_eligible", "sft_reason")}
|
||
rec.update(source_reference=str(rd.relative_to(BENCH)), reason_code="PRIVATE_SOURCE_REFERENCE_ONLY",
|
||
trajectory={"source_manifest": frozen.manifest, "validation": validation})
|
||
write(policy, rec)
|
||
write("sft" if sft_ok else "retained", rec)
|
||
write("disposition", {"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
|
||
"source": str(rd.relative_to(BENCH)), "disposition": "accepted" if sft_ok else "retained",
|
||
"review_binding_sha256": content_review.get("binding_sha256"),
|
||
"review_receipt_sha256": content_review.get("receipt_sha256"),
|
||
"reasons": [] if sft_ok else [sft_reason]})
|
||
stats["written"] += 1
|
||
stats["agent_written"] += 1
|
||
stats[f"{policy}_written"] += 1
|
||
stats["partial"] += int(trajectory["partial"])
|
||
_inc(stats["by_terminal"], verdict["terminal"])
|
||
_inc(stats["by_execution_terminal"], verdict.get("execution_terminal") or rj.get("state"))
|
||
_inc(stats["by_march"], job["march"])
|
||
stats["sft_eligible"] += int(sft_ok)
|
||
if sft_ok:
|
||
_inc(stats["sft_eligible_by_march"], job["march"])
|
||
|
||
def materialize(run_ids: list[str], dataset_id: str, *, refresh=False, purpose=_UNSET, policy=_UNSET,
|
||
min_agent_version=_UNSET, agent_version=_UNSET, min_reward=_UNSET,
|
||
max_turns=_UNSET, max_tokens=_UNSET, review=False) -> dict:
|
||
"""Refresh one stable output directory; locks writers, replaces files atomically.
|
||
|
||
The manifest commits last and supplies hashes/counts. Concurrent consumers
|
||
should share .export.lock or check those hashes; the directory is not swapped.
|
||
Source rollout/run artifacts are never modified.
|
||
"""
|
||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", dataset_id):
|
||
raise ValueError("dataset_id must be a simple directory name")
|
||
if review and not (os.environ.get("BENCH_REVIEW_URL") and os.environ.get("BENCH_REVIEW_MODEL")):
|
||
raise ValueError("--review requires BENCH_REVIEW_URL and BENCH_REVIEW_MODEL")
|
||
run_ids = list(dict.fromkeys(run_ids))
|
||
out_dir = BENCH / "datasets" / dataset_id
|
||
if out_dir.is_symlink():
|
||
raise ValueError("refusing to write a symlink dataset")
|
||
try:
|
||
out_dir.mkdir(parents=True, exist_ok=False)
|
||
except FileExistsError:
|
||
if not refresh:
|
||
raise FileExistsError(f"dataset exists: {out_dir}; use --refresh explicitly")
|
||
if (out_dir / "VOID").exists():
|
||
raise ValueError(f"refusing to refresh VOID dataset: {dataset_id}")
|
||
requested = {"purpose": purpose, "policy": policy, "agent_version": agent_version,
|
||
"min_agent_version": min_agent_version, "min_reward": min_reward,
|
||
"max_turns": max_turns, "max_tokens": max_tokens}
|
||
with (out_dir / ".export.lock").open("a") as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
prior = _read_json(out_dir / "manifest.json")
|
||
if prior and set(prior.get("runs", [])) != set(run_ids):
|
||
raise ValueError("refresh must keep the same run set; choose a new dataset_id")
|
||
defaults = {"purpose": "all", "policy": None, "agent_version": None,
|
||
"min_agent_version": 0, "min_reward": None,
|
||
"max_turns": 40, "max_tokens": 200000}
|
||
previous = prior.get("filters", {})
|
||
if min_agent_version is None or (min_agent_version is _UNSET and "min_agent_version" not in previous):
|
||
version_file = BENCH / "agent/VERSION"
|
||
defaults["min_agent_version"] = int(version_file.read_text()) if version_file.exists() else 1
|
||
resolved = {key: previous.get(key, default) if requested[key] is _UNSET else requested[key]
|
||
for key, default in defaults.items()}
|
||
scope = previous.get("min_agent_version_runner", "*" if prior else "pi")
|
||
if min_agent_version is not _UNSET:
|
||
scope = "pi" if min_agent_version is None else "*"
|
||
if resolved["min_agent_version"] is None:
|
||
resolved["min_agent_version"] = defaults["min_agent_version"] if min_agent_version is None else 0
|
||
if resolved["purpose"] not in ("all", "sft", "rl"):
|
||
raise ValueError("purpose must be all/sft/rl")
|
||
if resolved["policy"] not in (None, "teacher", "student"):
|
||
raise ValueError("policy must be teacher/student or None")
|
||
if not isinstance(resolved["min_agent_version"], int) or resolved["min_agent_version"] < 0:
|
||
raise ValueError("min_agent_version must be a nonnegative integer")
|
||
a = argparse.Namespace(runs=run_ids, min_agent_version_runner=scope, review=review, **resolved)
|
||
with tempfile.TemporaryDirectory(prefix=".refresh-", dir=out_dir) as temp:
|
||
work = Path(temp)
|
||
manifest = _write_dataset(a, work, dataset_id)
|
||
if prior.get("created_at"):
|
||
manifest["created_at"] = prior["created_at"]
|
||
manifest["updated_at"] = time.time()
|
||
manifest["filter_history"] = list(prior.get("filter_history", []))
|
||
if prior and previous != manifest["filters"]:
|
||
explicit = sorted(k for k, v in requested.items() if v is not _UNSET)
|
||
manifest["filter_history"].append({"changed_at": manifest["updated_at"],
|
||
"reason": "explicit_filter_change" if explicit else "legacy_filter_metadata_upgrade",
|
||
"explicit_fields": explicit, "before": previous, "after": manifest["filters"]})
|
||
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||
for name in [*manifest["files"], "manifest.json"]:
|
||
os.replace(work / name, out_dir / name)
|
||
return manifest
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--runs", nargs="+", required=True)
|
||
ap.add_argument("--policy", choices=["teacher", "student"], default=argparse.SUPPRESS)
|
||
ap.add_argument("--include-timeout", action="store_true", help="compatibility flag: timeout evidence is now included by default")
|
||
ap.add_argument("--purpose", choices=["all", "sft", "rl"], default=argparse.SUPPRESS,
|
||
help="all=保留终态轨迹并标注用途;sft=完整诚实成功候选;rl=本次可核验 PASS/FAIL 候选")
|
||
ap.add_argument("--min-reward", type=float, default=argparse.SUPPRESS)
|
||
ap.add_argument("--max-turns", type=int, default=argparse.SUPPRESS)
|
||
ap.add_argument("--max-tokens", type=int, default=argparse.SUPPRESS)
|
||
ap.add_argument("--dataset-id")
|
||
ap.add_argument("--review", action="store_true", help="explicitly review script-qualified candidates once; requires reviewer environment")
|
||
ap.add_argument("--refresh", action="store_true", help="explicitly refresh this dataset from its source runs")
|
||
ap.add_argument("--agent-version", default=argparse.SUPPRESS)
|
||
ap.add_argument("--min-agent-version", type=int, default=argparse.SUPPRESS)
|
||
a = ap.parse_args()
|
||
filters = {key: value for key, value in vars(a).items() if key in (
|
||
"purpose", "policy", "min_agent_version", "agent_version", "min_reward", "max_turns", "max_tokens")}
|
||
manifest = materialize(a.runs, a.dataset_id or next_dataset_id(), refresh=a.refresh, review=a.review, **filters)
|
||
print(json.dumps(manifest, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|