data-gen-bpu/export/to_agentrl.py
ming01.wang 498187569c fix: close rollout finalization and review publication gaps
Reuse execution, stage and review evidence to recover database registration and
SFT publication without replaying completed work. Keep finished futures during
SQLite contention, and hand durable results to startup recovery when draining.
Release proven local leases independently of unresolved board isolation.

Keep UNKNOWN review retries explicit and archived, publish completed receipts
after transient export failures, and prevent a malformed sample from aborting
the entire dataset. Scope routine reconciliation to newly finished attempts.
Add failure-injection coverage and the production acceptance plan.
2026-09-29 16:09:01 +08:00

992 lines
55 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""L4 data materialization: preserve Pi source events, verify, then publish.
python export/to_agentrl.py --runs r20260914-01 [r…] [--policy teacher] [--include-timeout]
Legacy analysis-only cleaning (never used for strict Pi SFT):
R1 双方言归一:OpenAI / Anthropic 形状 → {role, content, tool_calls?, tool_call_id?}
R2 剥离 thinking / reasoning 块
R3 system 只在首条且只有一条(多条合并)
R4 保留完整工具调用/结果;缺失配对的轨迹标记 partial,不进入成功 SFT
R5 去空消息
R6 合并相邻同 role 的纯文本消息
R7 tool 结果截断(头 6k + 尾 2k 字符)
R8 隔离:无 assistant 消息,或 messages < 2(排除理由落 exclusions.jsonl)
轨迹源:agent_end.messages[],中断时恢复已完成的 message_end;不会捏造 assistant 回答。
默认保存全部终态;SFT 候选另行标记。pipeline.jsonl 保存真实确定性流程记录,和 Agent 消息分开。
"""
from __future__ import annotations
import argparse
import contextlib
import fcntl
import hashlib
import json
import math
import sqlite3
import os
import re
import sys
import tempfile
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
BENCH = Path(os.environ.get("BENCH_ROOT", HERE.parent))
sys.path.insert(0, str(HERE.parent))
sys.path.insert(0, str(HERE))
from redact import redact_messages, redact_preserving # noqa: E402
from export import trajectory as preserved # noqa: E402
from reward import compute as compute_reward, RECIPE_VERSION # noqa: E402
TOOL_HEAD, TOOL_TAIL = 6000, 2000
_TOOLS_CACHE = {}
_UNSET = object()
# ---------------------------------------------------------------- 轨迹提取
def system_prompt(rd: Path) -> tuple[str, str]:
"""取这条轨迹的系统提示 → (text, source)。
pi 的 --append-system-prompt 内容**不出现在** agent_end.messages[] 里(实测 0.73.1):
导出时若不补回来,SFT 样本就只有 user 任务 + assistant 动作,没有"只能写 workspace 等"这些规则,
等于训练一个看不见规则的策略。优先读 rollout 自己的快照,其次按 cmd.json 里的路径读当前文件(会漂移,故记 source)。
"""
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
segments = _read_json(rd / "agent/raw-system.json", default=[])
if isinstance(segments, list) and segments and all(isinstance(v, str) for v in segments):
return "\n\n".join(segments), "runner-raw-system"
return "", "missing"
snap = rd / "agent" / "system.md"
if snap.exists():
return snap.read_text(errors="replace"), "snapshot"
cmd = rd / "agent" / "cmd.json"
if cmd.exists():
try:
argv = json.loads(cmd.read_text())["cmd"]
i = argv.index("--append-system-prompt")
f = Path(argv[i + 1])
if f.exists():
return f.read_text(errors="replace"), "live-file"
except (ValueError, KeyError, IndexError, OSError):
pass
return "", "missing"
def tool_defs(rd: Path) -> tuple[list, str]:
"""这条轨迹的工具定义 → (list, source)。快照没有就按 cmd.json 的 --tools 白名单现 dump 一次。"""
snap = rd / "agent" / "tools.json"
if snap.exists():
try:
definitions = json.loads(snap.read_text())
if isinstance(definitions, list):
return definitions, "snapshot"
except json.JSONDecodeError:
pass
if _read_json(rd / "rollout.json").get("runner", "pi") != "pi":
return [], "missing"
cmd = rd / "agent" / "cmd.json"
if cmd.exists():
try:
argv = json.loads(cmd.read_text())["cmd"]
wl = argv[argv.index("--tools") + 1]
import subprocess
from harness import paths
node = os.path.expanduser(str(paths.config()["paths"].get("node_bin") or "node"))
if "/" in node and not Path(node).is_absolute():
node = str(BENCH / node)
env = os.environ.copy()
env["PATH"] = str(Path(node).parent) + os.pathsep + env.get("PATH", "/usr/bin:/bin")
script = BENCH / "agent" / "dump_tools.js"
key = (node, str(script), script.stat().st_mtime_ns, wl)
if key in _TOOLS_CACHE:
return _TOOLS_CACHE[key], "redumped"
r = subprocess.run([node, str(script), wl],
capture_output=True, text=True, timeout=60, env=env)
if r.returncode == 0 and r.stdout.strip():
definitions = json.loads(r.stdout)
if isinstance(definitions, list) and definitions:
_TOOLS_CACHE[key] = definitions
return definitions, "redumped"
except Exception:
pass
return [], "missing"
def load_trajectory(rd: Path) -> tuple[list[dict], dict]:
"""Read every Pi segment; container mirrors cannot replace earlier history."""
runner = _read_json(rd / "rollout.json").get("runner", "pi")
if runner != "pi":
from harness.runners import get
return get(runner).trajectory(rd)
scanned = preserved.inventory(preserved.freeze(rd))
return [item["payload"] for item in scanned["messages"] if item["event_type"] != "system_snapshot"], {
"source": "pi-segment-events" if scanned["agent_end"] else "message_end",
"partial": bool(scanned["errors"]), "agent_end": scanned["agent_end"], "errors": scanned["errors"]}
def load_messages(rd: Path) -> list[dict] | None:
return load_trajectory(rd)[0] or None
# ---------------------------------------------------------------- R1–R8
def _text_of(content) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts = []
for c in content:
if isinstance(c, dict):
t = c.get("type")
if t in ("text", "output_text") and c.get("text"):
parts.append(c["text"])
elif t in ("thinking", "reasoning", "redacted_thinking"):
continue # R2
elif isinstance(c, str):
parts.append(c)
return "\n".join(parts)
return "" if content is None else str(content)
def r1_normalize(m: dict) -> list[dict]:
"""一条原始消息 → 0..n 条归一消息(Anthropic 的 tool_use/tool_result 混在 content 里要拆)。"""
role = m.get("role")
content = m.get("content")
out: list[dict] = []
if role in ("toolResult", "tool_result", "tool"):
return [{"role": "tool", "tool_call_id": m.get("toolCallId") or m.get("tool_call_id"),
"content": _text_of(content) or json.dumps(m.get("output") or m.get("result") or "", ensure_ascii=False)}]
if role == "assistant":
tool_calls = []
if isinstance(content, list):
for c in content:
if isinstance(c, dict) and c.get("type") in ("toolCall", "tool_use", "function_call"):
tool_calls.append({"id": c.get("id"), "name": c.get("name"),
"arguments": c.get("arguments") if "arguments" in c else c.get("input")})
for tc in m.get("tool_calls") or []:
fn = tc.get("function") or {}
tool_calls.append({"id": tc.get("id"), "name": fn.get("name") or tc.get("name"),
"arguments": fn.get("arguments") if fn else tc.get("arguments")})
msg = {"role": "assistant", "content": _text_of(content)}
if tool_calls:
msg["tool_calls"] = tool_calls
return [msg]
if role == "custom":
# pi 扩展塞进会话的消息(如 budget-kill 的预算提醒):模型是当 user 输入看到的,训练样本也要有
t = _text_of(content)
return [{"role": "user", "content": f"[{m.get('customType', 'extension')}] {t}"}] if t else []
if role in ("user", "system"):
if isinstance(content, list) and any(isinstance(c, dict) and c.get("type") == "tool_result" for c in content):
for c in content:
if isinstance(c, dict) and c.get("type") == "tool_result":
out.append({"role": "tool", "tool_call_id": c.get("tool_use_id"), "content": _text_of(c.get("content"))})
else:
t = _text_of([c])
if t:
out.append({"role": "user", "content": t})
return out
return [{"role": role, "content": _text_of(content)}]
return []
def clean(messages: list[dict]) -> tuple[list[dict], dict]:
stats = {"in": len(messages)}
msgs: list[dict] = []
for m in messages:
msgs.extend(r1_normalize(m))
# R3 system 合并到首条
systems = [m["content"] for m in msgs if m["role"] == "system"]
msgs = [m for m in msgs if m["role"] != "system"]
if systems:
msgs.insert(0, {"role": "system", "content": "\n\n".join(s for s in systems if s)})
# R5 去空
msgs = [m for m in msgs if m.get("content") or m.get("tool_calls") or m["role"] == "tool"]
# R7 tool 结果截断
for m in msgs:
if m["role"] == "tool" and len(m["content"]) > TOOL_HEAD + TOOL_TAIL:
m["content"] = m["content"][:TOOL_HEAD] + f"\n…[truncated {len(m['content']) - TOOL_HEAD - TOOL_TAIL} chars]…\n" + m["content"][-TOOL_TAIL:]
stats["r7_truncated"] = stats.get("r7_truncated", 0) + 1
# R6 合并相邻同 role 纯文本
merged: list[dict] = []
for m in msgs:
if merged and m["role"] == merged[-1]["role"] and m["role"] in ("user", "assistant") \
and not m.get("tool_calls") and not merged[-1].get("tool_calls"):
merged[-1]["content"] += "\n" + m["content"]
stats["r6_merged"] = stats.get("r6_merged", 0) + 1
else:
merged.append(m)
msgs = merged
# R4 Preserve completed tool exchanges, including a trailing tool result.
# Dropping that result used to manufacture a dangling assistant call.
# Incomplete/interior-invalid exchanges remain useful evidence, but cannot
# enter SFT; never fabricate tool results or an assistant final answer.
pending, seen = set(), set()
missing, orphan, duplicates = [], [], []
for m in msgs:
if m["role"] != "tool" and pending:
missing.extend(sorted(pending)); pending.clear()
if m["role"] == "assistant":
for tc in m.get("tool_calls") or []:
tid = tc.get("id")
if not tid or tid in seen:
duplicates.append(tid)
if tid:
seen.add(tid); pending.add(tid)
elif m["role"] == "tool":
tid = m.get("tool_call_id")
if tid not in pending:
orphan.append(tid)
else:
pending.remove(tid)
missing.extend(sorted(pending))
stats["tool_pairing"] = {"valid": not (missing or orphan or duplicates),
"missing_results": len(missing), "orphan_results": len(orphan),
"duplicate_or_missing_call_ids": len(duplicates)}
stats["trailing_user"] = bool(msgs and msgs[-1]["role"] == "user")
# R8
n_assist = sum(m["role"] == "assistant" for m in msgs)
stats.update({"out": len(msgs), "assistant_turns": n_assist, "drop": n_assist == 0 or len(msgs) < 2})
return msgs, stats
# ---------------------------------------------------------------- 主流程
def sft_eligibility(verdict: dict, reward: float) -> tuple[bool, str]:
"""Outcome eligibility only; the caller also checks transcript integrity.
Execution TIMEOUT is separate from measured terminal PASS/FAIL. A completed,
measured success is not rejected solely for exhausting its budget.
Legacy TIMEOUT verdicts stay evidence until reverified, never assumed PASS.
"""
if verdict.get("mode") == "repair":
return False, "historical-repair-context"
t = verdict.get("terminal")
if t == "PASS":
return True, "PASS"
if t == "BLOCKED":
return False, "BLOCKED-unproven"
if t == "TIMEOUT":
return False, "legacy-TIMEOUT-needs-reverification"
return False, f"{t}-not-success-demonstration"
@contextlib.contextmanager
def readonly_db():
state = Path(os.environ.get('BENCH_STATE_DIR', BENCH / 'state'))
con = sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True, timeout=60)
con.row_factory = sqlite3.Row
try:
yield con
finally:
con.close()
DB_TERMINAL_STATES = ('PASS', 'FAIL', 'BLOCKED', 'UNVERIFIED', 'TIMEOUT', 'DONE', 'ERROR', 'CANCELLED', 'DEADLINE_EXCEEDED', 'BUDGET_EXHAUSTED', 'INFRA_ERROR', 'LOST')
def _database_terminal(rd, source_root=None):
root = Path(source_root or BENCH)
state = root / 'state' if source_root is not None else Path(os.environ.get('BENCH_STATE_DIR', root / 'state'))
try:
with contextlib.closing(sqlite3.connect(f'file:{state / "jobs.sqlite"}?mode=ro', uri=True)) as con:
row = con.execute('SELECT state FROM rollouts WHERE run_id=? AND job_id=? AND kk=?',
(rd.parents[2].name, rd.parents[1].name, int(rd.name))).fetchone()
return row is not None and row[0] in DB_TERMINAL_STATES
except (OSError, sqlite3.Error):
return False
def scan(run_ids: list[str]) -> list[Path]:
# DB index only: never enumerate the JuiceFS run tree.
with readonly_db() as con:
rows = [row for rid in run_ids for row in con.execute(
"SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state IN (" +
','.join('?' for _ in DB_TERMINAL_STATES) + ')', (rid, *DB_TERMINAL_STATES))]
return sorted(BENCH / 'runs' / rid / jid / 'rollouts' / f'{kk:02d}' for rid, jid, kk in rows)
def annotation_reasons(rows, rid, jid, kk):
reasons = set()
for arun, ajob, note in rows:
if arun != rid or ajob not in (None, jid):
continue
match = re.search(r'rollout#(\d+)\b', note or '')
if match and int(match[1]) != kk:
continue
for needle, reason in [('contaminated:', 'contaminated'), ('stalled_killed', 'stalled_killed')]:
if needle in (note or ''):
reasons.add(reason)
return reasons
def next_dataset_id() -> str:
d = time.strftime("%Y%m%d")
existing = sorted((BENCH / "datasets").glob(f"ds{d}-*"))
return f"ds{d}-{len(existing) + 1:02d}"
def _read_json(path: Path, default=None):
try:
return json.loads(path.read_text())
except (OSError, ValueError):
return {} if default is None else default
def _gate_sources(rd: Path) -> dict:
"""Fingerprint the verifier's bounded inputs; never copy model artifacts."""
from harness.verify import rollout_hbm_path
files = {"job.json": rd.parent.parent / "job.json", "claim.json": rd / "claim.json",
"rollout.json": rd / "rollout.json",
"terminal_result.json": rd / "terminal_result.json"}
for pattern in ("[0-9][0-9]-*/meta.json", "[0-9][0-9]-*/a*/meta.json", "06-hbm/a*/stdout.log", "_queue/*/request.json"):
for path in (rd / "stages").glob(pattern):
files[str(path.relative_to(rd))] = path
if path.name == "request.json":
recipe = _read_json(path).get("recipe")
if recipe:
files["recipe:" + hashlib.sha256(str(recipe).encode()).hexdigest()] = Path(recipe)
job = _read_json(files["job.json"])
files["hbm"] = rollout_hbm_path(rd, job.get("job_id", rd.parent.parent.name))
stage_predictions = {}
from harness.verify import protected_predictions_path
for source in tuple(files.values()):
if source.name != "meta.json":
continue
meta = _read_json(source)
ref = ((meta.get("measurements") or {}).get("G3") or {}).get("predictions")
if ref:
key = "predictions:" + str(source.relative_to(rd))
try:
files[key] = protected_predictions_path(rd / "stages", ref)
except (OSError, ValueError, TypeError):
stage_predictions[key] = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
terminal = _read_json(rd / "terminal_result.json")
terminal = terminal if isinstance(terminal, dict) else {}
invalid_prediction = None
if terminal.get("predictions"):
from harness.verify import terminal_predictions_path
try:
files["terminal:predictions"] = terminal_predictions_path(rd, terminal)
except (OSError, ValueError, TypeError):
invalid_prediction = {"invalid": "UNSAFE_OR_MISSING_PREDICTION_SOURCE"}
manifest = dict(stage_predictions)
if invalid_prediction:
manifest["terminal:predictions"] = invalid_prediction
for name, path in sorted(files.items()):
try:
digest, size = hashlib.sha256(), 0
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1 << 20), b""):
digest.update(chunk); size += len(chunk)
manifest[name] = {"sha256": digest.hexdigest(), "bytes": size}
except FileNotFoundError:
manifest[name] = {"missing": True}
return manifest
def _hard_gate_check(job, verdict, manifest, current, measurements=None):
from harness.verify import eval_set
required = verdict.get("required") or job.get("gates", {}).get("required", [])
failures = [gate for gate in required if (verdict.get("gates", {}).get(gate) or {}).get("verdict") not in ("pass", "pass-screen")]
g3 = (verdict.get("gates") or {}).get("G3") or {}
expected_n = ((job.get("data") or {}).get("t1") or {}).get("n")
if "G3" in required and (g3.get("source") != "board" or expected_n is not None and g3.get("n") != expected_n):
failures.append("G3_SOURCE_OR_SAMPLE_COUNT")
if "G3" in required:
expected_set = eval_set(job)
measured_g3 = (measurements or {}).get("G3") or {}
measured_set = measured_g3.get("eval_set")
fields = ("dataset", "n", "content_sha", "manifest_sha", "eval_set_id", "evaluator_version")
def valid_identity(value):
return (isinstance(value, dict) and type(value.get('n')) is int and value['n'] > 0
and all(isinstance(value.get(k), str) and value[k].strip() for k in ('dataset', 'evaluator_version'))
and all(isinstance(value.get(k), str) and re.fullmatch('[0-9a-fA-F]{64}', value[k])
for k in ('content_sha', 'manifest_sha')))
if (not expected_set or not isinstance(measured_set, dict)
or not valid_identity(expected_set) or not valid_identity(measured_set)
or any((key != "eval_set_id" and expected_set.get(key) is None)
or measured_set.get(key) != expected_set[key] for key in fields)):
failures.append("G3_EVAL_SET_IDENTITY")
if expected_set and any(key in measured_g3 and measured_g3[key] != expected_set.get(key) for key in fields):
failures.append("G3_EVAL_SET_IDENTITY_CONFLICT")
# Current task and training consume the same versioned scorer result.
from harness.gates import SCORING_RULE
if verdict.get("scoring_rule") != SCORING_RULE:
failures.append("SCORING_RULE_MISMATCH")
if not required:
failures.append("REQUIRED_GATES_MISSING")
if manifest != current:
failures.append("GATE_SOURCE_CHANGED")
if verdict.get("terminal") != "PASS":
failures.append("OUTCOME")
return {"pass": not failures, "required": required, "failures": failures}
def normal_final_response(messages):
"""An agent_end marker cannot turn a tool tail or failed generation into a final."""
last = messages[-1] if messages else {}
content = last.get("content")
text = content if isinstance(content, str) else "".join(
item.get("text", "") for item in content or [] if isinstance(item, dict) and item.get("type") == "text")
ok = (last.get("role") == "assistant" and bool(text.strip()) and not preserved._calls(last)
and last.get("stopReason") not in ("error", "aborted", "length", "toolUse", "tool_calls"))
return {"pass": ok, "reason": "PASS" if ok else "MISSING_NORMAL_FINAL_RESPONSE"}
def validate_exported_record(source_root: Path, record: dict, lineage=None, excluded=None) -> dict:
"""Independent publication check against unchanged source and hard gates.
Optional sidecar arguments contain only this attempt's rows. Omitting them
checks the record payload; publication must additionally supply sidecars.
"""
from harness.verify import verify_rollout, collect_measurements
root = Path(source_root).resolve()
rd = root / "runs" / record["run_id"] / record["job_id"] / "rollouts" / f"{int(record['rollout']):02d}"
if not rd.resolve().is_relative_to(root):
raise ValueError("source path leaves bench root")
frozen = preserved.freeze(rd)
expected = preserved.reconstruct(frozen)
trajectory = record.get("trajectory", {})
actual = {**expected, "messages": record.get("messages"), "tools": record.get("tools"),
"source_ids": trajectory.get("source_ids"), "source_manifest": trajectory.get("source_manifest"),
"redaction": record.get("redact_stats"), "context": trajectory.get("context")}
for key, rows in (("lineage", lineage), ("excluded", excluded)):
if rows is not None:
fields = ("message_id", "observations", "event_type") + (("reason",) if key == "excluded" else ())
actual[key] = [{field: row[field] for field in fields} for row in rows]
checks = preserved.validate(frozen, actual)
checks["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(actual["messages"])
gate_before = _gate_sources(rd)
verdict = verify_rollout(rd, write=False)
measured = collect_measurements(rd)[0]
job = _read_json(rd.parent.parent / "job.json")
gate_check = _hard_gate_check(job, verdict, trajectory.get("hard_gate_manifest"), gate_before, measured)
from harness.verify import eval_set
if record.get("eval_set") != (measured.get("G3") or {}).get("eval_set") or record.get("required_eval_set") != eval_set(job):
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["EXPORTED_EVAL_SET_CHANGED"]}
if gate_before != _gate_sources(rd):
gate_check = {**gate_check, "pass": False, "failures": gate_check["failures"] + ["GATE_SOURCE_CHANGED"]}
checks["checks"]["HARD_GATES"] = gate_check
from export.content_review import check as check_content_review
review = check_content_review(rd, verdict, frozen, gate_before)
if record.get("content_review") != review:
review = {**review, "pass": False, "reason": "EXPORTED_REVIEW_CHANGED"}
checks["checks"]["CONTENT_REVIEW"] = review
checks['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd, root)}
try:
# Publication uses the explicitly supplied source, never the exporting
# process's current project or BENCH_STATE_DIR. Reuse collection's rule.
uri = (root / 'state/jobs.sqlite').as_uri() + '?mode=ro'
with contextlib.closing(sqlite3.connect(uri, uri=True)) as con:
con.execute('PRAGMA query_only=ON')
rows = con.execute('SELECT run_id,job_id,note FROM annotations WHERE run_id=?',
(record['run_id'],)).fetchall()
reasons = sorted(annotation_reasons(rows, record['run_id'], record['job_id'], int(record['rollout'])))
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': not reasons, 'reasons': reasons}
except (OSError, sqlite3.Error, TypeError, ValueError):
checks['checks']['DB_SOURCE_ANNOTATIONS'] = {'pass': False, 'reasons': ['DB_ANNOTATIONS_UNAVAILABLE']}
if not preserved.source_unchanged(rd, frozen):
checks["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
checks["pass"] = all(check["pass"] for check in checks["checks"].values())
return checks
def _inc(counts: dict, key, n=1):
counts[str(key)] = counts.get(str(key), 0) + n
def _log_tail(path: Path, limit=4096):
try:
with path.open("rb") as stream:
stream.seek(0, os.SEEK_END)
size = stream.tell()
stream.seek(max(0, size - limit))
return {"text": stream.read(limit).decode("utf-8", errors="replace"), "truncated": size > limit}
except OSError:
return None
def _safe_record(record: dict) -> tuple[dict, dict]:
# Stage argv can contain secrets split across list elements. Preserve the
# command structure while masking explicit secret keys/flags before regexes.
secret_keys = {"password", "passwd", "api_key", "apikey", "authorization", "secret", "access_token"}
def mask(value):
if isinstance(value, dict):
return {k: "<REDACTED>" if k.lower() in secret_keys else mask(v) for k, v in value.items()}
if isinstance(value, list):
out = [mask(v) for v in value]
for i, v in enumerate(out[:-1]):
if isinstance(v, str) and (v.lstrip("-").lower().replace("-", "_") in secret_keys
or (v == "-p" and "sshpass" in out)):
out[i + 1] = "<REDACTED>"
return out
return value
return redact_preserving(mask(record))
def pipeline_records(run_ids: list[str]):
"""One actual execution record per finished oracle job, including failures.
DB-indexed only. No recursive traversal of model/data trees and no simulated
LLM dialogue. Missing evidence is recorded rather than hiding a finished job.
"""
from harness.verify import completion_summary, eval_set
with readonly_db() as con:
jobs = [dict(row) for rid in run_ids for row in con.execute(
"SELECT * FROM jobs WHERE run_id=? AND mode='oracle' AND state IN ('PASS','FAIL','UNVERIFIED')", (rid,))]
stages = [dict(row) for rid in run_ids for row in con.execute(
"SELECT * FROM stages WHERE run_id=? AND rollout=-1 ORDER BY job_id,stage,attempt", (rid,))]
by_job = {}
for stage in stages:
by_job.setdefault((stage["run_id"], stage["job_id"]), []).append(stage)
for row in sorted(jobs, key=lambda r: (r["run_id"], r["job_id"])):
rid, jid = row["run_id"], row["job_id"]
jd = BENCH / "runs" / rid / jid
job = _read_json(jd / "job.json")
result = _read_json(jd / "oracle" / "result.json")
result_source = "oracle/result.json"
if not result and row.get("result"):
try:
result = json.loads(row["result"])
result_source = "state/jobs.sqlite:jobs.result"
except ValueError:
pass
actions = []
for stage in by_job.get((rid, jid), []):
sd = jd / "oracle" / "stages" / stage["stage"] / f"a{stage['attempt']:02d}"
meta = _read_json(sd / "meta.json")
actions.append({"stage": stage["stage"], "attempt": stage["attempt"],
"database": stage, "meta": meta, "meta_available": bool(meta),
"command": meta.get("cmd"),
"source": str(sd.relative_to(BENCH)),
"logs": {name: str((sd / name).relative_to(BENCH))
for name in ("stdout.log", "stderr.log") if (sd / name).exists()}})
if stage["state"] not in ("OK", "SKIPPED"):
actions[-1]["diagnostic_tail"] = {name: tail for name in ("stdout.log", "stderr.log")
if (tail := _log_tail(sd / name)) is not None}
cp = jd / "oracle" / "stages" / "04-calibrate" / "compile.yaml"
record = {
"schema": "bench.pipeline-training/1", "source": "deterministic_pipeline",
"id": hashlib.sha1(f"{rid}/{jid}/oracle".encode()).hexdigest()[:16],
"run_id": rid, "job_id": jid, "model": job.get("model", row.get("model")),
"eval_set": eval_set(job), "continuation": job.get("continuation"),
"task": job.get("task", row.get("task")), "march": job.get("march", row.get("march")),
"backend": job.get("backend", row.get("backend")), "terminal": result.get("terminal", row["state"]),
"completion": completion_summary(result.get("gates") or {}, result.get("stages") or {}),
"started_at": row.get("started_at"), "finished_at": row.get("finished_at"),
"job": {k: job[k] for k in ("model", "task", "march", "backend", "qcfg", "model_spec", "data", "gates") if k in job},
"compile_config": cp.read_text(errors="replace") if cp.exists() else None,
"stages": actions, "result": result,
"evidence": {"job_available": bool(job), "result_available": bool(result),
"result_source": result_source if result else None,
"job_path": str((jd / "job.json").relative_to(BENCH))},
"training_uses": ["workflow_analysis", "outcome_learning_candidate"],
"sft_eligible": False, "sft_reason": "deterministic-pipeline-not-agent-dialogue",
}
record, redaction = _safe_record(record)
record["redact_stats"] = redaction
yield record
def _write_dataset(a, work: Path, ds_id: str) -> dict:
with readonly_db() as con:
annotations = con.execute("SELECT run_id, job_id, note FROM annotations").fetchall()
coverage = {rid: {
"oracle": dict(con.execute("SELECT state,count(*) FROM jobs WHERE run_id=? AND mode='oracle' GROUP BY state", (rid,))),
"agent": dict(con.execute("SELECT state,count(*) FROM rollouts WHERE run_id=? GROUP BY state", (rid,))),
} for rid in a.runs}
stats = {"scanned": 0, "written": 0, "agent_written": 0, "teacher_written": 0, "student_written": 0,
"pipeline_written": 0, "excluded": 0, "dropped": {}, "review": 0, "partial": 0,
"by_terminal": {}, "by_execution_terminal": {}, "pipeline_by_terminal": {}, "sft_eligible": 0,
"by_march": {}, "sft_eligible_by_march": {}, "reverified": 0}
with contextlib.ExitStack() as stack:
names = ("teacher", "student", "pipeline", "exclusions", "sft", "retained", "lineage",
"operations", "excluded_observations", "validation", "annotations", "source-manifest", "disposition")
files = {name: stack.enter_context((work / f"{name}.jsonl").open("w")) for name in names}
source_seen = set()
def write(name, rec):
files[name].write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
if name == "source-manifest":
source_seen.add((rec["run_id"], rec["job_id"], rec["rollout"]))
def exclude(rd, reasons, review_binding=None, review_receipt=None):
stats["excluded"] += 1
for reason in reasons:
_inc(stats["dropped"], reason)
write("exclusions", {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name,
"rollout": int(rd.name), "source": str(rd.relative_to(BENCH)),
"reasons": sorted(reasons)})
disposition = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name),
"source": str(rd.relative_to(BENCH)), "reasons": sorted(reasons),
"disposition": "no-usable" if "no_usable_messages" in reasons else "retained"}
if review_binding:
disposition["review_binding_sha256"] = review_binding
disposition["review_receipt_sha256"] = review_receipt
write("disposition", disposition)
write("retained", disposition)
if a.purpose == "all":
for record in pipeline_records(a.runs):
write("pipeline", record)
stats["pipeline_written"] += 1
_inc(stats["pipeline_by_terminal"], record["terminal"])
for rd in scan(a.runs):
_write_agent(rd, a, stats, annotations, write, exclude)
ident = (rd.parents[2].name, rd.parents[1].name, int(rd.name))
if ident not in source_seen:
source = preserved.freeze(rd)
try:
gate_sources = _gate_sources(rd)
except (OSError, ValueError, TypeError, KeyError) as error:
# Rejected evidence must not abort the whole run's export.
gate_sources = {}
source.errors.append(f"HARD_GATES_UNAVAILABLE:{type(error).__name__}")
write("source-manifest", {"run_id": ident[0], "job_id": ident[1], "rollout": ident[2],
"source": str(rd.relative_to(BENCH)), "files": source.manifest,
"errors": source.errors, "hard_gates": gate_sources})
file_info = {}
for name in names:
path = work / f"{name}.jsonl"
with path.open("rb") as stream:
digest, lines = hashlib.sha256(), 0
for line in stream:
digest.update(line); lines += 1
file_info[path.name] = {"sha256": digest.hexdigest(), "records": lines, "bytes": path.stat().st_size}
manifest = {"schema": "bench.dataset/2", "dataset_id": ds_id, "runs": a.runs,
"reward_recipe": RECIPE_VERSION, "created_at": time.time(), "coverage": coverage,
"filters": {"min_agent_version": a.min_agent_version,
"min_agent_version_runner": a.min_agent_version_runner,
"max_turns": a.max_turns, "max_tokens": a.max_tokens,
"agent_version": a.agent_version, "agent_version_mode": "exact" if a.agent_version else "minimum",
"annotations_read": True, "dropped_counts_overlap": True, "policy": a.policy,
"include_timeout": True, "purpose": a.purpose, "min_reward": a.min_reward},
"stats": stats, "files": file_info,
"notes": ["pipeline.jsonl contains historical deterministic evidence only for purpose=all; SFT/RL export reads Agent evidence only.",
"Verdict v4 and reward v0.7 use only this rollout evidence; Oracle references are history only.",
"SFT/reward candidates are labels; no train split, framework conversion, or online RL readiness is implied.",
"Writers hold .export.lock; manifest is published last. Readers can lock or verify file hashes for a coherent refresh."]}
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
return manifest
def _write_agent(rd, a, stats, annotations, write, exclude):
from harness.verify import eval_set, verify_rollout, collect_measurements, VERDICT_SCHEMA, VERIFICATION_RULE
stats["scanned"] += 1
original_verdict = _read_json(rd / "verdict.json")
verdict = original_verdict
rj = _read_json(rd / "rollout.json")
if not verdict:
exclude(rd, {"missing_verdict"}); return
policy = verdict.get("policy") or rj.get("policy") or "teacher"
if a.policy and policy != a.policy:
exclude(rd, {"policy_filter"}); return
if policy not in ("teacher", "student"):
exclude(rd, {"unknown_policy"}); return
reasons = annotation_reasons(annotations, rd.parents[2].name, rd.parents[1].name, int(rd.name))
runner = rj.get("runner", "pi")
version = str(rj.get("agent_config_version") or 1)
if a.agent_version is not None:
if version != str(a.agent_version):
reasons.add("old_version")
elif a.min_agent_version_runner in ("*", runner):
try:
if int(version) < a.min_agent_version:
reasons.add("old_version")
except ValueError:
reasons.add("invalid_version")
if reasons:
exclude(rd, reasons); return
original_sha = hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest()
reverified = verdict.get("schema") != VERDICT_SCHEMA
try:
gate_manifest = _gate_sources(rd)
verdict = verify_rollout(rd, write=False)
stats["reverified"] += int(reverified)
except Exception as error:
exclude(rd, {f"reverification_failed:{type(error).__name__}"}); return
verification = {"schema": verdict.get("schema"),
"rule": verdict.get("verification_rule", VERIFICATION_RULE),
"method": "readonly-reverification"}
strict = None
try:
if runner == "pi":
frozen = preserved.freeze(rd)
strict = preserved.reconstruct(frozen)
validation = preserved.validate(frozen, strict)
validation["checks"]["NORMAL_FINAL_RESPONSE"] = normal_final_response(strict["messages"])
validation["checks"]["HARD_GATES"] = _hard_gate_check(_read_json(rd.parent.parent / "job.json"), verdict,
gate_manifest, _gate_sources(rd), collect_measurements(rd)[0])
if not preserved.source_unchanged(rd, frozen):
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
msgs, tools = strict["messages"], strict["tools"]
legacy_messages, _ = clean(msgs)
strict["annotations"].extend(preserved.legacy_truncation_audit(msgs, strict["source_ids"], legacy_messages))
raw = msgs
trajectory = {k: v for k, v in strict.items() if k not in ("messages", "tools", "redaction", "lineage", "operations", "excluded")}
trajectory.update(source="pi-segment-events" if strict["agent_end"] else "message_end",
partial=any(not check["pass"] for name, check in validation["checks"].items() if name != "HARD_GATES"),
validation=validation, hard_gate_manifest=gate_manifest)
ident = {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name)}
for name, values in (("lineage", strict["lineage"]), ("operations", strict["operations"]),
("excluded_observations", strict["excluded"]),
("annotations", strict["annotations"] if validation["checks"]["PRIVACY"]["pass"] else [])):
for value in values:
write(name, {**ident, **value})
write("source-manifest", {**ident, "source": str(rd.relative_to(BENCH)), "files": frozen.manifest,
"hard_gates": gate_manifest})
pairing = validation["checks"]["CALL_TOOL_CLOSURE"]
cst = {"in": len(msgs), "out": len(msgs), "assistant_turns": sum(m.get("role") == "assistant" for m in msgs),
"drop": not validation["checks"]["USABLE_ASSISTANT"]["pass"], "r7_truncated": 0,
"tool_pairing": {"valid": pairing["pass"], "missing_results": pairing["missing"],
"orphan_results": pairing["orphan"], "duplicate_or_missing_call_ids": pairing["call_id_conflict"]}}
sys_text = frozen.files.get("agent/system.md", b"").decode("utf-8", errors="strict")
sys_src = strict["context"]["source"] if sys_text else "missing"
tools_src, rst = ("snapshot" if tools else "missing"), strict["redaction"]
else:
raw, trajectory = load_trajectory(rd)
msgs, cst = clean(raw)
sys_text, sys_src = system_prompt(rd)
if sys_text:
msgs.insert(0, {"role": "system", "content": sys_text})
tools, tools_src = tool_defs(rd)
msgs, rst = redact_messages(msgs)
trajectory["strict_sft_reason"] = "UNSUPPORTED_SOURCE"
except (OSError, ValueError, KeyError, TypeError) as exc:
exclude(rd, {f"invalid_trajectory:{type(exc).__name__}"}); return
if not raw:
exclude(rd, {"no_trajectory"}); return
cst["system_source"] = sys_src
if sys_src == "runner-raw-system":
trajectory["system_segments"] = _read_json(rd / "agent/raw-system.json", default=[])
if cst["drop"]:
if strict is not None:
write("validation", {**ident, **validation})
exclude(rd, {"no_usable_messages"}); return
jd = rd.parent.parent
job = json.loads((jd / "job.json").read_text())
mode = verdict.get("mode") or rj.get("mode")
verdict = {**verdict, "mode": mode}
rw = compute_reward(verdict, rj.get("max_turns", a.max_turns), rj.get("max_tokens", a.max_tokens))
sft_ok, sft_reason = sft_eligibility(verdict, rw["reward"])
required = verdict.get("required") or job.get("gates", {}).get("required", [])
if sft_ok and (not required or any((verdict.get("gates", {}).get(g) or {}).get("verdict") not in ("pass", "pass-screen") for g in required)):
sft_ok, sft_reason = False, "required-hard-gate-not-passed"
if sft_ok and strict is None:
sft_ok, sft_reason = False, "unsupported-strict-sft-source"
if sft_ok and strict is not None and not validation["checks"]["HARD_GATES"]["pass"]:
sft_ok, sft_reason = False, "hard-gate-validation-failed"
trajectory["partial"] = trajectory["partial"] or not cst["tool_pairing"]["valid"] or cst.get("trailing_user", False)
if sft_ok and trajectory["partial"]:
sft_ok, sft_reason = False, "partial-transcript"
if sft_ok and (not tools or not sys_text):
sft_ok, sft_reason = False, "missing-tools-or-system"
known_tools = {t.get("name") or (t.get("function") or {}).get("name") for t in tools if isinstance(t, dict)}
used_tools = {tc.get("name") for msg in msgs for tc in preserved._calls(msg)}
missing_tools = sorted(name for name in used_tools - known_tools if name)
trajectory["missing_tool_definitions"] = missing_tools
if sft_ok and missing_tools:
sft_ok, sft_reason = False, "missing-called-tool-definitions"
if strict is not None:
validation['checks']['DB_SOURCE_TERMINAL'] = {'pass': _database_terminal(rd)}
if not validation['checks']['DB_SOURCE_TERMINAL']['pass']:
validation['pass'] = False
sft_ok, sft_reason = False, 'source-not-terminal'
if not preserved.source_unchanged(rd, frozen):
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
validation["pass"] = False
trajectory["partial"] = True
sft_ok, sft_reason = False, "source-changed"
if a.min_reward is not None and rw["reward"] < a.min_reward:
exclude(rd, {"min_reward"}); return
content_review = {"pass": False, "reason": "NOT_SCRIPT_QUALIFIED"}
if sft_ok:
from export.content_review import check as check_content_review
if getattr(a, "review", False) and a.purpose in ("all", "sft"):
from export.content_review import review_once
try:
review_once(rd)
except (OSError, ValueError):
sft_ok, sft_reason = False, "content-review-unavailable"
content_review = check_content_review(rd, verdict, frozen, gate_manifest)
if not preserved.source_unchanged(rd, frozen) or _gate_sources(rd) != gate_manifest:
sft_ok, sft_reason = False, "source-changed"
validation["checks"]["SOURCE_INTEGRITY"] = {"pass": False, "errors": ["SOURCE_CHANGED"]}
if not content_review["pass"]:
sft_ok, sft_reason = False, "content-review-" + content_review["reason"].lower()
if strict is not None:
validation["checks"]["CONTENT_REVIEW"] = content_review
validation["pass"] = all(check["pass"] for check in validation["checks"].values())
write("validation", {**ident, **validation})
if a.purpose == "sft" and not sft_ok:
exclude(rd, {f"sft:{sft_reason}"}, content_review.get("binding_sha256"),
content_review.get("receipt_sha256")); return
reward_candidate = verdict["terminal"] in ("PASS", "FAIL") and mode != "repair" and not rw.get("unverifiable")
if a.purpose == "rl" and not reward_candidate:
exclude(rd, {"rl:historical-repair-or-unverified-outcome"}); return
if a.min_reward is not None and rw["reward"] < a.min_reward:
exclude(rd, {"min_reward"}); return
rec = {
"schema": "bench.agent-training/2", "source": "agent_events",
"id": hashlib.sha1(f"{verdict['run_id']}/{verdict['job_id']}/{verdict['rollout']}".encode()).hexdigest()[:16],
"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
"eval_set": ((collect_measurements(rd)[0].get("G3") or {}).get("eval_set")),
"required_eval_set": eval_set(job),
"continuation": {**job["continuation"], "source_rollout": verdict["rollout"]} if job.get("continuation") else None,
"mode": mode, "policy": policy, "runner": runner, "model": rj.get("model"),
"agent_config_version": str(rj.get("agent_config_version", "1")),
"runner_version": rj.get("runner_version"),
"task": job["task"], "march": job["march"],
"board_type": (job.get("target") or {}).get("board_type"), "backend": job["backend"],
"messages": msgs,
"verdict": {k: verdict[k] for k in ("schema", "verification_rule", "terminal", "measured_terminal", "execution_terminal", "timeout_reason", "completion", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict},
"gates": {g: (verdict["gates"].get(g) or {}).get("verdict") for g in ("G1", "G2", "G3", "G4")},
"verification": verification,
"reward": rw["reward"], "reward_terms": rw["terms"], "reward_recipe": rw["recipe"],
"reward_unverifiable": rw.get("unverifiable"), # v0.4:board_offline = 板卡故障导致没法测,不是 agent 的结果
"sft_eligible": sft_ok, "sft_reason": sft_reason,
"training_uses": {"analysis": True, "sft_candidate": sft_ok, "reward_learning_candidate": reward_candidate,
"review_required": not content_review["pass"]},
"content_review": content_review,
"trajectory": trajectory,
"execution": {"terminal": verdict.get("execution_terminal") or rj.get("state"),
"timeout_reason": verdict.get("timeout_reason") or rj.get("timeout_reason"),
"started_at": rj.get("started_at"), "finished_at": rj.get("finished_at"),
"max_turns": rj.get("max_turns"), "max_tokens": rj.get("max_tokens")},
"provenance": {"events": str((rd / "agent" / "events.jsonl").relative_to(BENCH)),
"verdict": str((rd / "verdict.json").relative_to(BENCH)),
"original_verdict": {"sha256": original_sha, "schema": original_verdict.get("schema"),
"terminal": original_verdict.get("terminal"),
"review": original_verdict.get("review")}},
"cost": verdict.get("cost"),
"tools": tools,
"tools_source": tools_src,
"system_source": sys_src,
"system_sha1": hashlib.sha1(sys_text.encode()).hexdigest()[:12] if sys_text else None,
"clean_stats": cst, "redact_stats": rst,
}
# Preserve the already validated context exactly; redact metadata separately
# so a second traversal cannot silently rewrite call/source identities.
context = {key: rec.pop(key) for key in ("messages", "tools", "trajectory", "redact_stats")}
rec, _ = _safe_record(rec)
rec.update(context)
if strict is not None and not validation["checks"]["PRIVACY"]["pass"]:
# Identity cannot be rewritten safely. Keep original bytes only in the
# controlled source; public evidence contains a hash/reference receipt.
rec = {key: rec[key] for key in ("schema", "source", "id", "run_id", "job_id", "rollout", "sft_eligible", "sft_reason")}
rec.update(source_reference=str(rd.relative_to(BENCH)), reason_code="PRIVATE_SOURCE_REFERENCE_ONLY",
trajectory={"source_manifest": frozen.manifest, "validation": validation})
write(policy, rec)
write("sft" if sft_ok else "retained", rec)
write("disposition", {"run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"],
"source": str(rd.relative_to(BENCH)), "disposition": "accepted" if sft_ok else "retained",
"review_binding_sha256": content_review.get("binding_sha256"),
"review_receipt_sha256": content_review.get("receipt_sha256"),
"reasons": [] if sft_ok else [sft_reason]})
stats["written"] += 1
stats["agent_written"] += 1
stats[f"{policy}_written"] += 1
stats["partial"] += int(trajectory["partial"])
_inc(stats["by_terminal"], verdict["terminal"])
_inc(stats["by_execution_terminal"], verdict.get("execution_terminal") or rj.get("state"))
_inc(stats["by_march"], job["march"])
stats["sft_eligible"] += int(sft_ok)
if sft_ok:
_inc(stats["sft_eligible_by_march"], job["march"])
def materialize(run_ids: list[str], dataset_id: str, *, refresh=False, purpose=_UNSET, policy=_UNSET,
min_agent_version=_UNSET, agent_version=_UNSET, min_reward=_UNSET,
max_turns=_UNSET, max_tokens=_UNSET, review=False) -> dict:
"""Refresh one stable output directory; locks writers, replaces files atomically.
The manifest commits last and supplies hashes/counts. Concurrent consumers
should share .export.lock or check those hashes; the directory is not swapped.
Source rollout/run artifacts are never modified.
"""
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", dataset_id):
raise ValueError("dataset_id must be a simple directory name")
if review and not (os.environ.get("BENCH_REVIEW_URL") and os.environ.get("BENCH_REVIEW_MODEL")):
raise ValueError("--review requires BENCH_REVIEW_URL and BENCH_REVIEW_MODEL")
run_ids = list(dict.fromkeys(run_ids))
out_dir = BENCH / "datasets" / dataset_id
if out_dir.is_symlink():
raise ValueError("refusing to write a symlink dataset")
try:
out_dir.mkdir(parents=True, exist_ok=False)
except FileExistsError:
if not refresh:
raise FileExistsError(f"dataset exists: {out_dir}; use --refresh explicitly")
if (out_dir / "VOID").exists():
raise ValueError(f"refusing to refresh VOID dataset: {dataset_id}")
requested = {"purpose": purpose, "policy": policy, "agent_version": agent_version,
"min_agent_version": min_agent_version, "min_reward": min_reward,
"max_turns": max_turns, "max_tokens": max_tokens}
with (out_dir / ".export.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
prior = _read_json(out_dir / "manifest.json")
if prior and set(prior.get("runs", [])) != set(run_ids):
raise ValueError("refresh must keep the same run set; choose a new dataset_id")
defaults = {"purpose": "all", "policy": None, "agent_version": None,
"min_agent_version": 0, "min_reward": None,
"max_turns": 40, "max_tokens": 200000}
previous = prior.get("filters", {})
if min_agent_version is None or (min_agent_version is _UNSET and "min_agent_version" not in previous):
version_file = BENCH / "agent/VERSION"
defaults["min_agent_version"] = int(version_file.read_text()) if version_file.exists() else 1
resolved = {key: previous.get(key, default) if requested[key] is _UNSET else requested[key]
for key, default in defaults.items()}
scope = previous.get("min_agent_version_runner", "*" if prior else "pi")
if min_agent_version is not _UNSET:
scope = "pi" if min_agent_version is None else "*"
if resolved["min_agent_version"] is None:
resolved["min_agent_version"] = defaults["min_agent_version"] if min_agent_version is None else 0
if resolved["purpose"] not in ("all", "sft", "rl"):
raise ValueError("purpose must be all/sft/rl")
if resolved["policy"] not in (None, "teacher", "student"):
raise ValueError("policy must be teacher/student or None")
if not isinstance(resolved["min_agent_version"], int) or resolved["min_agent_version"] < 0:
raise ValueError("min_agent_version must be a nonnegative integer")
a = argparse.Namespace(runs=run_ids, min_agent_version_runner=scope, review=review, **resolved)
with tempfile.TemporaryDirectory(prefix=".refresh-", dir=out_dir) as temp:
work = Path(temp)
manifest = _write_dataset(a, work, dataset_id)
if prior.get("created_at"):
manifest["created_at"] = prior["created_at"]
manifest["updated_at"] = time.time()
manifest["filter_history"] = list(prior.get("filter_history", []))
if prior and previous != manifest["filters"]:
explicit = sorted(k for k, v in requested.items() if v is not _UNSET)
manifest["filter_history"].append({"changed_at": manifest["updated_at"],
"reason": "explicit_filter_change" if explicit else "legacy_filter_metadata_upgrade",
"explicit_fields": explicit, "before": previous, "after": manifest["filters"]})
(work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
for name in [*manifest["files"], "manifest.json"]:
os.replace(work / name, out_dir / name)
return manifest
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--runs", nargs="+", required=True)
ap.add_argument("--policy", choices=["teacher", "student"], default=argparse.SUPPRESS)
ap.add_argument("--include-timeout", action="store_true", help="compatibility flag: timeout evidence is now included by default")
ap.add_argument("--purpose", choices=["all", "sft", "rl"], default=argparse.SUPPRESS,
help="all=保留终态轨迹并标注用途;sft=完整诚实成功候选;rl=本次可核验 PASS/FAIL 候选")
ap.add_argument("--min-reward", type=float, default=argparse.SUPPRESS)
ap.add_argument("--max-turns", type=int, default=argparse.SUPPRESS)
ap.add_argument("--max-tokens", type=int, default=argparse.SUPPRESS)
ap.add_argument("--dataset-id")
ap.add_argument("--review", action="store_true", help="explicitly review script-qualified candidates once; requires reviewer environment")
ap.add_argument("--refresh", action="store_true", help="explicitly refresh this dataset from its source runs")
ap.add_argument("--agent-version", default=argparse.SUPPRESS)
ap.add_argument("--min-agent-version", type=int, default=argparse.SUPPRESS)
a = ap.parse_args()
filters = {key: value for key, value in vars(a).items() if key in (
"purpose", "policy", "min_agent_version", "agent_version", "min_reward", "max_turns", "max_tokens")}
manifest = materialize(a.runs, a.dataset_id or next_dataset_id(), refresh=a.refresh, review=a.review, **filters)
print(json.dumps(manifest, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())