data-gen-bpu/harness/continuation.py

333 lines
18 KiB
Python

"""A remaining-work handoff: inspect a JSON plan, then apply it after draining.
No matrix expansion, execution-history copying, or second scheduler. The plan is
also the receipt used to check exact job/rollout identities after the handoff.
"""
from __future__ import annotations
from collections import Counter
from copy import deepcopy
from contextlib import ExitStack, contextmanager
import hashlib
import json
from pathlib import Path
import re
import tarfile
import time
from . import db, paths, registry
from .eval import EVALUATOR_VERSION
def _sha(value) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def _read(path: Path):
return json.loads(path.read_text())
def _write_json(path: Path, value: dict, *, readonly: bool = False) -> None:
temporary = path.with_name(f".{path.name}.tmp")
temporary.unlink(missing_ok=True)
temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False))
if readonly:
temporary.chmod(0o444)
temporary.replace(path)
def load_eval_set(manifest: str | Path) -> tuple[str, dict]:
"""Validate the actual package, not just an advertised sample count."""
manifest = Path(manifest).resolve()
m = _read(manifest)
tar = manifest.parent / m["tar"]
if Path(m["tar"]).name != m["tar"]:
raise ValueError("T1 manifest.tar must be a filename")
with tar.open("rb") as stream:
digest = hashlib.sha256()
for chunk in iter(lambda: stream.read(1 << 20), b""):
digest.update(chunk)
h = digest.hexdigest()
if h != m["sha256"]:
raise ValueError(f"T1 package hash mismatch: {tar}")
with tarfile.open(tar) as archive:
inner = json.load(archive.extractfile("manifest.json"))
items = inner["items"]
names = set(archive.getnames())
if (not items or len(items) != m["n"] or inner["n"] != m["n"]
or len({str(i["id"]) for i in items}) != len(items)
or any(i["file"] not in names for i in items)
or inner["dataset"] != m["dataset"]
or inner["sha256"] != m["content_sha256"]):
raise ValueError(f"T1 manifest/items mismatch: {tar}")
return m["dataset"], {
"path": str(tar), "n": len(items), "seed": m["seed"],
"manifest_sha": h, "content_sha": m["content_sha256"],
"eval_set_id": m.get("eval_set_id") or f"{m['dataset']}-n{len(items)}-{m['content_sha256'][:12]}",
}
def _snapshot(con, source: str) -> dict:
"""Read SQLite rows only; shared-filesystem reads must not hold a DB lock."""
run = db.get_run(con, source)
if not run:
raise ValueError(f"unknown source run: {source}")
jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE run_id=? ORDER BY job_id,mode,policy", (source,))]
rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (source,))]
for row in rollouts:
row["runner"] = db.rollout_runner(con, source, row["job_id"], row["kk"])
stages = [dict(r) for r in con.execute("SELECT * FROM stages WHERE run_id=? ORDER BY job_id,rollout,stage,attempt", (source,))]
return {"run": run, "jobs": jobs, "rollouts": rollouts, "stages": stages}
def _snapshot_files(source: str, rows: dict) -> dict:
specs = {jid: _read(paths.RUNS / source / jid / "job.json")
for jid in sorted({j["job_id"] for j in rows["jobs"]} | {r["job_id"] for r in rows["rollouts"]})}
return {"specs": specs,
"registry_sha": registry.registry_fingerprint(include_board_resources=False)}
def plan(source: str, manifests: list[str], target: str | None = None) -> dict:
replacements = dict(load_eval_set(p) for p in manifests)
if len(replacements) != len(manifests):
raise ValueError("one T1 manifest per dataset is required")
with db.readonly() as con:
con.execute("BEGIN")
snapshot = _snapshot(con, source)
target = target or db.next_run_id(con, time.strftime("%Y%m%d"))
if db.get_run(con, target):
raise ValueError(f"target run already exists: {target}")
snapshot.update(_snapshot_files(source, snapshot))
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", target) or target == source:
raise ValueError("target must be a new, simple run id")
old_manifest = json.loads(snapshot["run"]["manifest"])
if old_manifest.get("continuation") or old_manifest.get("continued_to"):
raise ValueError("source has already been continued; use its saved handoff receipt")
expected = old_manifest.get("registry_task_fingerprint")
matches = (expected == snapshot["registry_sha"] if expected else
old_manifest.get("registry_fingerprint") == registry.registry_fingerprint())
initial_boards = paths.RUNS / source / "board_registry.initial.yaml"
if not matches and not expected and initial_boards.exists():
# Legacy runs froze the whole registry, including the board inventory.
# Match that original inventory and the current static task rules.
initial = initial_boards.read_bytes()
matches = (registry.registry_fingerprint(board_bytes=initial) == old_manifest.get("registry_fingerprint")
and registry.registry_fingerprint(include_board_resources=False, board_bytes=initial)
== snapshot["registry_sha"])
if not matches:
raise ValueError("source registry changed; restore its task rules before planning")
jobs, blockers = [], []
oracle_ids = {j["job_id"] for j in snapshot["jobs"] if j["mode"] == "oracle"}
source_jobs = {j["job_id"]: j for j in snapshot["jobs"] if j["mode"] == "oracle"}
# E2E agents can prepare their own inputs without an Oracle task.
for row in snapshot["jobs"]:
if row["mode"] == "e2e":
source_jobs.setdefault(row["job_id"], row)
if any(r["state"] in ("QUEUED", "RUNNING", "VERIFYING")
and (r["job_id"] not in source_jobs
or (r["job_id"] not in oracle_ids and r["mode"] != "e2e"))
for r in snapshot["rollouts"]):
blockers.append("rollout has no source Oracle or independent E2E task")
for row in source_jobs.values():
jid = row["job_id"]
jd = paths.RUNS / source / jid
existing = [r for r in snapshot["rollouts"] if r["job_id"] == jid]
oracle = row["mode"] == "oracle" and row["state"] == "PENDING"
if row["state"] == "ORACLE_RUNNING":
blockers.append(f"oracle still running: {jid}")
if oracle and (row["started_at"] is not None or row["finished_at"] is not None
or row["result"] is not None or any(s["job_id"] == jid and s["rollout"] == -1 for s in snapshot["stages"])
or any((jd / "oracle" / "stages").glob("*/a*")) or (jd / "oracle" / "result.json").exists()):
blockers.append(f"PENDING has execution history: {jid}")
queued = []
for r in existing:
if r["state"] in ("RUNNING", "VERIFYING"):
blockers.append(f"rollout still active: {jid}#{r['kk']}")
if r["state"] != "QUEUED":
continue
if (r["started_at"] is not None or r["finished_at"] is not None or r["verdict"] is not None
or (jd / "rollouts" / f"{r['kk']:02d}").exists()
or any(s["job_id"] == jid and s["rollout"] == r["kk"] for s in snapshot["stages"])):
blockers.append(f"QUEUED has execution history: {jid}#{r['kk']}")
queued.append({"kk": r["kk"], "mode": r["mode"], "policy": r["policy"], "runner": r["runner"], "source_exists": True})
job = deepcopy(snapshot["specs"][jid])
if not oracle and not queued:
continue
if row["mode"] == "oracle" and not oracle and row["state"] not in ("PREPARED", "PASS", "FAIL", "UNVERIFIED"):
blockers.append(f"oracle dependency not finished: {jid} ({row['state']})")
ds = job["data"]["dataset"]
if job["data"]["t1"].get("n") not in (None, 0):
if ds in replacements:
job["data"]["t1"] = deepcopy(replacements[ds])
elif job["data"]["t1"].get("eval_set_id"):
# Newly admitted jobs may already freeze their intended subset.
# Preserve it after checking its bytes; no extra handoff required.
frozen = job["data"]["t1"]
actual_ds, actual = load_eval_set(Path(frozen["path"]).parent / "manifest.json")
if actual_ds != ds or actual != frozen:
raise ValueError(f"frozen explicit T1 changed: {jid}")
else:
raise ValueError(f"missing replacement T1 manifest for {ds} ({jid})")
job["run_id"] = target
job["evaluator_version"] = EVALUATOR_VERSION
job["continuation"] = {"source_run_id": source, "source_job_id": jid}
if row["mode"] == "oracle" and not oracle:
job["oracle_ref"] = {"run_id": source, "job_id": jid}
job["rollout_plans"] = [{"mode": mode, "policy": policy, "runner": runner, "k": n}
for (mode, policy, runner), n in Counter((r["mode"], r["policy"], r["runner"]) for r in queued).items()]
jobs.append({"job": job, "oracle": oracle, "source_oracle_state": row["state"] if row["mode"] == "oracle" else None,
"source_error_class": row["error_class"], "estimate_s": row["estimate_s"], "rollouts": queued})
if any(s["state"] == "RUNNING" for s in snapshot["stages"]):
blockers.append("stages still running")
counts = {"oracle": sum(j["oracle"] for j in jobs),
"agent_queued": sum(r["source_exists"] for j in jobs for r in j["rollouts"]),
"agent_due_repairs": 0, "conditional_repairs": 0}
result = {"schema": "bench.continuation/1", "source_run_id": source, "target_run_id": target,
"source_sha256": _sha(snapshot), "eval_manifests": [str(Path(p).resolve()) for p in manifests],
"jobs": jobs, "conditional_repairs": [], "counts": counts, "blockers": blockers}
result["plan_id"] = _sha(result)
return result
def receipt(run_id: str) -> dict | None:
p = paths.RUNS / run_id / "continuation.json"
return _read(p) if p.exists() else None
@contextmanager
def _drained(con, source: str, target: str):
import fcntl
from .pipeline import drain_file, collect_drain_file, scheduler_alive
if not drain_file(source).exists() or not collect_drain_file(source).exists():
raise ValueError("drain both pipeline and collector before applying")
if scheduler_alive():
raise ValueError("pipeline scheduler still running; wait for it to exit")
with ExitStack() as locks:
# Reuse collector locks to serialize file preparation for both ends.
# Different drained sources must not prepare the same target together.
for run_id in (source, target):
p = paths.STATE / "locks" / f"collect-{run_id}.lock"
p.parent.mkdir(parents=True, exist_ok=True)
f = locks.enter_context(p.open("a"))
try:
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise ValueError("collector still running or target preparation busy; wait for it to exit") from None
active = con.execute("SELECT 1 FROM jobs WHERE run_id=? AND state='ORACLE_RUNNING' UNION ALL "
"SELECT 1 FROM rollouts WHERE run_id=? AND state IN ('RUNNING','VERIFYING') UNION ALL "
"SELECT 1 FROM stages WHERE run_id=? AND state='RUNNING'", (source, source, source)).fetchone()
if active:
raise ValueError("source still has active work")
yield
def apply(handoff: dict) -> dict:
from .pipeline import build_manifest
if handoff.get("plan_id") != _sha({k: v for k, v in handoff.items() if k != "plan_id"}):
raise ValueError("plan was edited; regenerate it instead")
source, target = handoff["source_run_id"], handoff["target_run_id"]
with db.readonly() as con:
previous = db.get_run(con, target)
if previous:
if json.loads(previous["manifest"]).get("continuation", {}).get("plan_id") != handoff["plan_id"]:
raise ValueError("target belongs to another plan")
return check(target)
if handoff["blockers"] or not handoff["jobs"]:
raise ValueError("plan is blocked or empty; regenerate after draining")
# Rebuild the read-only plan: catches changed packages, configuration and state.
fresh = plan(source, handoff["eval_manifests"], target)
if fresh != handoff:
raise ValueError("source changed since preview; regenerate the plan")
manifest = build_manifest()
manifest["continuation"] = {"source_run_id": source, "plan_id": handoff["plan_id"]}
rd = paths.RUNS / target
with db.connect() as con, _drained(con, source, target):
rows = _snapshot(con, source)
files = _snapshot_files(source, rows)
if _sha({**rows, **files}) != handoff["source_sha256"]:
raise ValueError("source changed during apply; regenerate the plan")
if rd.exists():
saved = receipt(target)
if saved != handoff and (saved is not None or any(p.name != ".continuation.json.tmp" for p in rd.iterdir())):
raise ValueError(f"target directory already exists: {rd}")
rd.mkdir(parents=True, exist_ok=True)
# Files precede the DB commit. An interrupted preparation is retryable
# only with the same receipt; no schedulable run exists until commit.
_write_json(rd / "continuation.json", handoff)
_write_json(rd / "manifest.json", manifest)
for entry in handoff["jobs"]:
job = entry["job"]
jid = job["job_id"]
jd = rd / jid
jd.mkdir(exist_ok=True)
jp = jd / "job.json"
if jp.exists():
if _read(jp) != job:
raise ValueError(f"prepared job differs: {jp}")
else:
_write_json(jp, job, readonly=True)
if not (jd / "data").exists() and not (jd / "data").is_symlink():
(jd / "data").symlink_to(paths.DATA)
if entry["oracle"]:
(jd / "oracle" / "stages").mkdir(parents=True, exist_ok=True)
(jd / "oracle" / "artifacts").mkdir(exist_ok=True)
# Preparation can be slow on shared storage. Keep the collector lock,
# but acquire the SQLite writer lock only after all file work is done.
if _snapshot_files(source, rows) != files:
raise ValueError("source changed during apply; regenerate the plan")
con.execute("BEGIN IMMEDIATE")
if _sha({**_snapshot(con, source), **files}) != handoff["source_sha256"]:
raise ValueError("source changed during apply; regenerate the plan")
for entry in handoff["jobs"]:
job = entry["job"]
jid = job["job_id"]
if entry["oracle"]:
db.upsert_job(con, target, {**job, "mode": "oracle"}, entry["estimate_s"])
con.execute("UPDATE jobs SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND mode='oracle'", (source, jid))
for r in entry["rollouts"]:
db.rollout_upsert(con, target, jid, r["kk"], r["mode"], r["policy"], runner=r.get("runner", "pi"))
con.execute("UPDATE rollouts SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND kk=?", (source, jid, r["kk"]))
db.create_run(con, target, manifest, f"remaining work from {source}")
old = json.loads(db.get_run(con, source)["manifest"])
old["continued_to"] = target
con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(old), source))
con.commit() # release the collector lock only after the ownership change
return check(target)
def enqueue_repairs(run_id: str, handoff: dict, job_ids=None, modes=None) -> int:
"""历史交接回执兼容入口;旧 conditional slots 不再激活。"""
with db.connect() as con:
db.assert_run_open(con, run_id)
return 0
def check(run_id: str) -> dict:
handoff = receipt(run_id)
if not handoff:
raise ValueError("run has no continuation receipt")
if handoff["plan_id"] != _sha({k: v for k, v in handoff.items() if k != "plan_id"}):
raise ValueError("continuation receipt was edited")
for entry in handoff["jobs"]:
jp = paths.RUNS / run_id / entry["job"]["job_id"] / "job.json"
if not jp.exists() or _read(jp) != entry["job"]:
raise ValueError(f"handoff job snapshot missing or changed: {jp}")
expected_jobs = {j["job"]["job_id"] for j in handoff["jobs"] if j["oracle"]}
expected = {(j["job"]["job_id"], r["kk"], r["mode"], r["policy"], r.get("runner", "pi")) for j in handoff["jobs"] for r in j["rollouts"]}
conditional = {(j["job_id"], r["kk"], r["mode"], r["policy"], r.get("runner", "pi")) for j in handoff["conditional_repairs"] for r in j["rollouts"]}
with db.readonly() as con:
jobs = db.list_jobs(con, run_id, mode="oracle")
rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))]
for row in rows:
row["runner"] = db.rollout_runner(con, run_id, row["job_id"], row["kk"])
source = handoff["source_run_id"]
moved_jobs = {r[0] for r in con.execute("SELECT job_id FROM jobs WHERE run_id=? AND mode='oracle' AND state='TRANSFERRED'", (source,))}
moved = {(r[0], r[1], r[2], r[3], db.rollout_runner(con, source, r[0], r[1])) for r in con.execute("SELECT job_id,kk,mode,policy FROM rollouts WHERE run_id=? AND state='TRANSFERRED'", (source,))}
actual = {(r["job_id"], r["kk"], r["mode"], r["policy"], r["runner"]) for r in rows}
actual_jobs = {j["job_id"] for j in jobs}
if actual_jobs != expected_jobs or moved_jobs != expected_jobs or moved != expected or not expected <= actual or actual - expected - conditional:
raise ValueError("handoff identity mismatch: missing, additional, or untransferred tasks")
return {"run_id": run_id, "source_run_id": source, "plan_id": handoff["plan_id"], "identities_match": True,
"counts": handoff["counts"], "oracle_states": dict(Counter(j["state"] for j in jobs)),
"agent_states": dict(Counter(r["state"] for r in rows)), "activated_repairs": len(actual - expected)}