333 lines
18 KiB
Python
333 lines
18 KiB
Python
"""A remaining-work handoff: inspect a JSON plan, then apply it after draining.
|
|
|
|
No matrix expansion, execution-history copying, or second scheduler. The plan is
|
|
also the receipt used to check exact job/rollout identities after the handoff.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from copy import deepcopy
|
|
from contextlib import ExitStack, contextmanager
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tarfile
|
|
import time
|
|
|
|
from . import db, paths, registry
|
|
from .eval import EVALUATOR_VERSION
|
|
|
|
|
|
def _sha(value) -> str:
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
|
|
|
|
|
def _read(path: Path):
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def _write_json(path: Path, value: dict, *, readonly: bool = False) -> None:
|
|
temporary = path.with_name(f".{path.name}.tmp")
|
|
temporary.unlink(missing_ok=True)
|
|
temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False))
|
|
if readonly:
|
|
temporary.chmod(0o444)
|
|
temporary.replace(path)
|
|
|
|
|
|
def load_eval_set(manifest: str | Path) -> tuple[str, dict]:
|
|
"""Validate the actual package, not just an advertised sample count."""
|
|
manifest = Path(manifest).resolve()
|
|
m = _read(manifest)
|
|
tar = manifest.parent / m["tar"]
|
|
if Path(m["tar"]).name != m["tar"]:
|
|
raise ValueError("T1 manifest.tar must be a filename")
|
|
with tar.open("rb") as stream:
|
|
digest = hashlib.sha256()
|
|
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
|
digest.update(chunk)
|
|
h = digest.hexdigest()
|
|
if h != m["sha256"]:
|
|
raise ValueError(f"T1 package hash mismatch: {tar}")
|
|
with tarfile.open(tar) as archive:
|
|
inner = json.load(archive.extractfile("manifest.json"))
|
|
items = inner["items"]
|
|
names = set(archive.getnames())
|
|
if (not items or len(items) != m["n"] or inner["n"] != m["n"]
|
|
or len({str(i["id"]) for i in items}) != len(items)
|
|
or any(i["file"] not in names for i in items)
|
|
or inner["dataset"] != m["dataset"]
|
|
or inner["sha256"] != m["content_sha256"]):
|
|
raise ValueError(f"T1 manifest/items mismatch: {tar}")
|
|
return m["dataset"], {
|
|
"path": str(tar), "n": len(items), "seed": m["seed"],
|
|
"manifest_sha": h, "content_sha": m["content_sha256"],
|
|
"eval_set_id": m.get("eval_set_id") or f"{m['dataset']}-n{len(items)}-{m['content_sha256'][:12]}",
|
|
}
|
|
|
|
|
|
def _snapshot(con, source: str) -> dict:
|
|
"""Read SQLite rows only; shared-filesystem reads must not hold a DB lock."""
|
|
run = db.get_run(con, source)
|
|
if not run:
|
|
raise ValueError(f"unknown source run: {source}")
|
|
jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE run_id=? ORDER BY job_id,mode,policy", (source,))]
|
|
rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (source,))]
|
|
for row in rollouts:
|
|
row["runner"] = db.rollout_runner(con, source, row["job_id"], row["kk"])
|
|
stages = [dict(r) for r in con.execute("SELECT * FROM stages WHERE run_id=? ORDER BY job_id,rollout,stage,attempt", (source,))]
|
|
return {"run": run, "jobs": jobs, "rollouts": rollouts, "stages": stages}
|
|
|
|
|
|
def _snapshot_files(source: str, rows: dict) -> dict:
|
|
specs = {jid: _read(paths.RUNS / source / jid / "job.json")
|
|
for jid in sorted({j["job_id"] for j in rows["jobs"]} | {r["job_id"] for r in rows["rollouts"]})}
|
|
return {"specs": specs,
|
|
"registry_sha": registry.registry_fingerprint(include_board_resources=False)}
|
|
|
|
|
|
def plan(source: str, manifests: list[str], target: str | None = None) -> dict:
|
|
replacements = dict(load_eval_set(p) for p in manifests)
|
|
if len(replacements) != len(manifests):
|
|
raise ValueError("one T1 manifest per dataset is required")
|
|
with db.readonly() as con:
|
|
con.execute("BEGIN")
|
|
snapshot = _snapshot(con, source)
|
|
target = target or db.next_run_id(con, time.strftime("%Y%m%d"))
|
|
if db.get_run(con, target):
|
|
raise ValueError(f"target run already exists: {target}")
|
|
snapshot.update(_snapshot_files(source, snapshot))
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", target) or target == source:
|
|
raise ValueError("target must be a new, simple run id")
|
|
old_manifest = json.loads(snapshot["run"]["manifest"])
|
|
if old_manifest.get("continuation") or old_manifest.get("continued_to"):
|
|
raise ValueError("source has already been continued; use its saved handoff receipt")
|
|
expected = old_manifest.get("registry_task_fingerprint")
|
|
matches = (expected == snapshot["registry_sha"] if expected else
|
|
old_manifest.get("registry_fingerprint") == registry.registry_fingerprint())
|
|
initial_boards = paths.RUNS / source / "board_registry.initial.yaml"
|
|
if not matches and not expected and initial_boards.exists():
|
|
# Legacy runs froze the whole registry, including the board inventory.
|
|
# Match that original inventory and the current static task rules.
|
|
initial = initial_boards.read_bytes()
|
|
matches = (registry.registry_fingerprint(board_bytes=initial) == old_manifest.get("registry_fingerprint")
|
|
and registry.registry_fingerprint(include_board_resources=False, board_bytes=initial)
|
|
== snapshot["registry_sha"])
|
|
if not matches:
|
|
raise ValueError("source registry changed; restore its task rules before planning")
|
|
jobs, blockers = [], []
|
|
oracle_ids = {j["job_id"] for j in snapshot["jobs"] if j["mode"] == "oracle"}
|
|
source_jobs = {j["job_id"]: j for j in snapshot["jobs"] if j["mode"] == "oracle"}
|
|
# E2E agents can prepare their own inputs without an Oracle task.
|
|
for row in snapshot["jobs"]:
|
|
if row["mode"] == "e2e":
|
|
source_jobs.setdefault(row["job_id"], row)
|
|
if any(r["state"] in ("QUEUED", "RUNNING", "VERIFYING")
|
|
and (r["job_id"] not in source_jobs
|
|
or (r["job_id"] not in oracle_ids and r["mode"] != "e2e"))
|
|
for r in snapshot["rollouts"]):
|
|
blockers.append("rollout has no source Oracle or independent E2E task")
|
|
for row in source_jobs.values():
|
|
jid = row["job_id"]
|
|
jd = paths.RUNS / source / jid
|
|
existing = [r for r in snapshot["rollouts"] if r["job_id"] == jid]
|
|
oracle = row["mode"] == "oracle" and row["state"] == "PENDING"
|
|
if row["state"] == "ORACLE_RUNNING":
|
|
blockers.append(f"oracle still running: {jid}")
|
|
if oracle and (row["started_at"] is not None or row["finished_at"] is not None
|
|
or row["result"] is not None or any(s["job_id"] == jid and s["rollout"] == -1 for s in snapshot["stages"])
|
|
or any((jd / "oracle" / "stages").glob("*/a*")) or (jd / "oracle" / "result.json").exists()):
|
|
blockers.append(f"PENDING has execution history: {jid}")
|
|
queued = []
|
|
for r in existing:
|
|
if r["state"] in ("RUNNING", "VERIFYING"):
|
|
blockers.append(f"rollout still active: {jid}#{r['kk']}")
|
|
if r["state"] != "QUEUED":
|
|
continue
|
|
if (r["started_at"] is not None or r["finished_at"] is not None or r["verdict"] is not None
|
|
or (jd / "rollouts" / f"{r['kk']:02d}").exists()
|
|
or any(s["job_id"] == jid and s["rollout"] == r["kk"] for s in snapshot["stages"])):
|
|
blockers.append(f"QUEUED has execution history: {jid}#{r['kk']}")
|
|
queued.append({"kk": r["kk"], "mode": r["mode"], "policy": r["policy"], "runner": r["runner"], "source_exists": True})
|
|
job = deepcopy(snapshot["specs"][jid])
|
|
if not oracle and not queued:
|
|
continue
|
|
if row["mode"] == "oracle" and not oracle and row["state"] not in ("PREPARED", "PASS", "FAIL", "UNVERIFIED"):
|
|
blockers.append(f"oracle dependency not finished: {jid} ({row['state']})")
|
|
ds = job["data"]["dataset"]
|
|
if job["data"]["t1"].get("n") not in (None, 0):
|
|
if ds in replacements:
|
|
job["data"]["t1"] = deepcopy(replacements[ds])
|
|
elif job["data"]["t1"].get("eval_set_id"):
|
|
# Newly admitted jobs may already freeze their intended subset.
|
|
# Preserve it after checking its bytes; no extra handoff required.
|
|
frozen = job["data"]["t1"]
|
|
actual_ds, actual = load_eval_set(Path(frozen["path"]).parent / "manifest.json")
|
|
if actual_ds != ds or actual != frozen:
|
|
raise ValueError(f"frozen explicit T1 changed: {jid}")
|
|
else:
|
|
raise ValueError(f"missing replacement T1 manifest for {ds} ({jid})")
|
|
job["run_id"] = target
|
|
job["evaluator_version"] = EVALUATOR_VERSION
|
|
job["continuation"] = {"source_run_id": source, "source_job_id": jid}
|
|
if row["mode"] == "oracle" and not oracle:
|
|
job["oracle_ref"] = {"run_id": source, "job_id": jid}
|
|
job["rollout_plans"] = [{"mode": mode, "policy": policy, "runner": runner, "k": n}
|
|
for (mode, policy, runner), n in Counter((r["mode"], r["policy"], r["runner"]) for r in queued).items()]
|
|
jobs.append({"job": job, "oracle": oracle, "source_oracle_state": row["state"] if row["mode"] == "oracle" else None,
|
|
"source_error_class": row["error_class"], "estimate_s": row["estimate_s"], "rollouts": queued})
|
|
if any(s["state"] == "RUNNING" for s in snapshot["stages"]):
|
|
blockers.append("stages still running")
|
|
counts = {"oracle": sum(j["oracle"] for j in jobs),
|
|
"agent_queued": sum(r["source_exists"] for j in jobs for r in j["rollouts"]),
|
|
"agent_due_repairs": 0, "conditional_repairs": 0}
|
|
result = {"schema": "bench.continuation/1", "source_run_id": source, "target_run_id": target,
|
|
"source_sha256": _sha(snapshot), "eval_manifests": [str(Path(p).resolve()) for p in manifests],
|
|
"jobs": jobs, "conditional_repairs": [], "counts": counts, "blockers": blockers}
|
|
result["plan_id"] = _sha(result)
|
|
return result
|
|
|
|
|
|
def receipt(run_id: str) -> dict | None:
|
|
p = paths.RUNS / run_id / "continuation.json"
|
|
return _read(p) if p.exists() else None
|
|
|
|
|
|
@contextmanager
|
|
def _drained(con, source: str, target: str):
|
|
import fcntl
|
|
from .pipeline import drain_file, collect_drain_file, scheduler_alive
|
|
if not drain_file(source).exists() or not collect_drain_file(source).exists():
|
|
raise ValueError("drain both pipeline and collector before applying")
|
|
if scheduler_alive():
|
|
raise ValueError("pipeline scheduler still running; wait for it to exit")
|
|
with ExitStack() as locks:
|
|
# Reuse collector locks to serialize file preparation for both ends.
|
|
# Different drained sources must not prepare the same target together.
|
|
for run_id in (source, target):
|
|
p = paths.STATE / "locks" / f"collect-{run_id}.lock"
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
f = locks.enter_context(p.open("a"))
|
|
try:
|
|
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise ValueError("collector still running or target preparation busy; wait for it to exit") from None
|
|
active = con.execute("SELECT 1 FROM jobs WHERE run_id=? AND state='ORACLE_RUNNING' UNION ALL "
|
|
"SELECT 1 FROM rollouts WHERE run_id=? AND state IN ('RUNNING','VERIFYING') UNION ALL "
|
|
"SELECT 1 FROM stages WHERE run_id=? AND state='RUNNING'", (source, source, source)).fetchone()
|
|
if active:
|
|
raise ValueError("source still has active work")
|
|
yield
|
|
|
|
|
|
def apply(handoff: dict) -> dict:
|
|
from .pipeline import build_manifest
|
|
if handoff.get("plan_id") != _sha({k: v for k, v in handoff.items() if k != "plan_id"}):
|
|
raise ValueError("plan was edited; regenerate it instead")
|
|
source, target = handoff["source_run_id"], handoff["target_run_id"]
|
|
with db.readonly() as con:
|
|
previous = db.get_run(con, target)
|
|
if previous:
|
|
if json.loads(previous["manifest"]).get("continuation", {}).get("plan_id") != handoff["plan_id"]:
|
|
raise ValueError("target belongs to another plan")
|
|
return check(target)
|
|
if handoff["blockers"] or not handoff["jobs"]:
|
|
raise ValueError("plan is blocked or empty; regenerate after draining")
|
|
# Rebuild the read-only plan: catches changed packages, configuration and state.
|
|
fresh = plan(source, handoff["eval_manifests"], target)
|
|
if fresh != handoff:
|
|
raise ValueError("source changed since preview; regenerate the plan")
|
|
manifest = build_manifest()
|
|
manifest["continuation"] = {"source_run_id": source, "plan_id": handoff["plan_id"]}
|
|
rd = paths.RUNS / target
|
|
with db.connect() as con, _drained(con, source, target):
|
|
rows = _snapshot(con, source)
|
|
files = _snapshot_files(source, rows)
|
|
if _sha({**rows, **files}) != handoff["source_sha256"]:
|
|
raise ValueError("source changed during apply; regenerate the plan")
|
|
if rd.exists():
|
|
saved = receipt(target)
|
|
if saved != handoff and (saved is not None or any(p.name != ".continuation.json.tmp" for p in rd.iterdir())):
|
|
raise ValueError(f"target directory already exists: {rd}")
|
|
rd.mkdir(parents=True, exist_ok=True)
|
|
# Files precede the DB commit. An interrupted preparation is retryable
|
|
# only with the same receipt; no schedulable run exists until commit.
|
|
_write_json(rd / "continuation.json", handoff)
|
|
_write_json(rd / "manifest.json", manifest)
|
|
for entry in handoff["jobs"]:
|
|
job = entry["job"]
|
|
jid = job["job_id"]
|
|
jd = rd / jid
|
|
jd.mkdir(exist_ok=True)
|
|
jp = jd / "job.json"
|
|
if jp.exists():
|
|
if _read(jp) != job:
|
|
raise ValueError(f"prepared job differs: {jp}")
|
|
else:
|
|
_write_json(jp, job, readonly=True)
|
|
if not (jd / "data").exists() and not (jd / "data").is_symlink():
|
|
(jd / "data").symlink_to(paths.DATA)
|
|
if entry["oracle"]:
|
|
(jd / "oracle" / "stages").mkdir(parents=True, exist_ok=True)
|
|
(jd / "oracle" / "artifacts").mkdir(exist_ok=True)
|
|
# Preparation can be slow on shared storage. Keep the collector lock,
|
|
# but acquire the SQLite writer lock only after all file work is done.
|
|
if _snapshot_files(source, rows) != files:
|
|
raise ValueError("source changed during apply; regenerate the plan")
|
|
con.execute("BEGIN IMMEDIATE")
|
|
if _sha({**_snapshot(con, source), **files}) != handoff["source_sha256"]:
|
|
raise ValueError("source changed during apply; regenerate the plan")
|
|
for entry in handoff["jobs"]:
|
|
job = entry["job"]
|
|
jid = job["job_id"]
|
|
if entry["oracle"]:
|
|
db.upsert_job(con, target, {**job, "mode": "oracle"}, entry["estimate_s"])
|
|
con.execute("UPDATE jobs SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND mode='oracle'", (source, jid))
|
|
for r in entry["rollouts"]:
|
|
db.rollout_upsert(con, target, jid, r["kk"], r["mode"], r["policy"], runner=r.get("runner", "pi"))
|
|
con.execute("UPDATE rollouts SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND kk=?", (source, jid, r["kk"]))
|
|
db.create_run(con, target, manifest, f"remaining work from {source}")
|
|
old = json.loads(db.get_run(con, source)["manifest"])
|
|
old["continued_to"] = target
|
|
con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(old), source))
|
|
con.commit() # release the collector lock only after the ownership change
|
|
return check(target)
|
|
|
|
|
|
def enqueue_repairs(run_id: str, handoff: dict, job_ids=None, modes=None) -> int:
|
|
"""历史交接回执兼容入口;旧 conditional slots 不再激活。"""
|
|
with db.connect() as con:
|
|
db.assert_run_open(con, run_id)
|
|
return 0
|
|
|
|
|
|
def check(run_id: str) -> dict:
|
|
handoff = receipt(run_id)
|
|
if not handoff:
|
|
raise ValueError("run has no continuation receipt")
|
|
if handoff["plan_id"] != _sha({k: v for k, v in handoff.items() if k != "plan_id"}):
|
|
raise ValueError("continuation receipt was edited")
|
|
for entry in handoff["jobs"]:
|
|
jp = paths.RUNS / run_id / entry["job"]["job_id"] / "job.json"
|
|
if not jp.exists() or _read(jp) != entry["job"]:
|
|
raise ValueError(f"handoff job snapshot missing or changed: {jp}")
|
|
expected_jobs = {j["job"]["job_id"] for j in handoff["jobs"] if j["oracle"]}
|
|
expected = {(j["job"]["job_id"], r["kk"], r["mode"], r["policy"], r.get("runner", "pi")) for j in handoff["jobs"] for r in j["rollouts"]}
|
|
conditional = {(j["job_id"], r["kk"], r["mode"], r["policy"], r.get("runner", "pi")) for j in handoff["conditional_repairs"] for r in j["rollouts"]}
|
|
with db.readonly() as con:
|
|
jobs = db.list_jobs(con, run_id, mode="oracle")
|
|
rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))]
|
|
for row in rows:
|
|
row["runner"] = db.rollout_runner(con, run_id, row["job_id"], row["kk"])
|
|
source = handoff["source_run_id"]
|
|
moved_jobs = {r[0] for r in con.execute("SELECT job_id FROM jobs WHERE run_id=? AND mode='oracle' AND state='TRANSFERRED'", (source,))}
|
|
moved = {(r[0], r[1], r[2], r[3], db.rollout_runner(con, source, r[0], r[1])) for r in con.execute("SELECT job_id,kk,mode,policy FROM rollouts WHERE run_id=? AND state='TRANSFERRED'", (source,))}
|
|
actual = {(r["job_id"], r["kk"], r["mode"], r["policy"], r["runner"]) for r in rows}
|
|
actual_jobs = {j["job_id"] for j in jobs}
|
|
if actual_jobs != expected_jobs or moved_jobs != expected_jobs or moved != expected or not expected <= actual or actual - expected - conditional:
|
|
raise ValueError("handoff identity mismatch: missing, additional, or untransferred tasks")
|
|
return {"run_id": run_id, "source_run_id": source, "plan_id": handoff["plan_id"], "identities_match": True,
|
|
"counts": handoff["counts"], "oracle_states": dict(Counter(j["state"] for j in jobs)),
|
|
"agent_states": dict(Counter(r["state"] for r in rows)), "activated_repairs": len(actual - expected)}
|