data-gen-bpu/scripts/test_continuation.py

658 lines
36 KiB
Python

"""Remaining-work handoff against real temporary SQLite and T1 packages."""
from contextlib import ExitStack, nullcontext
import copy
import fcntl
import hashlib
import io
import json
import os
from pathlib import Path
import tarfile
import tempfile
import unittest
from unittest.mock import patch
from harness import continuation, coverage, db, paths, pipeline, registry, rollout
class ContinuationTests(unittest.TestCase):
source = "old-batch"
target = "screen-100"
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("RUNS", "STATE", "DATA", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
database = paths.STATE / "jobs.sqlite"
connect, readonly = db.connect, db.readonly
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database)))
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
"e2e": {"enabled": True, "k": 4, "policies": ["teacher"]},
"repair": {"enabled": False}}}
self.stack.enter_context(patch.object(registry, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
self.stack.enter_context(patch.object(pipeline, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
self.real_fingerprint = registry.registry_fingerprint
self.stack.enter_context(patch.object(registry, "registry_fingerprint", return_value="frozen-registry"))
self.manifest = {"registry_task_fingerprint": "frozen-registry", "venv": {},
"evaluator_version": "0.5.0"}
self.stack.enter_context(patch.object(pipeline, "build_manifest", side_effect=lambda: copy.deepcopy(self.manifest)))
with db.connect() as con:
db.create_run(con, self.source, self.manifest)
self.eval_manifest = self.make_t1()
def collection_config(self):
# Validate the real collector contract without reading installation
# credentials or opening its shared quota ledger.
from harness.execution import PHASES, POLICY_FIELDS
policy = {field: 10 for field in POLICY_FIELDS}
return {"pools": {"agent": {"workers": 3}},
"rollouts": {"private_home": True},
"endpoints": {"teacher": {}, "student": {}},
"collection_v1": {"version": 1, "quota_ledger": str(self.root / "quota.sqlite"),
"quota_groups": {"fixture": {"max_concurrency": 1, "rpm": 10, "tpm": 1000}},
"credential_groups": {"teacher": "fixture"},
"lifecycle": {"version": 1, **policy,
"phases": {phase: dict(policy) for phase in PHASES}}}}
def make_t1(self):
directory = paths.DATA / "t1/imagenet1k/n100"
directory.mkdir(parents=True)
items = [{"id": str(i), "file": f"images/{i}.jpg", "label": i} for i in range(100)]
content_sha = hashlib.sha256(json.dumps(items).encode()).hexdigest()
inner = {"dataset": "imagenet1k", "n": 100, "seed": 17, "sha256": content_sha, "items": items}
package = directory / "17.tar"
with tarfile.open(package, "w") as archive:
for name, payload in [("manifest.json", json.dumps(inner).encode()),
*((item["file"], item["id"].encode()) for item in items)]:
member = tarfile.TarInfo(name)
member.size = len(payload)
archive.addfile(member, io.BytesIO(payload))
manifest = directory / "manifest.json"
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 100, "seed": 17, "tar": package.name,
"sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
"content_sha256": content_sha, "eval_set_id": "imagenet-fixed-100"}))
return manifest
def job(self, name="model-a", state="PENDING", error_class=None):
jid = f"cls--{name}--nash-m--ptq--int8"
job = {"job_id": jid, "run_id": self.source, "task": "cls", "model": name, "march": "nash-m",
"backend": "ptq", "qcfg": "int8", "priority": "A", "mode": "oracle", "policy": None,
"evaluator_version": "0.5.0", "model_spec": {"model_id": name, "task": "cls", "priority": "A"},
"data": {"dataset": "imagenet1k", "t1": {"path": "/old/2000.tar", "n": 2000,
"manifest_sha": "1" * 64, "content_sha": "2" * 64}},
"rollout_plans": [{"mode": "e2e", "policy": "teacher", "k": 4}]}
directory = paths.RUNS / self.source / jid
directory.mkdir(parents=True)
(directory / "job.json").write_text(json.dumps(job))
with db.connect() as con:
db.upsert_job(con, self.source, job, 10)
if state != "PENDING":
db.set_job_state(con, self.source, jid, state, error_class=error_class,
result={"terminal": state})
if state in ("PASS", "FAIL", "UNVERIFIED"):
(directory / "oracle/artifacts").mkdir(parents=True)
(directory / "oracle/artifacts/model.hbm").write_bytes(b"original HBM")
(directory / "oracle/result.json").write_text(json.dumps({"terminal": state, "n": 2000}))
if state == "PREPARED":
(directory / "oracle").mkdir()
(directory / "oracle/preparation.json").write_text(json.dumps({
"schema": "bench.preparation/1", "terminal": state, "n": 2000}))
return jid
def rollout(self, jid, kk, state="QUEUED", mode="e2e", policy="teacher"):
with db.connect() as con:
db.rollout_upsert(con, self.source, jid, kk, mode, policy, state)
if state != "QUEUED":
fields = {"started_at": 1.0}
if state not in ("RUNNING", "VERIFYING"):
fields.update(finished_at=2.0, verdict=json.dumps({"terminal": state}))
db.rollout_set(con, self.source, jid, kk, **fields)
def drain(self):
pipeline.drain_file(self.source).touch()
pipeline.collect_drain_file(self.source).touch()
def plan(self):
return continuation.plan(self.source, [str(self.eval_manifest)], self.target)
def file_snapshot(self):
return {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
def test_runner_identity_survives_plan_apply_and_check(self):
jid = self.job(state="PASS")
with db.connect() as con:
db.rollout_upsert(con, self.source, jid, 0, "e2e", "teacher", runner="pi")
db.rollout_upsert(con, self.source, jid, 1, "e2e", "teacher", runner="opencode")
planned = self.plan()
self.assertEqual([r["runner"] for r in planned["jobs"][0]["rollouts"]], ["pi", "opencode"])
self.assertEqual({p["runner"] for p in planned["jobs"][0]["job"]["rollout_plans"]}, {"pi", "opencode"})
self.drain()
with patch.object(pipeline, "scheduler_alive", return_value=[]):
self.assertTrue(continuation.apply(planned)["identities_match"])
with db.connect() as con:
self.assertEqual([r["runner"] for r in db.list_rollouts(con, self.target, jid)], ["pi", "opencode"])
con.execute("UPDATE rollout_context SET runner='pi' WHERE run_id=? AND kk=1", (self.target,))
with self.assertRaisesRegex(ValueError, "identity mismatch"):
continuation.check(self.target)
def test_runner_change_after_preview_invalidates_plan(self):
jid = self.job(state="PASS")
self.rollout(jid, 0)
planned = self.plan()
self.drain()
with db.connect() as con:
con.execute("UPDATE rollout_context SET runner='opencode' WHERE run_id=?", (self.source,))
with self.assertRaisesRegex(ValueError, "source changed"):
continuation.apply(planned)
def test_already_frozen_subset_continues_without_another_replacement(self):
jid = self.job()
self.rollout(jid, 0)
job_path = paths.RUNS / self.source / jid / "job.json"
job = json.loads(job_path.read_text())
_, frozen = continuation.load_eval_set(self.eval_manifest)
job["data"]["t1"] = frozen
job_path.write_text(json.dumps(job))
plan = continuation.plan(self.source, [], self.target)
self.assertEqual(plan["blockers"], [])
self.assertEqual(plan["jobs"][0]["job"]["data"]["t1"], frozen)
self.drain()
with patch.object(pipeline, "scheduler_alive", return_value=[]):
continuation.apply(plan)
check = continuation.check(self.target)
self.assertTrue(check["identities_match"], check)
def test_implicit_or_modified_subset_still_requires_explicit_identity(self):
jid = self.job()
self.rollout(jid, 0)
with self.assertRaisesRegex(ValueError, "missing replacement T1"):
continuation.plan(self.source, [], self.target)
job_path = paths.RUNS / self.source / jid / "job.json"
job = json.loads(job_path.read_text())
_, frozen = continuation.load_eval_set(self.eval_manifest)
job["data"]["t1"] = {**frozen, "content_sha": "0" * 64}
job_path.write_text(json.dumps(job))
with self.assertRaisesRegex(ValueError, "frozen explicit T1 changed"):
continuation.plan(self.source, [], self.target)
def test_preview_is_readonly_and_preserves_remaining_sampling_identity(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
before = self.file_snapshot()
plan = self.plan()
self.assertEqual(self.file_snapshot(), before)
self.assertEqual(plan["blockers"], [])
self.assertEqual(plan["counts"], {"oracle": 0, "agent_queued": 3, "agent_due_repairs": 0,
"conditional_repairs": 0})
self.assertEqual(len(plan["jobs"]), 1)
entry = plan["jobs"][0]
self.assertFalse(entry["oracle"])
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in entry["rollouts"]],
[(1, "e2e", "teacher"), (2, "e2e", "teacher"), (3, "e2e", "teacher")])
self.assertEqual(entry["job"]["oracle_ref"], {"run_id": self.source, "job_id": jid})
self.assertEqual(entry["job"]["data"]["t1"]["n"], 100)
def test_legacy_board_snapshot_allows_only_inventory_changes(self):
self.job()
directory = self.root / "registry"
directory.mkdir()
boards = b"version: 1\nmarch_by_board: {s100p: nash-m}\nboards: {one: {slot: 0}}\n"
(directory / "boards.yaml").write_bytes(boards)
(directory / "matrix.yaml").write_text("modes: {e2e: {k: 4}}\n")
(directory / "models.csv").write_text("model_id,task\nmodel-a,cls\n")
with patch.object(registry, "REGISTRY", directory), \
patch.object(registry, "registry_fingerprint", side_effect=self.real_fingerprint):
manifest = {"registry_fingerprint": registry.registry_fingerprint()}
with db.connect() as con:
con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(manifest), self.source))
initial = paths.RUNS / self.source / "board_registry.initial.yaml"
initial.write_bytes(boards)
expanded = boards.replace(b"one: {slot: 0}", b"one: {slot: 0}, two: {slot: 1}")
(directory / "boards.yaml").write_bytes(expanded)
self.assertNotEqual(registry.registry_fingerprint(), manifest["registry_fingerprint"])
self.assertEqual(self.plan()["blockers"], [])
# A saved board file cannot excuse changed k, model data or march rules.
for name, content in (("matrix.yaml", b"modes: {e2e: {k: 5}}\n"),
("models.csv", b"model_id,task\nmodel-a,det\n"),
("boards.yaml", expanded.replace(b"s100p: nash-m", b"s100p: nash-p"))):
with self.subTest(changed=name):
path = directory / name
original = path.read_bytes()
path.write_bytes(content)
with self.assertRaisesRegex(ValueError, "source registry changed"):
self.plan()
path.write_bytes(original)
initial.unlink()
with self.assertRaisesRegex(ValueError, "source registry changed"):
self.plan()
def test_prepared_source_continues_exact_slots_without_oracle_result_and_exports(self):
from export import to_agentrl
from harness import verify
jid = self.job(state="PREPARED")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2}
jp = paths.RUNS / self.source / jid / "job.json"
job = json.loads(jp.read_text())
job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}}
jp.write_text(json.dumps(job))
handoff = self.plan()
self.assertEqual(handoff["blockers"], [])
self.assertEqual(handoff["counts"], {"oracle": 0, "agent_queued": 3,
"agent_due_repairs": 0, "conditional_repairs": 0})
self.drain()
self.assertTrue(continuation.apply(handoff)["identities_match"])
jd = paths.RUNS / self.target / jid
ready, pending = pipeline._collection_snapshot(self.target, None, None)
self.assertFalse(pending)
self.assertEqual([(r["kk"], r["job_state"], r["completed"], r["started"]) for r in ready],
[(kk, "PREPARED", 1, 1) for kk in (1, 2, 3)])
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
rd = jd / "rollouts/01"
(rd / "agent").mkdir(parents=True)
for name, value in {
"rollout.json": {"state": "DONE", "mode": "e2e", "policy": "teacher", "agent_config_version": 1},
"agent/events.jsonl": {"type": "agent_end", "messages": [
{"role": "user", "content": "Synthetic test task."},
{"role": "assistant", "content": "Synthetic test response."}]},
"agent/tools.json": [{"name": "bash"}],
}.items():
(rd / name).write_text(json.dumps(value))
(rd / "agent/system.md").write_text("Synthetic test system.")
verdict = verify.verify_rollout(rd)
self.assertNotIn("oracle_ref", verdict)
with patch.object(to_agentrl, "BENCH", self.root), \
patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}):
exported = to_agentrl.materialize([self.target], "prepared-test", min_agent_version=1)
self.assertEqual(exported["stats"]["agent_written"], 1)
self.assertEqual(exported["stats"]["pipeline_written"], 0)
record = json.loads((self.root / "datasets/prepared-test/teacher.jsonl").read_text())
# Continuation freezes the requested T1; preparation alone supplies no
# measured evaluation identity and must not manufacture SFT success.
self.assertIsNone(record["eval_set"])
self.assertEqual(record["required_eval_set"]["n"], 100)
self.assertEqual(record["required_eval_set"]["eval_set_id"], "imagenet-fixed-100")
self.assertFalse(record["sft_eligible"])
self.assertEqual((self.root / "datasets/prepared-test/sft.jsonl").read_text(), "")
self.assertEqual(record["continuation"], {"source_run_id": self.source,
"source_job_id": jid, "source_rollout": 1})
self.assertNotIn("oracle_ref", record)
self.assertFalse((jp.parent / "oracle/result.json").exists())
self.assertFalse((jd / "oracle").exists())
with db.readonly() as con:
self.assertIsNone(db.get_job(con, self.target, jid))
def test_continuation_needs_no_oracle_evidence_and_never_releases_repair(self):
jid = self.job(state="PREPARED")
self.rollout(jid, 1, mode="repair")
evidence = paths.RUNS / self.source / jid / "oracle/preparation.json"
evidence.unlink()
self.assertEqual(self.plan()["blockers"], [])
evidence.write_text(json.dumps({"terminal": "FAIL"}))
self.assertEqual(self.plan()["blockers"], [])
evidence.write_text(json.dumps({"terminal": "PREPARED"}))
self.drain()
continuation.apply(self.plan())
ready, pending = pipeline._collection_snapshot(self.target, None, None)
self.assertEqual(ready, [])
self.assertFalse(pending)
def test_first_pass_carries_source_attempts_for_prepared_and_pending_jobs(self):
jobs = {}
for name, state, history in (("fresh", "PREPARED", None), ("covered", "PREPARED", "PASS"),
("failed", "PREPARED", "UNVERIFIED"), ("pending", "PENDING", "FAIL")):
jid = jobs[name] = self.job(name, state=state)
if history:
self.rollout(jid, 0, history)
if history == "UNVERIFIED":
with db.connect() as con:
db.rollout_set(con, self.source, jid, 0, verdict=None)
for kk in (1, 2, 3):
self.rollout(jid, kk)
self.drain()
continuation.apply(self.plan())
started = []
def complete(run_id, job_id, kk, *args, **kwargs):
started.append((job_id, kk))
with db.connect() as con:
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3.0)
return {"terminal": "PASS"}
ready, _ = pipeline._collection_snapshot(self.target, None, None)
self.assertEqual([r["job_id"] for r in ready[:3]], [jobs["fresh"]] * 3)
self.assertEqual({r["job_id"] for r in ready if not r["started"] and not r["completed"]},
{jobs["fresh"]})
# The collector's actual first-pass filter starts only the fresh model.
with patch.object(rollout, "run_rollout", side_effect=complete), \
patch.object(rollout, "setup_models_json"), \
patch.object(pipeline, "refresh_run_dataset"), \
patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \
patch.object(paths, "config", side_effect=self.collection_config), \
patch("harness.collection_runtime.preflight"):
result = pipeline.collect_rollouts(self.target, parallel=1, first_pass=True, log=lambda _: None)
self.assertEqual(result, {"PASS": 1})
self.assertEqual(started, [(jobs["fresh"], 1)])
def test_first_pass_ignores_source_repair_but_keeps_source_e2e_history(self):
repair_only = self.job("repair-only", state="PREPARED")
e2e_covered = self.job("e2e-covered", state="PREPARED")
self.rollout(repair_only, 7, "PASS", mode="repair")
self.rollout(e2e_covered, 7, "PASS", mode="e2e")
for jid in (repair_only, e2e_covered):
self.rollout(jid, 8)
self.rollout(jid, 9)
with db.readonly() as con:
before = db.list_rollouts(con, self.source, repair_only)[0]
self.drain()
continuation.apply(self.plan())
started = []
def complete(run_id, job_id, kk, *args, **kwargs):
started.append((job_id, kk))
with db.connect() as con:
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3)
return {"terminal": "PASS"}
with patch.object(rollout, "run_rollout", side_effect=complete), \
patch.object(rollout, "setup_models_json"), \
patch.object(pipeline, "refresh_run_dataset"), \
patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \
patch.object(paths, "config", side_effect=self.collection_config), \
patch("harness.collection_runtime.preflight"):
result = pipeline.collect_rollouts(self.target, parallel=1, modes=["e2e"], first_pass=True, log=lambda _: None)
self.assertEqual(result, {"PASS": 1})
self.assertEqual(started, [(repair_only, 8)])
with db.readonly() as con:
self.assertEqual(db.list_rollouts(con, self.source, repair_only)[0], before)
self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, repair_only)], ["PASS", "QUEUED"])
self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, e2e_covered)], ["QUEUED", "QUEUED"])
def test_apply_agent_only_preserves_history_and_repeated_apply_is_idempotent(self):
jid = self.job(state="FAIL", error_class="compile_fail")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
old_directory = paths.RUNS / self.source / jid
before = {p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}
with db.readonly() as con:
old_job = db.get_job(con, self.source, jid)
old_finished = db.list_rollouts(con, self.source, jid)[0]
self.drain()
handoff = self.plan()
report = continuation.apply(handoff)
self.assertTrue(report["identities_match"])
self.assertEqual(report["agent_states"], {"QUEUED": 3})
self.assertEqual(continuation.apply(handoff), report)
with db.readonly() as con:
self.assertEqual(db.list_jobs(con, self.target), [])
self.assertEqual(db.get_job(con, self.source, jid), old_job)
original = db.list_rollouts(con, self.source, jid)
self.assertEqual(original[0], old_finished)
self.assertEqual([r["state"] for r in original[1:]], ["TRANSFERRED"] * 3)
moved = db.list_rollouts(con, self.target, jid)
self.assertEqual([r["kk"] for r in moved], [1, 2, 3])
self.assertEqual({p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}, before)
ready, pending = pipeline._collection_snapshot(self.target, None, None)
self.assertEqual([r["kk"] for r in ready], [1, 2, 3])
self.assertFalse(pending)
def test_apply_requires_both_drains_and_stopped_schedulers(self):
self.job()
handoff = self.plan()
for mark in (None, pipeline.drain_file(self.source)):
if mark:
mark.touch()
with self.assertRaisesRegex(ValueError, "drain both"):
continuation.apply(handoff)
self.assertFalse((paths.RUNS / self.target).exists())
self.drain()
with patch.object(pipeline, "scheduler_alive", return_value=True):
with self.assertRaisesRegex(ValueError, "scheduler still running"):
continuation.apply(handoff)
lock = paths.STATE / "locks" / f"collect-{self.source}.lock"
lock.parent.mkdir(exist_ok=True)
with lock.open("a") as holder:
fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB)
with self.assertRaisesRegex(ValueError, "collector still running"):
continuation.apply(handoff)
self.assertTrue(continuation.apply(handoff)["identities_match"])
def test_active_work_and_requeued_history_block_handoff(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "RUNNING")
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
self.assertTrue(any("still active" in b for b in handoff["blockers"]))
with self.assertRaisesRegex(ValueError, "blocked"):
continuation.apply(handoff)
with db.connect() as con:
db.rollout_set(con, self.source, jid, 0, state="QUEUED")
self.assertTrue(any("execution history" in b for b in self.plan()["blockers"]))
def test_stale_or_edited_plan_is_rejected_without_creating_target(self):
jid = self.job(state="PASS")
self.rollout(jid, 0)
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
modified = copy.deepcopy(handoff)
modified["jobs"][0]["rollouts"].pop()
with self.assertRaisesRegex(ValueError, "edited"):
continuation.apply(modified)
with db.connect() as con:
db.rollout_set(con, self.source, jid, 0, state="FAIL", started_at=1.0, finished_at=2.0)
with self.assertRaisesRegex(ValueError, "source changed"):
continuation.apply(handoff)
self.assertFalse((paths.RUNS / self.target).exists())
def test_no_due_or_conditional_repairs_even_with_legacy_config(self):
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
"policies": ["teacher", "student"], "error_classes": ["compile_fail"]}
jid = self.job()
self.rollout(jid, 0, "FAIL")
self.rollout(jid, 1)
failed = self.job("failed", state="FAIL", error_class="compile_fail")
self.rollout(failed, 4, "FAIL", mode="repair")
self.rollout(failed, 7, mode="repair")
# Neither missing nor corrupt historical Oracle evidence blocks E2E handoff.
(paths.RUNS / self.source / failed / "oracle/result.json").write_text("corrupt JSON")
self.drain()
handoff = self.plan()
self.assertEqual(handoff["counts"]["conditional_repairs"], 0)
self.assertEqual(handoff["counts"]["agent_due_repairs"], 0)
self.assertEqual(handoff["counts"]["agent_queued"], 2)
self.assertEqual(handoff["conditional_repairs"], [])
continuation.apply(handoff)
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
self.assertEqual(continuation.check(self.target)["activated_repairs"], 0)
with db.readonly() as con:
moved = db.list_rollouts(con, self.target, failed)
original = db.list_rollouts(con, self.source, failed)
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in moved],
[(7, "repair", "teacher", "QUEUED")])
self.assertEqual([(r["kk"], r["state"]) for r in original], [(4, "FAIL"), (7, "TRANSFERRED")])
ready, _ = pipeline._collection_snapshot(self.target, None, None)
self.assertEqual([(r["job_id"], r["kk"]) for r in ready], [(jid, 1)])
def test_legacy_conditional_receipt_is_checked_but_never_activated(self):
jid = self.job()
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
continuation.apply(handoff)
# Model a receipt written by the previous implementation, including a
# repair which may already have been activated before the upgrade.
handoff["conditional_repairs"] = [{"job_id": jid, "error_classes": ["compile_fail"], "rollouts": [
{"kk": 2, "mode": "repair", "policy": "teacher", "source_exists": False},
{"kk": 3, "mode": "repair", "policy": "teacher", "source_exists": False}]}]
handoff["counts"]["conditional_repairs"] = 2
handoff["plan_id"] = continuation._sha({k: v for k, v in handoff.items() if k != "plan_id"})
(paths.RUNS / self.target / "continuation.json").write_text(json.dumps(handoff))
with db.connect() as con:
db.set_job_state(con, self.target, jid, "FAIL", error_class="compile_fail")
db.rollout_upsert(con, self.target, jid, 2, "repair", "teacher")
self.assertEqual(continuation.enqueue_repairs(self.target, handoff), 0)
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
checked = continuation.check(self.target)
self.assertTrue(checked["identities_match"])
self.assertEqual(checked["activated_repairs"], 1)
ready, _ = pipeline._collection_snapshot(self.target, None, None)
self.assertEqual([(r["kk"], r["mode"]) for r in ready], [(1, "e2e")])
with db.readonly() as con:
self.assertEqual([(r["kk"], r["mode"]) for r in db.list_rollouts(con, self.target, jid)],
[(1, "e2e"), (2, "repair")])
def test_agent_only_coverage_counts_queue_and_exports_without_fake_oracle(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
self.drain()
continuation.apply(self.plan())
with db.connect() as con:
db.rollout_set(con, self.target, jid, 1, state="PASS", started_at=3.0, finished_at=4.0)
models = [{"model_id": "model-a", "task": "cls", "priority": "A", "status": "ready"}]
with patch.object(coverage, "models", return_value=models), \
patch.object(coverage, "exported_keys", return_value={(self.target, jid, 1), (self.source, jid, 0)}):
actual = coverage.collect(self.target)["model-a"]
self.assertEqual(actual["jobs"], [])
self.assertEqual(dict(actual["oracle"]), {})
self.assertEqual(actual["planned_rollouts"], 3)
self.assertEqual(dict(actual["rollout"]), {"PASS": 1, "QUEUED": 2})
self.assertEqual(actual["exported"], 1)
self.assertTrue(continuation.check(self.target)["identities_match"])
def test_interrupted_filesystem_preparation_retries_without_losing_old_queue(self):
jid = self.job()
self.rollout(jid, 0)
self.drain()
handoff = self.plan()
with patch.object(db, "create_run", side_effect=RuntimeError("injected failure before commit")):
with self.assertRaisesRegex(RuntimeError, "injected failure"):
continuation.apply(handoff)
self.assertEqual(continuation.receipt(self.target), handoff)
self.assertTrue((paths.RUNS / self.target / jid / "job.json").exists())
with db.readonly() as con:
self.assertIsNone(db.get_run(con, self.target))
self.assertEqual(db.list_jobs(con, self.target), [])
self.assertEqual(db.list_rollouts(con, self.target, jid), [])
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
self.assertEqual(db.list_rollouts(con, self.source, jid)[0]["state"], "QUEUED")
self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"]))
report = continuation.apply(handoff)
self.assertTrue(report["identities_match"])
self.assertEqual(report["oracle_states"], {"PENDING": 1})
self.assertEqual(report["agent_states"], {"QUEUED": 1})
def test_interrupted_readonly_job_temp_file_can_be_replaced_on_retry(self):
jid = self.job()
self.drain()
handoff = self.plan()
replace = Path.replace
def interrupt_job_replace(source, target):
if source.name == ".job.json.tmp":
raise OSError("interrupted after chmod before replace")
return replace(source, target)
with patch.object(Path, "replace", autospec=True, side_effect=interrupt_job_replace):
with self.assertRaisesRegex(OSError, "after chmod"):
continuation.apply(handoff)
temporary = paths.RUNS / self.target / jid / ".job.json.tmp"
self.assertEqual(temporary.stat().st_mode & 0o777, 0o444)
self.assertFalse(temporary.with_name("job.json").exists())
with db.readonly() as con:
self.assertIsNone(db.get_run(con, self.target))
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
self.assertTrue(continuation.apply(handoff)["identities_match"])
self.assertFalse(temporary.exists())
self.assertEqual(temporary.with_name("job.json").stat().st_mode & 0o777, 0o444)
def test_orphan_queued_rollout_blocks_plan_instead_of_being_omitted(self):
jid = self.job(state="PASS")
self.rollout(jid, 3)
with db.connect() as con:
con.execute("DELETE FROM jobs WHERE run_id=? AND job_id=?", (self.source, jid))
handoff = self.plan()
self.assertTrue(any("no source Oracle" in item for item in handoff["blockers"]))
self.drain()
with self.assertRaisesRegex(ValueError, "blocked"):
continuation.apply(handoff)
def test_direct_legacy_rollout_claims_before_preparation_and_closes_failed_attempt(self):
jid = self.job(state="PASS")
for kk, error in ((4, SystemExit("missing preparation inputs")), (5, RuntimeError("preparation failed"))):
with self.subTest(error=type(error).__name__):
def fail_preparation(*args, **kwargs):
with db.readonly() as con:
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
self.assertEqual(current["state"], "RUNNING")
self.assertIsNotNone(current["started_at"])
self.assertTrue(any("still active" in item for item in self.plan()["blockers"]))
raise error
with patch.object(rollout, "_execute_rollout", side_effect=fail_preparation) as execute:
with self.assertRaises(type(error)):
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
with self.assertRaisesRegex(ValueError, "claimed or finished"):
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
self.assertEqual(execute.call_count, 1)
with db.readonly() as con:
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
self.assertEqual(current["state"], "UNVERIFIED")
self.assertIsNotNone(current["finished_at"])
def test_closed_source_and_target_full_expansion_are_rejected(self):
jid = self.job()
self.rollout(jid, 0)
self.drain()
continuation.apply(self.plan())
source_calls = [lambda: pipeline.expand(self.source), lambda: pipeline.enqueue_repairs(self.source),
lambda: continuation.enqueue_repairs(self.source, {}),
lambda: pipeline.enqueue_e2e_coverage(self.source),
lambda: pipeline.collect_rollouts(self.source, dry_run=True),
lambda: pipeline.run_oracle(self.source, dry_run=True)]
for call in source_calls:
with self.assertRaisesRegex(ValueError, "execution is closed"):
call()
with db.connect() as con:
with self.assertRaisesRegex(ValueError, "execution is closed"):
db.claim_job(con, self.source, jid, force=True)
with db.connect() as con:
with self.assertRaisesRegex(ValueError, "execution is closed"):
db.claim_rollout(con, self.source, jid, 0)
for call in (lambda: pipeline.expand(self.target), lambda: pipeline.enqueue_e2e_coverage(self.target),
lambda: pipeline.collect_rollouts(self.target, coverage="all", dry_run=True)):
with self.assertRaisesRegex(ValueError, "continuation run"):
call()
def test_identity_check_catches_substitution_even_when_counts_match(self):
jid = self.job(state="PASS")
self.rollout(jid, 1)
self.rollout(jid, 2)
self.drain()
continuation.apply(self.plan())
with db.connect() as con:
con.execute("UPDATE rollouts SET kk=3 WHERE run_id=? AND job_id=? AND kk=2", (self.target, jid))
with self.assertRaisesRegex(ValueError, "identity mismatch"):
continuation.check(self.target)
if __name__ == "__main__":
unittest.main()