data-gen-bpu/scripts/test_continuation.py

393 lines
21 KiB
Python

"""Remaining-work handoff against real temporary SQLite and T1 packages."""
from contextlib import ExitStack
import copy
import fcntl
import hashlib
import io
import json
from pathlib import Path
import tarfile
import tempfile
import unittest
from unittest.mock import patch
from harness import continuation, coverage, db, paths, pipeline, registry, rollout
class ContinuationTests(unittest.TestCase):
source = "old-batch"
target = "screen-100"
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("RUNS", "STATE", "DATA", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
database = paths.STATE / "jobs.sqlite"
connect, readonly = db.connect, db.readonly
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database)))
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
"e2e": {"enabled": True, "k": 4, "policies": ["teacher"]},
"repair": {"enabled": False}}}
self.stack.enter_context(patch.object(registry, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
self.stack.enter_context(patch.object(pipeline, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
self.stack.enter_context(patch.object(registry, "registry_fingerprint", return_value="frozen-registry"))
self.manifest = {"registry_task_fingerprint": "frozen-registry", "venv": {},
"evaluator_version": "0.5.0"}
self.stack.enter_context(patch.object(pipeline, "build_manifest", side_effect=lambda: copy.deepcopy(self.manifest)))
with db.connect() as con:
db.create_run(con, self.source, self.manifest)
self.eval_manifest = self.make_t1()
def make_t1(self):
directory = paths.DATA / "t1/imagenet1k/n100"
directory.mkdir(parents=True)
items = [{"id": str(i), "file": f"images/{i}.jpg", "label": i} for i in range(100)]
content_sha = hashlib.sha256(json.dumps(items).encode()).hexdigest()
inner = {"dataset": "imagenet1k", "n": 100, "seed": 17, "sha256": content_sha, "items": items}
package = directory / "17.tar"
with tarfile.open(package, "w") as archive:
for name, payload in [("manifest.json", json.dumps(inner).encode()),
*((item["file"], item["id"].encode()) for item in items)]:
member = tarfile.TarInfo(name)
member.size = len(payload)
archive.addfile(member, io.BytesIO(payload))
manifest = directory / "manifest.json"
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 100, "seed": 17, "tar": package.name,
"sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
"content_sha256": content_sha, "eval_set_id": "imagenet-fixed-100"}))
return manifest
def job(self, name="model-a", state="PENDING", error_class=None):
jid = f"cls--{name}--nash-m--ptq--int8"
job = {"job_id": jid, "run_id": self.source, "task": "cls", "model": name, "march": "nash-m",
"backend": "ptq", "qcfg": "int8", "priority": "A", "mode": "oracle", "policy": None,
"evaluator_version": "0.5.0", "model_spec": {"model_id": name, "task": "cls", "priority": "A"},
"data": {"dataset": "imagenet1k", "t1": {"path": "/old/2000.tar", "n": 2000,
"manifest_sha": "1" * 64, "content_sha": "2" * 64}},
"rollout_plans": [{"mode": "e2e", "policy": "teacher", "k": 4}]}
directory = paths.RUNS / self.source / jid
directory.mkdir(parents=True)
(directory / "job.json").write_text(json.dumps(job))
with db.connect() as con:
db.upsert_job(con, self.source, job, 10)
if state != "PENDING":
db.set_job_state(con, self.source, jid, state, error_class=error_class,
result={"terminal": state})
if state in ("PASS", "FAIL", "UNVERIFIED"):
(directory / "oracle/artifacts").mkdir(parents=True)
(directory / "oracle/artifacts/model.hbm").write_bytes(b"original HBM")
(directory / "oracle/result.json").write_text(json.dumps({"terminal": state, "n": 2000}))
return jid
def rollout(self, jid, kk, state="QUEUED", mode="e2e", policy="teacher"):
with db.connect() as con:
db.rollout_upsert(con, self.source, jid, kk, mode, policy, state)
if state != "QUEUED":
fields = {"started_at": 1.0}
if state not in ("RUNNING", "VERIFYING"):
fields.update(finished_at=2.0, verdict=json.dumps({"terminal": state}))
db.rollout_set(con, self.source, jid, kk, **fields)
def drain(self):
pipeline.drain_file(self.source).touch()
pipeline.collect_drain_file(self.source).touch()
def plan(self):
return continuation.plan(self.source, [str(self.eval_manifest)], self.target)
def file_snapshot(self):
return {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
def test_preview_is_readonly_and_preserves_remaining_sampling_identity(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
before = self.file_snapshot()
plan = self.plan()
self.assertEqual(self.file_snapshot(), before)
self.assertEqual(plan["blockers"], [])
self.assertEqual(plan["counts"], {"oracle": 0, "agent_queued": 3, "agent_due_repairs": 0,
"conditional_repairs": 0})
self.assertEqual(len(plan["jobs"]), 1)
entry = plan["jobs"][0]
self.assertFalse(entry["oracle"])
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in entry["rollouts"]],
[(1, "e2e", "teacher"), (2, "e2e", "teacher"), (3, "e2e", "teacher")])
self.assertEqual(entry["job"]["oracle_ref"], {"run_id": self.source, "job_id": jid})
self.assertEqual(entry["job"]["data"]["t1"]["n"], 100)
def test_apply_agent_only_preserves_history_and_repeated_apply_is_idempotent(self):
jid = self.job(state="FAIL", error_class="compile_fail")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
old_directory = paths.RUNS / self.source / jid
before = {p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}
with db.readonly() as con:
old_job = db.get_job(con, self.source, jid)
old_finished = db.list_rollouts(con, self.source, jid)[0]
self.drain()
handoff = self.plan()
report = continuation.apply(handoff)
self.assertTrue(report["identities_match"])
self.assertEqual(report["agent_states"], {"QUEUED": 3})
self.assertEqual(continuation.apply(handoff), report)
with db.readonly() as con:
self.assertEqual(db.list_jobs(con, self.target), [])
self.assertEqual(db.get_job(con, self.source, jid), old_job)
original = db.list_rollouts(con, self.source, jid)
self.assertEqual(original[0], old_finished)
self.assertEqual([r["state"] for r in original[1:]], ["TRANSFERRED"] * 3)
moved = db.list_rollouts(con, self.target, jid)
self.assertEqual([r["kk"] for r in moved], [1, 2, 3])
self.assertEqual({p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}, before)
with patch.object(rollout, "inputs_ready", return_value=True):
ready, pending = pipeline._collection_snapshot(self.target, None, None, set())
self.assertEqual([r["kk"] for r in ready], [1, 2, 3])
self.assertFalse(pending)
def test_apply_requires_both_drains_and_stopped_schedulers(self):
self.job()
handoff = self.plan()
for mark in (None, pipeline.drain_file(self.source)):
if mark:
mark.touch()
with self.assertRaisesRegex(ValueError, "drain both"):
continuation.apply(handoff)
self.assertFalse((paths.RUNS / self.target).exists())
self.drain()
with patch.object(pipeline, "scheduler_alive", return_value=True):
with self.assertRaisesRegex(ValueError, "scheduler still running"):
continuation.apply(handoff)
lock = paths.STATE / "locks" / f"collect-{self.source}.lock"
lock.parent.mkdir(exist_ok=True)
with lock.open("a") as holder:
fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB)
with self.assertRaisesRegex(ValueError, "collector still running"):
continuation.apply(handoff)
self.assertTrue(continuation.apply(handoff)["identities_match"])
def test_active_work_and_requeued_history_block_handoff(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "RUNNING")
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
self.assertTrue(any("still active" in b for b in handoff["blockers"]))
with self.assertRaisesRegex(ValueError, "blocked"):
continuation.apply(handoff)
with db.connect() as con:
db.rollout_set(con, self.source, jid, 0, state="QUEUED")
self.assertTrue(any("execution history" in b for b in self.plan()["blockers"]))
def test_stale_or_edited_plan_is_rejected_without_creating_target(self):
jid = self.job(state="PASS")
self.rollout(jid, 0)
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
modified = copy.deepcopy(handoff)
modified["jobs"][0]["rollouts"].pop()
with self.assertRaisesRegex(ValueError, "edited"):
continuation.apply(modified)
with db.connect() as con:
db.rollout_set(con, self.source, jid, 0, state="FAIL", started_at=1.0, finished_at=2.0)
with self.assertRaisesRegex(ValueError, "source changed"):
continuation.apply(handoff)
self.assertFalse((paths.RUNS / self.target).exists())
def test_conditional_repairs_keep_reserved_slots_and_frozen_policy(self):
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
"policies": ["teacher", "student"], "error_classes": ["compile_fail"]}
jid = self.job()
self.rollout(jid, 0, "FAIL")
self.rollout(jid, 1)
self.drain()
handoff = self.plan()
self.assertEqual(handoff["counts"]["conditional_repairs"], 4)
slots = handoff["conditional_repairs"][0]["rollouts"]
self.assertEqual([(r["kk"], r["policy"]) for r in slots],
[(2, "teacher"), (3, "teacher"), (4, "student"), (5, "student")])
continuation.apply(handoff)
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
with db.connect() as con:
db.set_job_state(con, self.target, jid, "FAIL", error_class="unrelated")
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
with db.connect() as con:
db.set_job_state(con, self.target, jid, "FAIL", error_class="compile_fail")
# Later registry edits cannot expand the frozen handoff's quota or policy.
self.mx["modes"]["repair"].update(k=50, policies=["another-policy"])
self.assertEqual(pipeline.enqueue_repairs(self.target, modes=["e2e"]), 0)
self.assertEqual(pipeline.enqueue_repairs(self.target), 4)
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
self.assertEqual(continuation.check(self.target)["activated_repairs"], 4)
with db.readonly() as con:
rows = db.list_rollouts(con, self.target, jid)
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in rows],
[(1, "e2e", "teacher"), (2, "repair", "teacher"), (3, "repair", "teacher"),
(4, "repair", "student"), (5, "repair", "student")])
def test_existing_failed_repairs_count_toward_original_quota(self):
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
"policies": ["teacher"]}
jid = self.job(state="FAIL", error_class="compile_fail")
self.rollout(jid, 7, "FAIL", mode="repair")
handoff = self.plan()
self.assertEqual(handoff["counts"]["agent_due_repairs"], 1)
self.assertEqual(handoff["jobs"][0]["rollouts"],
[{"kk": 8, "mode": "repair", "policy": "teacher", "source_exists": False}])
self.drain()
self.assertTrue(continuation.apply(handoff)["identities_match"])
with db.readonly() as con:
original = db.list_rollouts(con, self.source, jid)
self.assertEqual([(r["kk"], r["state"]) for r in original], [(7, "FAIL"), (8, "TRANSFERRED")])
moved = db.list_rollouts(con, self.target, jid)
self.assertEqual([(r["kk"], r["state"]) for r in moved], [(8, "QUEUED")])
def test_agent_only_coverage_counts_queue_and_exports_without_fake_oracle(self):
jid = self.job(state="PASS")
self.rollout(jid, 0, "FAIL")
for kk in (1, 2, 3):
self.rollout(jid, kk)
self.drain()
continuation.apply(self.plan())
with db.connect() as con:
db.rollout_set(con, self.target, jid, 1, state="PASS", started_at=3.0, finished_at=4.0)
models = [{"model_id": "model-a", "task": "cls", "priority": "A", "status": "ready"}]
with patch.object(coverage, "models", return_value=models), \
patch.object(coverage, "exported_keys", return_value={(self.target, jid, 1), (self.source, jid, 0)}):
actual = coverage.collect(self.target)["model-a"]
self.assertEqual(actual["jobs"], [])
self.assertEqual(dict(actual["oracle"]), {})
self.assertEqual(actual["planned_rollouts"], 3)
self.assertEqual(dict(actual["rollout"]), {"PASS": 1, "QUEUED": 2})
self.assertEqual(actual["exported"], 1)
self.assertTrue(continuation.check(self.target)["identities_match"])
def test_interrupted_filesystem_preparation_retries_without_losing_old_queue(self):
jid = self.job()
self.rollout(jid, 0)
self.drain()
handoff = self.plan()
with patch.object(db, "create_run", side_effect=RuntimeError("injected failure before commit")):
with self.assertRaisesRegex(RuntimeError, "injected failure"):
continuation.apply(handoff)
self.assertEqual(continuation.receipt(self.target), handoff)
self.assertTrue((paths.RUNS / self.target / jid / "job.json").exists())
with db.readonly() as con:
self.assertIsNone(db.get_run(con, self.target))
self.assertEqual(db.list_jobs(con, self.target), [])
self.assertEqual(db.list_rollouts(con, self.target, jid), [])
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
self.assertEqual(db.list_rollouts(con, self.source, jid)[0]["state"], "QUEUED")
self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"]))
report = continuation.apply(handoff)
self.assertTrue(report["identities_match"])
self.assertEqual(report["oracle_states"], {"PENDING": 1})
self.assertEqual(report["agent_states"], {"QUEUED": 1})
def test_interrupted_readonly_job_temp_file_can_be_replaced_on_retry(self):
jid = self.job()
self.drain()
handoff = self.plan()
replace = Path.replace
def interrupt_job_replace(source, target):
if source.name == ".job.json.tmp":
raise OSError("interrupted after chmod before replace")
return replace(source, target)
with patch.object(Path, "replace", autospec=True, side_effect=interrupt_job_replace):
with self.assertRaisesRegex(OSError, "after chmod"):
continuation.apply(handoff)
temporary = paths.RUNS / self.target / jid / ".job.json.tmp"
self.assertEqual(temporary.stat().st_mode & 0o777, 0o444)
self.assertFalse(temporary.with_name("job.json").exists())
with db.readonly() as con:
self.assertIsNone(db.get_run(con, self.target))
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
self.assertTrue(continuation.apply(handoff)["identities_match"])
self.assertFalse(temporary.exists())
self.assertEqual(temporary.with_name("job.json").stat().st_mode & 0o777, 0o444)
def test_orphan_queued_rollout_blocks_plan_instead_of_being_omitted(self):
jid = self.job(state="PASS")
self.rollout(jid, 3)
with db.connect() as con:
con.execute("DELETE FROM jobs WHERE run_id=? AND job_id=?", (self.source, jid))
handoff = self.plan()
self.assertTrue(any("no source Oracle" in item for item in handoff["blockers"]))
self.drain()
with self.assertRaisesRegex(ValueError, "blocked"):
continuation.apply(handoff)
def test_direct_legacy_rollout_claims_before_preparation_and_closes_failed_attempt(self):
jid = self.job(state="PASS")
for kk, error in ((4, SystemExit("missing preparation inputs")), (5, RuntimeError("preparation failed"))):
with self.subTest(error=type(error).__name__):
def fail_preparation(*args):
with db.readonly() as con:
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
self.assertEqual(current["state"], "RUNNING")
self.assertIsNotNone(current["started_at"])
self.assertTrue(any("still active" in item for item in self.plan()["blockers"]))
raise error
with patch.object(rollout, "_execute_rollout", side_effect=fail_preparation) as execute:
with self.assertRaises(type(error)):
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
with self.assertRaisesRegex(ValueError, "claimed or finished"):
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
self.assertEqual(execute.call_count, 1)
with db.readonly() as con:
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
self.assertEqual(current["state"], "UNVERIFIED")
self.assertIsNotNone(current["finished_at"])
def test_closed_source_and_target_full_expansion_are_rejected(self):
jid = self.job()
self.rollout(jid, 0)
self.drain()
continuation.apply(self.plan())
source_calls = [lambda: pipeline.expand(self.source), lambda: pipeline.enqueue_repairs(self.source),
lambda: pipeline.enqueue_e2e_coverage(self.source),
lambda: pipeline.collect_rollouts(self.source, dry_run=True),
lambda: pipeline.run_oracle(self.source, dry_run=True)]
for call in source_calls:
with self.assertRaisesRegex(ValueError, "execution is closed"):
call()
with db.connect() as con:
with self.assertRaisesRegex(ValueError, "execution is closed"):
db.claim_job(con, self.source, jid, force=True)
with db.connect() as con:
with self.assertRaisesRegex(ValueError, "execution is closed"):
db.claim_rollout(con, self.source, jid, 0)
for call in (lambda: pipeline.expand(self.target), lambda: pipeline.enqueue_e2e_coverage(self.target),
lambda: pipeline.collect_rollouts(self.target, coverage="all", dry_run=True)):
with self.assertRaisesRegex(ValueError, "continuation run"):
call()
def test_identity_check_catches_substitution_even_when_counts_match(self):
jid = self.job(state="PASS")
self.rollout(jid, 1)
self.rollout(jid, 2)
self.drain()
continuation.apply(self.plan())
with db.connect() as con:
con.execute("UPDATE rollouts SET kk=3 WHERE run_id=? AND job_id=? AND kk=2", (self.target, jid))
with self.assertRaisesRegex(ValueError, "identity mismatch"):
continuation.check(self.target)
if __name__ == "__main__":
unittest.main()