539 lines
29 KiB
Python
539 lines
29 KiB
Python
"""Remaining-work handoff against real temporary SQLite and T1 packages."""
|
|
from contextlib import ExitStack, nullcontext
|
|
import copy
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import continuation, coverage, db, paths, pipeline, registry, rollout
|
|
|
|
|
|
class ContinuationTests(unittest.TestCase):
|
|
source = "old-batch"
|
|
target = "screen-100"
|
|
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
for name in ("RUNS", "STATE", "DATA", "CACHE"):
|
|
directory = self.root / name.lower()
|
|
directory.mkdir()
|
|
self.stack.enter_context(patch.object(paths, name, directory))
|
|
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
|
|
database = paths.STATE / "jobs.sqlite"
|
|
connect, readonly = db.connect, db.readonly
|
|
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database)))
|
|
self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database)))
|
|
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
|
|
"e2e": {"enabled": True, "k": 4, "policies": ["teacher"]},
|
|
"repair": {"enabled": False}}}
|
|
self.stack.enter_context(patch.object(registry, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
|
|
self.stack.enter_context(patch.object(pipeline, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
|
|
self.real_fingerprint = registry.registry_fingerprint
|
|
self.stack.enter_context(patch.object(registry, "registry_fingerprint", return_value="frozen-registry"))
|
|
self.manifest = {"registry_task_fingerprint": "frozen-registry", "venv": {},
|
|
"evaluator_version": "0.5.0"}
|
|
self.stack.enter_context(patch.object(pipeline, "build_manifest", side_effect=lambda: copy.deepcopy(self.manifest)))
|
|
with db.connect() as con:
|
|
db.create_run(con, self.source, self.manifest)
|
|
self.eval_manifest = self.make_t1()
|
|
|
|
def make_t1(self):
|
|
directory = paths.DATA / "t1/imagenet1k/n100"
|
|
directory.mkdir(parents=True)
|
|
items = [{"id": str(i), "file": f"images/{i}.jpg", "label": i} for i in range(100)]
|
|
content_sha = hashlib.sha256(json.dumps(items).encode()).hexdigest()
|
|
inner = {"dataset": "imagenet1k", "n": 100, "seed": 17, "sha256": content_sha, "items": items}
|
|
package = directory / "17.tar"
|
|
with tarfile.open(package, "w") as archive:
|
|
for name, payload in [("manifest.json", json.dumps(inner).encode()),
|
|
*((item["file"], item["id"].encode()) for item in items)]:
|
|
member = tarfile.TarInfo(name)
|
|
member.size = len(payload)
|
|
archive.addfile(member, io.BytesIO(payload))
|
|
manifest = directory / "manifest.json"
|
|
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 100, "seed": 17, "tar": package.name,
|
|
"sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
|
|
"content_sha256": content_sha, "eval_set_id": "imagenet-fixed-100"}))
|
|
return manifest
|
|
|
|
def job(self, name="model-a", state="PENDING", error_class=None):
|
|
jid = f"cls--{name}--nash-m--ptq--int8"
|
|
job = {"job_id": jid, "run_id": self.source, "task": "cls", "model": name, "march": "nash-m",
|
|
"backend": "ptq", "qcfg": "int8", "priority": "A", "mode": "oracle", "policy": None,
|
|
"evaluator_version": "0.5.0", "model_spec": {"model_id": name, "task": "cls", "priority": "A"},
|
|
"data": {"dataset": "imagenet1k", "t1": {"path": "/old/2000.tar", "n": 2000,
|
|
"manifest_sha": "1" * 64, "content_sha": "2" * 64}},
|
|
"rollout_plans": [{"mode": "e2e", "policy": "teacher", "k": 4}]}
|
|
directory = paths.RUNS / self.source / jid
|
|
directory.mkdir(parents=True)
|
|
(directory / "job.json").write_text(json.dumps(job))
|
|
with db.connect() as con:
|
|
db.upsert_job(con, self.source, job, 10)
|
|
if state != "PENDING":
|
|
db.set_job_state(con, self.source, jid, state, error_class=error_class,
|
|
result={"terminal": state})
|
|
if state in ("PASS", "FAIL", "UNVERIFIED"):
|
|
(directory / "oracle/artifacts").mkdir(parents=True)
|
|
(directory / "oracle/artifacts/model.hbm").write_bytes(b"original HBM")
|
|
(directory / "oracle/result.json").write_text(json.dumps({"terminal": state, "n": 2000}))
|
|
if state == "PREPARED":
|
|
(directory / "oracle").mkdir()
|
|
(directory / "oracle/preparation.json").write_text(json.dumps({
|
|
"schema": "bench.preparation/1", "terminal": state, "n": 2000}))
|
|
return jid
|
|
|
|
def rollout(self, jid, kk, state="QUEUED", mode="e2e", policy="teacher"):
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, self.source, jid, kk, mode, policy, state)
|
|
if state != "QUEUED":
|
|
fields = {"started_at": 1.0}
|
|
if state not in ("RUNNING", "VERIFYING"):
|
|
fields.update(finished_at=2.0, verdict=json.dumps({"terminal": state}))
|
|
db.rollout_set(con, self.source, jid, kk, **fields)
|
|
|
|
def drain(self):
|
|
pipeline.drain_file(self.source).touch()
|
|
pipeline.collect_drain_file(self.source).touch()
|
|
|
|
def plan(self):
|
|
return continuation.plan(self.source, [str(self.eval_manifest)], self.target)
|
|
|
|
def file_snapshot(self):
|
|
return {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
|
|
|
def test_preview_is_readonly_and_preserves_remaining_sampling_identity(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
before = self.file_snapshot()
|
|
plan = self.plan()
|
|
self.assertEqual(self.file_snapshot(), before)
|
|
self.assertEqual(plan["blockers"], [])
|
|
self.assertEqual(plan["counts"], {"oracle": 0, "agent_queued": 3, "agent_due_repairs": 0,
|
|
"conditional_repairs": 0})
|
|
self.assertEqual(len(plan["jobs"]), 1)
|
|
entry = plan["jobs"][0]
|
|
self.assertFalse(entry["oracle"])
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in entry["rollouts"]],
|
|
[(1, "e2e", "teacher"), (2, "e2e", "teacher"), (3, "e2e", "teacher")])
|
|
self.assertEqual(entry["job"]["oracle_ref"], {"run_id": self.source, "job_id": jid})
|
|
self.assertEqual(entry["job"]["data"]["t1"]["n"], 100)
|
|
|
|
def test_legacy_board_snapshot_allows_only_inventory_changes(self):
|
|
self.job()
|
|
directory = self.root / "registry"
|
|
directory.mkdir()
|
|
boards = b"version: 1\nmarch_by_board: {s100p: nash-m}\nboards: {one: {slot: 0}}\n"
|
|
(directory / "boards.yaml").write_bytes(boards)
|
|
(directory / "matrix.yaml").write_text("modes: {e2e: {k: 4}}\n")
|
|
(directory / "models.csv").write_text("model_id,task\nmodel-a,cls\n")
|
|
with patch.object(registry, "REGISTRY", directory), \
|
|
patch.object(registry, "registry_fingerprint", side_effect=self.real_fingerprint):
|
|
manifest = {"registry_fingerprint": registry.registry_fingerprint()}
|
|
with db.connect() as con:
|
|
con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(manifest), self.source))
|
|
initial = paths.RUNS / self.source / "board_registry.initial.yaml"
|
|
initial.write_bytes(boards)
|
|
expanded = boards.replace(b"one: {slot: 0}", b"one: {slot: 0}, two: {slot: 1}")
|
|
(directory / "boards.yaml").write_bytes(expanded)
|
|
self.assertNotEqual(registry.registry_fingerprint(), manifest["registry_fingerprint"])
|
|
self.assertEqual(self.plan()["blockers"], [])
|
|
# A saved board file cannot excuse changed k, model data or march rules.
|
|
for name, content in (("matrix.yaml", b"modes: {e2e: {k: 5}}\n"),
|
|
("models.csv", b"model_id,task\nmodel-a,det\n"),
|
|
("boards.yaml", expanded.replace(b"s100p: nash-m", b"s100p: nash-p"))):
|
|
with self.subTest(changed=name):
|
|
path = directory / name
|
|
original = path.read_bytes()
|
|
path.write_bytes(content)
|
|
with self.assertRaisesRegex(ValueError, "source registry changed"):
|
|
self.plan()
|
|
path.write_bytes(original)
|
|
initial.unlink()
|
|
with self.assertRaisesRegex(ValueError, "source registry changed"):
|
|
self.plan()
|
|
|
|
def test_prepared_source_continues_exact_slots_without_oracle_result_and_exports(self):
|
|
from export import to_agentrl
|
|
from harness import verify
|
|
jid = self.job(state="PREPARED")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2}
|
|
jp = paths.RUNS / self.source / jid / "job.json"
|
|
job = json.loads(jp.read_text())
|
|
job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}}
|
|
jp.write_text(json.dumps(job))
|
|
handoff = self.plan()
|
|
self.assertEqual(handoff["blockers"], [])
|
|
self.assertEqual(handoff["counts"], {"oracle": 0, "agent_queued": 3,
|
|
"agent_due_repairs": 0, "conditional_repairs": 0})
|
|
self.drain()
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
jd = paths.RUNS / self.target / jid
|
|
with patch.object(rollout, "inputs_ready", return_value=True):
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None, set())
|
|
self.assertFalse(pending)
|
|
self.assertEqual([(r["kk"], r["job_state"], r["completed"], r["started"]) for r in ready],
|
|
[(kk, "PREPARED", 1, 1) for kk in (1, 2, 3)])
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
rd = jd / "rollouts/01"
|
|
(rd / "agent").mkdir(parents=True)
|
|
for name, value in {
|
|
"rollout.json": {"state": "DONE", "mode": "e2e", "policy": "teacher", "agent_config_version": 1},
|
|
"agent/events.jsonl": {"type": "agent_end", "messages": [
|
|
{"role": "user", "content": "Synthetic test task."},
|
|
{"role": "assistant", "content": "Synthetic test response."}]},
|
|
"agent/tools.json": [{"name": "bash"}],
|
|
}.items():
|
|
(rd / name).write_text(json.dumps(value))
|
|
(rd / "agent/system.md").write_text("Synthetic test system.")
|
|
verdict = verify.verify_rollout(rd)
|
|
self.assertIsNone(verdict["oracle_ref"])
|
|
with patch.object(to_agentrl, "BENCH", self.root), \
|
|
patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}):
|
|
exported = to_agentrl.materialize([self.target], "prepared-test", min_agent_version=1)
|
|
self.assertEqual(exported["stats"]["agent_written"], 1)
|
|
self.assertEqual(exported["stats"]["pipeline_written"], 0)
|
|
record = json.loads((self.root / "datasets/prepared-test/teacher.jsonl").read_text())
|
|
self.assertEqual(record["eval_set"]["n"], 100)
|
|
self.assertEqual(record["continuation"], {"source_run_id": self.source,
|
|
"source_job_id": jid, "source_rollout": 1})
|
|
self.assertIsNone(record["oracle_ref"])
|
|
self.assertFalse((jp.parent / "oracle/result.json").exists())
|
|
self.assertFalse((jd / "oracle").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_job(con, self.target, jid))
|
|
|
|
def test_prepared_source_requires_preparation_evidence_and_cannot_release_repair(self):
|
|
jid = self.job(state="PREPARED")
|
|
self.rollout(jid, 1, mode="repair")
|
|
evidence = paths.RUNS / self.source / jid / "oracle/preparation.json"
|
|
evidence.unlink()
|
|
self.assertIn(f"preparation evidence missing: {jid}", self.plan()["blockers"])
|
|
evidence.write_text(json.dumps({"terminal": "FAIL"}))
|
|
self.assertIn(f"preparation evidence missing: {jid}", self.plan()["blockers"])
|
|
evidence.write_text(json.dumps({"terminal": "PREPARED"}))
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
with patch.object(rollout, "inputs_ready", return_value=True):
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None, set())
|
|
self.assertEqual(ready, [])
|
|
self.assertFalse(pending)
|
|
|
|
def test_first_pass_carries_source_attempts_for_prepared_and_pending_jobs(self):
|
|
jobs = {}
|
|
for name, state, history in (("fresh", "PREPARED", None), ("covered", "PREPARED", "PASS"),
|
|
("failed", "PREPARED", "UNVERIFIED"), ("pending", "PENDING", "FAIL")):
|
|
jid = jobs[name] = self.job(name, state=state)
|
|
if history:
|
|
self.rollout(jid, 0, history)
|
|
if history == "UNVERIFIED":
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, verdict=None)
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
started = []
|
|
|
|
def complete(run_id, job_id, kk, *args, **kwargs):
|
|
started.append((job_id, kk))
|
|
with db.connect() as con:
|
|
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3.0)
|
|
return {"terminal": "PASS"}
|
|
|
|
with patch.object(rollout, "inputs_ready", return_value=True):
|
|
ready, _ = pipeline._collection_snapshot(self.target, None, None, set())
|
|
self.assertEqual([r["job_id"] for r in ready[:3]], [jobs["fresh"]] * 3)
|
|
self.assertEqual({r["job_id"] for r in ready if not r["started"] and not r["completed"]},
|
|
{jobs["fresh"]})
|
|
# The collector's actual first-pass filter starts only the fresh model.
|
|
with patch.object(rollout, "run_rollout", side_effect=complete), \
|
|
patch.object(rollout, "setup_models_json"), \
|
|
patch.object(pipeline, "refresh_run_dataset"), \
|
|
patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \
|
|
patch.object(paths, "config", return_value={"pools": {"agent": {"workers": 3}}}):
|
|
result = pipeline.collect_rollouts(self.target, parallel=1, first_pass=True, log=lambda _: None)
|
|
self.assertEqual(result, {"PASS": 1})
|
|
self.assertEqual(started, [(jobs["fresh"], 1)])
|
|
|
|
def test_apply_agent_only_preserves_history_and_repeated_apply_is_idempotent(self):
|
|
jid = self.job(state="FAIL", error_class="compile_fail")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
old_directory = paths.RUNS / self.source / jid
|
|
before = {p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}
|
|
with db.readonly() as con:
|
|
old_job = db.get_job(con, self.source, jid)
|
|
old_finished = db.list_rollouts(con, self.source, jid)[0]
|
|
self.drain()
|
|
handoff = self.plan()
|
|
report = continuation.apply(handoff)
|
|
self.assertTrue(report["identities_match"])
|
|
self.assertEqual(report["agent_states"], {"QUEUED": 3})
|
|
self.assertEqual(continuation.apply(handoff), report)
|
|
with db.readonly() as con:
|
|
self.assertEqual(db.list_jobs(con, self.target), [])
|
|
self.assertEqual(db.get_job(con, self.source, jid), old_job)
|
|
original = db.list_rollouts(con, self.source, jid)
|
|
self.assertEqual(original[0], old_finished)
|
|
self.assertEqual([r["state"] for r in original[1:]], ["TRANSFERRED"] * 3)
|
|
moved = db.list_rollouts(con, self.target, jid)
|
|
self.assertEqual([r["kk"] for r in moved], [1, 2, 3])
|
|
self.assertEqual({p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}, before)
|
|
with patch.object(rollout, "inputs_ready", return_value=True):
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None, set())
|
|
self.assertEqual([r["kk"] for r in ready], [1, 2, 3])
|
|
self.assertFalse(pending)
|
|
|
|
def test_apply_requires_both_drains_and_stopped_schedulers(self):
|
|
self.job()
|
|
handoff = self.plan()
|
|
for mark in (None, pipeline.drain_file(self.source)):
|
|
if mark:
|
|
mark.touch()
|
|
with self.assertRaisesRegex(ValueError, "drain both"):
|
|
continuation.apply(handoff)
|
|
self.assertFalse((paths.RUNS / self.target).exists())
|
|
self.drain()
|
|
with patch.object(pipeline, "scheduler_alive", return_value=True):
|
|
with self.assertRaisesRegex(ValueError, "scheduler still running"):
|
|
continuation.apply(handoff)
|
|
lock = paths.STATE / "locks" / f"collect-{self.source}.lock"
|
|
lock.parent.mkdir(exist_ok=True)
|
|
with lock.open("a") as holder:
|
|
fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
with self.assertRaisesRegex(ValueError, "collector still running"):
|
|
continuation.apply(handoff)
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
|
|
def test_active_work_and_requeued_history_block_handoff(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "RUNNING")
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
self.assertTrue(any("still active" in b for b in handoff["blockers"]))
|
|
with self.assertRaisesRegex(ValueError, "blocked"):
|
|
continuation.apply(handoff)
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, state="QUEUED")
|
|
self.assertTrue(any("execution history" in b for b in self.plan()["blockers"]))
|
|
|
|
def test_stale_or_edited_plan_is_rejected_without_creating_target(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0)
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
modified = copy.deepcopy(handoff)
|
|
modified["jobs"][0]["rollouts"].pop()
|
|
with self.assertRaisesRegex(ValueError, "edited"):
|
|
continuation.apply(modified)
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, state="FAIL", started_at=1.0, finished_at=2.0)
|
|
with self.assertRaisesRegex(ValueError, "source changed"):
|
|
continuation.apply(handoff)
|
|
self.assertFalse((paths.RUNS / self.target).exists())
|
|
|
|
def test_conditional_repairs_keep_reserved_slots_and_frozen_policy(self):
|
|
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
|
|
"policies": ["teacher", "student"], "error_classes": ["compile_fail"]}
|
|
jid = self.job()
|
|
self.rollout(jid, 0, "FAIL")
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
self.assertEqual(handoff["counts"]["conditional_repairs"], 4)
|
|
slots = handoff["conditional_repairs"][0]["rollouts"]
|
|
self.assertEqual([(r["kk"], r["policy"]) for r in slots],
|
|
[(2, "teacher"), (3, "teacher"), (4, "student"), (5, "student")])
|
|
continuation.apply(handoff)
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
with db.connect() as con:
|
|
db.set_job_state(con, self.target, jid, "FAIL", error_class="unrelated")
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
with db.connect() as con:
|
|
db.set_job_state(con, self.target, jid, "FAIL", error_class="compile_fail")
|
|
# Later registry edits cannot expand the frozen handoff's quota or policy.
|
|
self.mx["modes"]["repair"].update(k=50, policies=["another-policy"])
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target, modes=["e2e"]), 0)
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 4)
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
self.assertEqual(continuation.check(self.target)["activated_repairs"], 4)
|
|
with db.readonly() as con:
|
|
rows = db.list_rollouts(con, self.target, jid)
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in rows],
|
|
[(1, "e2e", "teacher"), (2, "repair", "teacher"), (3, "repair", "teacher"),
|
|
(4, "repair", "student"), (5, "repair", "student")])
|
|
|
|
def test_existing_failed_repairs_count_toward_original_quota(self):
|
|
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
|
|
"policies": ["teacher"]}
|
|
jid = self.job(state="FAIL", error_class="compile_fail")
|
|
self.rollout(jid, 7, "FAIL", mode="repair")
|
|
handoff = self.plan()
|
|
self.assertEqual(handoff["counts"]["agent_due_repairs"], 1)
|
|
self.assertEqual(handoff["jobs"][0]["rollouts"],
|
|
[{"kk": 8, "mode": "repair", "policy": "teacher", "source_exists": False}])
|
|
self.drain()
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
with db.readonly() as con:
|
|
original = db.list_rollouts(con, self.source, jid)
|
|
self.assertEqual([(r["kk"], r["state"]) for r in original], [(7, "FAIL"), (8, "TRANSFERRED")])
|
|
moved = db.list_rollouts(con, self.target, jid)
|
|
self.assertEqual([(r["kk"], r["state"]) for r in moved], [(8, "QUEUED")])
|
|
|
|
def test_agent_only_coverage_counts_queue_and_exports_without_fake_oracle(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.target, jid, 1, state="PASS", started_at=3.0, finished_at=4.0)
|
|
models = [{"model_id": "model-a", "task": "cls", "priority": "A", "status": "ready"}]
|
|
with patch.object(coverage, "models", return_value=models), \
|
|
patch.object(coverage, "exported_keys", return_value={(self.target, jid, 1), (self.source, jid, 0)}):
|
|
actual = coverage.collect(self.target)["model-a"]
|
|
self.assertEqual(actual["jobs"], [])
|
|
self.assertEqual(dict(actual["oracle"]), {})
|
|
self.assertEqual(actual["planned_rollouts"], 3)
|
|
self.assertEqual(dict(actual["rollout"]), {"PASS": 1, "QUEUED": 2})
|
|
self.assertEqual(actual["exported"], 1)
|
|
self.assertTrue(continuation.check(self.target)["identities_match"])
|
|
|
|
def test_interrupted_filesystem_preparation_retries_without_losing_old_queue(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
with patch.object(db, "create_run", side_effect=RuntimeError("injected failure before commit")):
|
|
with self.assertRaisesRegex(RuntimeError, "injected failure"):
|
|
continuation.apply(handoff)
|
|
self.assertEqual(continuation.receipt(self.target), handoff)
|
|
self.assertTrue((paths.RUNS / self.target / jid / "job.json").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.list_jobs(con, self.target), [])
|
|
self.assertEqual(db.list_rollouts(con, self.target, jid), [])
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
self.assertEqual(db.list_rollouts(con, self.source, jid)[0]["state"], "QUEUED")
|
|
self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"]))
|
|
report = continuation.apply(handoff)
|
|
self.assertTrue(report["identities_match"])
|
|
self.assertEqual(report["oracle_states"], {"PENDING": 1})
|
|
self.assertEqual(report["agent_states"], {"QUEUED": 1})
|
|
|
|
def test_interrupted_readonly_job_temp_file_can_be_replaced_on_retry(self):
|
|
jid = self.job()
|
|
self.drain()
|
|
handoff = self.plan()
|
|
replace = Path.replace
|
|
|
|
def interrupt_job_replace(source, target):
|
|
if source.name == ".job.json.tmp":
|
|
raise OSError("interrupted after chmod before replace")
|
|
return replace(source, target)
|
|
|
|
with patch.object(Path, "replace", autospec=True, side_effect=interrupt_job_replace):
|
|
with self.assertRaisesRegex(OSError, "after chmod"):
|
|
continuation.apply(handoff)
|
|
temporary = paths.RUNS / self.target / jid / ".job.json.tmp"
|
|
self.assertEqual(temporary.stat().st_mode & 0o777, 0o444)
|
|
self.assertFalse(temporary.with_name("job.json").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
self.assertFalse(temporary.exists())
|
|
self.assertEqual(temporary.with_name("job.json").stat().st_mode & 0o777, 0o444)
|
|
|
|
def test_orphan_queued_rollout_blocks_plan_instead_of_being_omitted(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 3)
|
|
with db.connect() as con:
|
|
con.execute("DELETE FROM jobs WHERE run_id=? AND job_id=?", (self.source, jid))
|
|
handoff = self.plan()
|
|
self.assertTrue(any("no source Oracle" in item for item in handoff["blockers"]))
|
|
self.drain()
|
|
with self.assertRaisesRegex(ValueError, "blocked"):
|
|
continuation.apply(handoff)
|
|
|
|
def test_direct_legacy_rollout_claims_before_preparation_and_closes_failed_attempt(self):
|
|
jid = self.job(state="PASS")
|
|
for kk, error in ((4, SystemExit("missing preparation inputs")), (5, RuntimeError("preparation failed"))):
|
|
with self.subTest(error=type(error).__name__):
|
|
def fail_preparation(*args):
|
|
with db.readonly() as con:
|
|
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
|
|
self.assertEqual(current["state"], "RUNNING")
|
|
self.assertIsNotNone(current["started_at"])
|
|
self.assertTrue(any("still active" in item for item in self.plan()["blockers"]))
|
|
raise error
|
|
|
|
with patch.object(rollout, "_execute_rollout", side_effect=fail_preparation) as execute:
|
|
with self.assertRaises(type(error)):
|
|
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
|
|
with self.assertRaisesRegex(ValueError, "claimed or finished"):
|
|
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
|
|
self.assertEqual(execute.call_count, 1)
|
|
with db.readonly() as con:
|
|
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
|
|
self.assertEqual(current["state"], "UNVERIFIED")
|
|
self.assertIsNotNone(current["finished_at"])
|
|
|
|
def test_closed_source_and_target_full_expansion_are_rejected(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
source_calls = [lambda: pipeline.expand(self.source), lambda: pipeline.enqueue_repairs(self.source),
|
|
lambda: pipeline.enqueue_e2e_coverage(self.source),
|
|
lambda: pipeline.collect_rollouts(self.source, dry_run=True),
|
|
lambda: pipeline.run_oracle(self.source, dry_run=True)]
|
|
for call in source_calls:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
call()
|
|
with db.connect() as con:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
db.claim_job(con, self.source, jid, force=True)
|
|
with db.connect() as con:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
db.claim_rollout(con, self.source, jid, 0)
|
|
for call in (lambda: pipeline.expand(self.target), lambda: pipeline.enqueue_e2e_coverage(self.target),
|
|
lambda: pipeline.collect_rollouts(self.target, coverage="all", dry_run=True)):
|
|
with self.assertRaisesRegex(ValueError, "continuation run"):
|
|
call()
|
|
|
|
def test_identity_check_catches_substitution_even_when_counts_match(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 1)
|
|
self.rollout(jid, 2)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
with db.connect() as con:
|
|
con.execute("UPDATE rollouts SET kk=3 WHERE run_id=? AND job_id=? AND kk=2", (self.target, jid))
|
|
with self.assertRaisesRegex(ValueError, "identity mismatch"):
|
|
continuation.check(self.target)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|