Reuse execution, stage and review evidence to recover database registration and SFT publication without replaying completed work. Keep finished futures during SQLite contention, and hand durable results to startup recovery when draining. Release proven local leases independently of unresolved board isolation. Keep UNKNOWN review retries explicit and archived, publish completed receipts after transient export failures, and prevent a malformed sample from aborting the entire dataset. Scope routine reconciliation to newly finished attempts. Add failure-injection coverage and the production acceptance plan.
793 lines
42 KiB
Python
793 lines
42 KiB
Python
"""Remaining-work handoff against real temporary SQLite and T1 packages."""
|
|
from contextlib import ExitStack, closing, nullcontext
|
|
import copy
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from pathlib import Path
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import continuation, coverage, db, paths, pipeline, registry, rollout
|
|
|
|
|
|
class ContinuationTests(unittest.TestCase):
|
|
source = "old-batch"
|
|
target = "screen-100"
|
|
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
for name in ("RUNS", "STATE", "DATA", "CACHE"):
|
|
directory = self.root / name.lower()
|
|
directory.mkdir()
|
|
self.stack.enter_context(patch.object(paths, name, directory))
|
|
self.stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}))
|
|
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
|
|
database = paths.STATE / "jobs.sqlite"
|
|
connect, readonly = db.connect, db.readonly
|
|
self.stack.enter_context(patch.object(db, "connect", side_effect=lambda **kwargs: connect(database, **kwargs)))
|
|
self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database)))
|
|
self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": {
|
|
"e2e": {"enabled": True, "k": 4, "policies": ["teacher"]},
|
|
"repair": {"enabled": False}}}
|
|
self.stack.enter_context(patch.object(registry, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
|
|
self.stack.enter_context(patch.object(pipeline, "matrix", side_effect=lambda: copy.deepcopy(self.mx)))
|
|
self.real_fingerprint = registry.registry_fingerprint
|
|
self.stack.enter_context(patch.object(registry, "registry_fingerprint", return_value="frozen-registry"))
|
|
self.manifest = {"registry_task_fingerprint": "frozen-registry", "venv": {},
|
|
"evaluator_version": "0.5.0"}
|
|
self.stack.enter_context(patch.object(pipeline, "build_manifest", side_effect=lambda: copy.deepcopy(self.manifest)))
|
|
with db.connect() as con:
|
|
db.create_run(con, self.source, self.manifest)
|
|
self.eval_manifest = self.make_t1()
|
|
|
|
def collection_config(self):
|
|
# Validate the real collector contract without reading installation
|
|
# credentials or opening its shared quota ledger.
|
|
from harness.execution import PHASES, POLICY_FIELDS
|
|
policy = {field: 10 for field in POLICY_FIELDS}
|
|
return {"pools": {"agent": {"workers": 3}},
|
|
"rollouts": {"private_home": True},
|
|
"endpoints": {"teacher": {}, "student": {}},
|
|
"collection_v1": {"version": 1, "quota_ledger": str(self.root / "quota.sqlite"),
|
|
"quota_groups": {"fixture": {"max_concurrency": 1, "rpm": 10, "tpm": 1000}},
|
|
"credential_groups": {"teacher": "fixture"},
|
|
"lifecycle": {"version": 1, **policy,
|
|
"phases": {phase: dict(policy) for phase in PHASES}}}}
|
|
|
|
def make_t1(self):
|
|
directory = paths.DATA / "t1/imagenet1k/n100"
|
|
directory.mkdir(parents=True)
|
|
items = [{"id": str(i), "file": f"images/{i}.jpg", "label": i} for i in range(100)]
|
|
content_sha = hashlib.sha256(json.dumps(items).encode()).hexdigest()
|
|
inner = {"dataset": "imagenet1k", "n": 100, "seed": 17, "sha256": content_sha, "items": items}
|
|
package = directory / "17.tar"
|
|
with tarfile.open(package, "w") as archive:
|
|
for name, payload in [("manifest.json", json.dumps(inner).encode()),
|
|
*((item["file"], item["id"].encode()) for item in items)]:
|
|
member = tarfile.TarInfo(name)
|
|
member.size = len(payload)
|
|
archive.addfile(member, io.BytesIO(payload))
|
|
manifest = directory / "manifest.json"
|
|
manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 100, "seed": 17, "tar": package.name,
|
|
"sha256": hashlib.sha256(package.read_bytes()).hexdigest(),
|
|
"content_sha256": content_sha, "eval_set_id": "imagenet-fixed-100"}))
|
|
return manifest
|
|
|
|
def job(self, name="model-a", state="PENDING", error_class=None):
|
|
jid = f"cls--{name}--nash-m--ptq--int8"
|
|
job = {"job_id": jid, "run_id": self.source, "task": "cls", "model": name, "march": "nash-m",
|
|
"backend": "ptq", "qcfg": "int8", "priority": "A", "mode": "oracle", "policy": None,
|
|
"evaluator_version": "0.5.0", "model_spec": {"model_id": name, "task": "cls", "priority": "A"},
|
|
"data": {"dataset": "imagenet1k", "t1": {"path": "/old/2000.tar", "n": 2000,
|
|
"manifest_sha": "1" * 64, "content_sha": "2" * 64}},
|
|
"rollout_plans": [{"mode": "e2e", "policy": "teacher", "k": 4}]}
|
|
directory = paths.RUNS / self.source / jid
|
|
directory.mkdir(parents=True)
|
|
(directory / "job.json").write_text(json.dumps(job))
|
|
with db.connect() as con:
|
|
db.upsert_job(con, self.source, job, 10)
|
|
if state != "PENDING":
|
|
db.set_job_state(con, self.source, jid, state, error_class=error_class,
|
|
result={"terminal": state})
|
|
if state in ("PASS", "FAIL", "UNVERIFIED"):
|
|
(directory / "oracle/artifacts").mkdir(parents=True)
|
|
(directory / "oracle/artifacts/model.hbm").write_bytes(b"original HBM")
|
|
(directory / "oracle/result.json").write_text(json.dumps({"terminal": state, "n": 2000}))
|
|
if state == "PREPARED":
|
|
(directory / "oracle").mkdir()
|
|
(directory / "oracle/preparation.json").write_text(json.dumps({
|
|
"schema": "bench.preparation/1", "terminal": state, "n": 2000}))
|
|
return jid
|
|
|
|
def rollout(self, jid, kk, state="QUEUED", mode="e2e", policy="teacher"):
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, self.source, jid, kk, mode, policy, state)
|
|
if state != "QUEUED":
|
|
fields = {"started_at": 1.0}
|
|
if state not in ("RUNNING", "VERIFYING"):
|
|
fields.update(finished_at=2.0, verdict=json.dumps({"terminal": state}))
|
|
db.rollout_set(con, self.source, jid, kk, **fields)
|
|
|
|
def drain(self):
|
|
pipeline.drain_file(self.source).touch()
|
|
pipeline.collect_drain_file(self.source).touch()
|
|
|
|
def plan(self):
|
|
return continuation.plan(self.source, [str(self.eval_manifest)], self.target)
|
|
|
|
def file_snapshot(self):
|
|
return {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
|
|
|
def test_runner_identity_survives_plan_apply_and_check(self):
|
|
jid = self.job(state="PASS")
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, self.source, jid, 0, "e2e", "teacher", runner="pi")
|
|
db.rollout_upsert(con, self.source, jid, 1, "e2e", "teacher", runner="opencode")
|
|
planned = self.plan()
|
|
self.assertEqual([r["runner"] for r in planned["jobs"][0]["rollouts"]], ["pi", "opencode"])
|
|
self.assertEqual({p["runner"] for p in planned["jobs"][0]["job"]["rollout_plans"]}, {"pi", "opencode"})
|
|
self.drain()
|
|
with patch.object(pipeline, "scheduler_alive", return_value=[]):
|
|
self.assertTrue(continuation.apply(planned)["identities_match"])
|
|
with db.connect() as con:
|
|
self.assertEqual([r["runner"] for r in db.list_rollouts(con, self.target, jid)], ["pi", "opencode"])
|
|
con.execute("UPDATE rollout_context SET runner='pi' WHERE run_id=? AND kk=1", (self.target,))
|
|
with self.assertRaisesRegex(ValueError, "identity mismatch"):
|
|
continuation.check(self.target)
|
|
|
|
def test_runner_change_after_preview_invalidates_plan(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0)
|
|
planned = self.plan()
|
|
self.drain()
|
|
with db.connect() as con:
|
|
con.execute("UPDATE rollout_context SET runner='opencode' WHERE run_id=?", (self.source,))
|
|
with self.assertRaisesRegex(ValueError, "source changed"):
|
|
continuation.apply(planned)
|
|
|
|
def test_already_frozen_subset_continues_without_another_replacement(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
job_path = paths.RUNS / self.source / jid / "job.json"
|
|
job = json.loads(job_path.read_text())
|
|
_, frozen = continuation.load_eval_set(self.eval_manifest)
|
|
job["data"]["t1"] = frozen
|
|
job_path.write_text(json.dumps(job))
|
|
plan = continuation.plan(self.source, [], self.target)
|
|
self.assertEqual(plan["blockers"], [])
|
|
self.assertEqual(plan["jobs"][0]["job"]["data"]["t1"], frozen)
|
|
self.drain()
|
|
with patch.object(pipeline, "scheduler_alive", return_value=[]):
|
|
continuation.apply(plan)
|
|
check = continuation.check(self.target)
|
|
self.assertTrue(check["identities_match"], check)
|
|
|
|
def test_implicit_or_modified_subset_still_requires_explicit_identity(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
with self.assertRaisesRegex(ValueError, "missing replacement T1"):
|
|
continuation.plan(self.source, [], self.target)
|
|
job_path = paths.RUNS / self.source / jid / "job.json"
|
|
job = json.loads(job_path.read_text())
|
|
_, frozen = continuation.load_eval_set(self.eval_manifest)
|
|
job["data"]["t1"] = {**frozen, "content_sha": "0" * 64}
|
|
job_path.write_text(json.dumps(job))
|
|
with self.assertRaisesRegex(ValueError, "frozen explicit T1 changed"):
|
|
continuation.plan(self.source, [], self.target)
|
|
|
|
def test_preview_is_readonly_and_preserves_remaining_sampling_identity(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
before = self.file_snapshot()
|
|
plan = self.plan()
|
|
self.assertEqual(self.file_snapshot(), before)
|
|
self.assertEqual(plan["blockers"], [])
|
|
self.assertEqual(plan["counts"], {"oracle": 0, "agent_queued": 3, "agent_due_repairs": 0,
|
|
"conditional_repairs": 0})
|
|
self.assertEqual(len(plan["jobs"]), 1)
|
|
entry = plan["jobs"][0]
|
|
self.assertFalse(entry["oracle"])
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in entry["rollouts"]],
|
|
[(1, "e2e", "teacher"), (2, "e2e", "teacher"), (3, "e2e", "teacher")])
|
|
self.assertEqual(entry["job"]["oracle_ref"], {"run_id": self.source, "job_id": jid})
|
|
self.assertEqual(entry["job"]["data"]["t1"]["n"], 100)
|
|
|
|
def test_legacy_board_snapshot_allows_only_inventory_changes(self):
|
|
self.job()
|
|
directory = self.root / "registry"
|
|
directory.mkdir()
|
|
boards = b"version: 1\nmarch_by_board: {s100p: nash-m}\nboards: {one: {slot: 0}}\n"
|
|
(directory / "boards.yaml").write_bytes(boards)
|
|
(directory / "matrix.yaml").write_text("modes: {e2e: {k: 4}}\n")
|
|
(directory / "models.csv").write_text("model_id,task\nmodel-a,cls\n")
|
|
with patch.object(registry, "REGISTRY", directory), \
|
|
patch.object(registry, "registry_fingerprint", side_effect=self.real_fingerprint):
|
|
manifest = {"registry_fingerprint": registry.registry_fingerprint()}
|
|
with db.connect() as con:
|
|
con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(manifest), self.source))
|
|
initial = paths.RUNS / self.source / "board_registry.initial.yaml"
|
|
initial.write_bytes(boards)
|
|
expanded = boards.replace(b"one: {slot: 0}", b"one: {slot: 0}, two: {slot: 1}")
|
|
(directory / "boards.yaml").write_bytes(expanded)
|
|
self.assertNotEqual(registry.registry_fingerprint(), manifest["registry_fingerprint"])
|
|
self.assertEqual(self.plan()["blockers"], [])
|
|
# A saved board file cannot excuse changed k, model data or march rules.
|
|
for name, content in (("matrix.yaml", b"modes: {e2e: {k: 5}}\n"),
|
|
("models.csv", b"model_id,task\nmodel-a,det\n"),
|
|
("boards.yaml", expanded.replace(b"s100p: nash-m", b"s100p: nash-p"))):
|
|
with self.subTest(changed=name):
|
|
path = directory / name
|
|
original = path.read_bytes()
|
|
path.write_bytes(content)
|
|
with self.assertRaisesRegex(ValueError, "source registry changed"):
|
|
self.plan()
|
|
path.write_bytes(original)
|
|
initial.unlink()
|
|
with self.assertRaisesRegex(ValueError, "source registry changed"):
|
|
self.plan()
|
|
|
|
def test_prepared_source_continues_exact_slots_without_oracle_result_and_exports(self):
|
|
from export import to_agentrl
|
|
from harness import verify
|
|
jid = self.job(state="PREPARED")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2}
|
|
jp = paths.RUNS / self.source / jid / "job.json"
|
|
job = json.loads(jp.read_text())
|
|
job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}}
|
|
jp.write_text(json.dumps(job))
|
|
handoff = self.plan()
|
|
self.assertEqual(handoff["blockers"], [])
|
|
self.assertEqual(handoff["counts"], {"oracle": 0, "agent_queued": 3,
|
|
"agent_due_repairs": 0, "conditional_repairs": 0})
|
|
self.drain()
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
jd = paths.RUNS / self.target / jid
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertFalse(pending)
|
|
self.assertEqual([(r["kk"], r["job_state"], r["completed"], r["started"]) for r in ready],
|
|
[(kk, "PREPARED", 1, 1) for kk in (1, 2, 3)])
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
rd = jd / "rollouts/01"
|
|
(rd / "agent").mkdir(parents=True)
|
|
for name, value in {
|
|
"rollout.json": {"state": "DONE", "mode": "e2e", "policy": "teacher", "agent_config_version": 1},
|
|
"agent/events.jsonl": {"type": "agent_end", "messages": [
|
|
{"role": "user", "content": "Synthetic test task."},
|
|
{"role": "assistant", "content": "Synthetic test response."}]},
|
|
"agent/tools.json": [{"name": "bash"}],
|
|
}.items():
|
|
(rd / name).write_text(json.dumps(value))
|
|
(rd / "agent/system.md").write_text("Synthetic test system.")
|
|
verdict = verify.verify_rollout(rd)
|
|
self.assertNotIn("oracle_ref", verdict)
|
|
with patch.object(to_agentrl, "BENCH", self.root), \
|
|
patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}):
|
|
exported = to_agentrl.materialize([self.target], "prepared-test", min_agent_version=1)
|
|
self.assertEqual(exported["stats"]["agent_written"], 1)
|
|
self.assertEqual(exported["stats"]["pipeline_written"], 0)
|
|
record = json.loads((self.root / "datasets/prepared-test/teacher.jsonl").read_text())
|
|
# Continuation freezes the requested T1; preparation alone supplies no
|
|
# measured evaluation identity and must not manufacture SFT success.
|
|
self.assertIsNone(record["eval_set"])
|
|
self.assertEqual(record["required_eval_set"]["n"], 100)
|
|
self.assertEqual(record["required_eval_set"]["eval_set_id"], "imagenet-fixed-100")
|
|
self.assertFalse(record["sft_eligible"])
|
|
self.assertEqual((self.root / "datasets/prepared-test/sft.jsonl").read_text(), "")
|
|
self.assertEqual(record["continuation"], {"source_run_id": self.source,
|
|
"source_job_id": jid, "source_rollout": 1})
|
|
self.assertNotIn("oracle_ref", record)
|
|
self.assertFalse((jp.parent / "oracle/result.json").exists())
|
|
self.assertFalse((jd / "oracle").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_job(con, self.target, jid))
|
|
|
|
def test_continuation_needs_no_oracle_evidence_and_never_releases_repair(self):
|
|
jid = self.job(state="PREPARED")
|
|
self.rollout(jid, 1, mode="repair")
|
|
evidence = paths.RUNS / self.source / jid / "oracle/preparation.json"
|
|
evidence.unlink()
|
|
self.assertEqual(self.plan()["blockers"], [])
|
|
evidence.write_text(json.dumps({"terminal": "FAIL"}))
|
|
self.assertEqual(self.plan()["blockers"], [])
|
|
evidence.write_text(json.dumps({"terminal": "PREPARED"}))
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertEqual(ready, [])
|
|
self.assertFalse(pending)
|
|
|
|
def test_first_pass_carries_source_attempts_for_prepared_and_pending_jobs(self):
|
|
jobs = {}
|
|
for name, state, history in (("fresh", "PREPARED", None), ("covered", "PREPARED", "PASS"),
|
|
("failed", "PREPARED", "UNVERIFIED"), ("pending", "PENDING", "FAIL")):
|
|
jid = jobs[name] = self.job(name, state=state)
|
|
if history:
|
|
self.rollout(jid, 0, history)
|
|
if history == "UNVERIFIED":
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, verdict=None)
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
started = []
|
|
|
|
def complete(run_id, job_id, kk, *args, **kwargs):
|
|
started.append((job_id, kk))
|
|
with db.connect() as con:
|
|
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3.0)
|
|
return {"terminal": "PASS"}
|
|
|
|
ready, _ = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertEqual([r["job_id"] for r in ready[:3]], [jobs["fresh"]] * 3)
|
|
self.assertEqual({r["job_id"] for r in ready if not r["started"] and not r["completed"]},
|
|
{jobs["fresh"]})
|
|
# The collector's actual first-pass filter starts only the fresh model.
|
|
with patch.object(rollout, "run_rollout", side_effect=complete), \
|
|
patch.object(rollout, "setup_models_json"), \
|
|
patch.object(pipeline, "refresh_run_dataset"), \
|
|
patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \
|
|
patch.object(paths, "config", side_effect=self.collection_config), \
|
|
patch("harness.collection_runtime.preflight"):
|
|
result = pipeline.collect_rollouts(self.target, parallel=1, first_pass=True, log=lambda _: None)
|
|
self.assertEqual(result, {"PASS": 1})
|
|
self.assertEqual(started, [(jobs["fresh"], 1)])
|
|
|
|
def test_first_pass_ignores_source_repair_but_keeps_source_e2e_history(self):
|
|
repair_only = self.job("repair-only", state="PREPARED")
|
|
e2e_covered = self.job("e2e-covered", state="PREPARED")
|
|
self.rollout(repair_only, 7, "PASS", mode="repair")
|
|
self.rollout(e2e_covered, 7, "PASS", mode="e2e")
|
|
for jid in (repair_only, e2e_covered):
|
|
self.rollout(jid, 8)
|
|
self.rollout(jid, 9)
|
|
with db.readonly() as con:
|
|
before = db.list_rollouts(con, self.source, repair_only)[0]
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
started = []
|
|
|
|
def complete(run_id, job_id, kk, *args, **kwargs):
|
|
started.append((job_id, kk))
|
|
with db.connect() as con:
|
|
db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3)
|
|
return {"terminal": "PASS"}
|
|
|
|
with patch.object(rollout, "run_rollout", side_effect=complete), \
|
|
patch.object(rollout, "setup_models_json"), \
|
|
patch.object(pipeline, "refresh_run_dataset"), \
|
|
patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \
|
|
patch.object(paths, "config", side_effect=self.collection_config), \
|
|
patch("harness.collection_runtime.preflight"):
|
|
result = pipeline.collect_rollouts(self.target, parallel=1, modes=["e2e"], first_pass=True, log=lambda _: None)
|
|
self.assertEqual(result, {"PASS": 1})
|
|
self.assertEqual(started, [(repair_only, 8)])
|
|
with db.readonly() as con:
|
|
self.assertEqual(db.list_rollouts(con, self.source, repair_only)[0], before)
|
|
self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, repair_only)], ["PASS", "QUEUED"])
|
|
self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, e2e_covered)], ["QUEUED", "QUEUED"])
|
|
|
|
def test_apply_agent_only_preserves_history_and_repeated_apply_is_idempotent(self):
|
|
jid = self.job(state="FAIL", error_class="compile_fail")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
old_directory = paths.RUNS / self.source / jid
|
|
before = {p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}
|
|
with db.readonly() as con:
|
|
old_job = db.get_job(con, self.source, jid)
|
|
old_finished = db.list_rollouts(con, self.source, jid)[0]
|
|
self.drain()
|
|
handoff = self.plan()
|
|
report = continuation.apply(handoff)
|
|
self.assertTrue(report["identities_match"])
|
|
self.assertEqual(report["agent_states"], {"QUEUED": 3})
|
|
self.assertEqual(continuation.apply(handoff), report)
|
|
with db.readonly() as con:
|
|
self.assertEqual(db.list_jobs(con, self.target), [])
|
|
self.assertEqual(db.get_job(con, self.source, jid), old_job)
|
|
original = db.list_rollouts(con, self.source, jid)
|
|
self.assertEqual(original[0], old_finished)
|
|
self.assertEqual([r["state"] for r in original[1:]], ["TRANSFERRED"] * 3)
|
|
moved = db.list_rollouts(con, self.target, jid)
|
|
self.assertEqual([r["kk"] for r in moved], [1, 2, 3])
|
|
self.assertEqual({p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}, before)
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertEqual([r["kk"] for r in ready], [1, 2, 3])
|
|
self.assertFalse(pending)
|
|
|
|
def test_apply_requires_both_drains_and_stopped_schedulers(self):
|
|
self.job()
|
|
handoff = self.plan()
|
|
for mark in (None, pipeline.drain_file(self.source)):
|
|
if mark:
|
|
mark.touch()
|
|
with self.assertRaisesRegex(ValueError, "drain both"):
|
|
continuation.apply(handoff)
|
|
self.assertFalse((paths.RUNS / self.target).exists())
|
|
self.drain()
|
|
with patch.object(pipeline, "scheduler_alive", return_value=True):
|
|
with self.assertRaisesRegex(ValueError, "scheduler still running"):
|
|
continuation.apply(handoff)
|
|
lock = paths.STATE / "locks" / f"collect-{self.source}.lock"
|
|
lock.parent.mkdir(exist_ok=True)
|
|
with lock.open("a") as holder:
|
|
fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
with self.assertRaisesRegex(ValueError, "collector still running"):
|
|
continuation.apply(handoff)
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
|
|
def test_active_work_and_requeued_history_block_handoff(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "RUNNING")
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
self.assertTrue(any("still active" in b for b in handoff["blockers"]))
|
|
with self.assertRaisesRegex(ValueError, "blocked"):
|
|
continuation.apply(handoff)
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, state="QUEUED")
|
|
self.assertTrue(any("execution history" in b for b in self.plan()["blockers"]))
|
|
|
|
def test_stale_or_edited_plan_is_rejected_without_creating_target(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0)
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
modified = copy.deepcopy(handoff)
|
|
modified["jobs"][0]["rollouts"].pop()
|
|
with self.assertRaisesRegex(ValueError, "edited"):
|
|
continuation.apply(modified)
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.source, jid, 0, state="FAIL", started_at=1.0, finished_at=2.0)
|
|
with self.assertRaisesRegex(ValueError, "source changed"):
|
|
continuation.apply(handoff)
|
|
self.assertFalse((paths.RUNS / self.target).exists())
|
|
|
|
def test_file_reads_and_preparation_do_not_block_another_sqlite_writer(self):
|
|
self.job()
|
|
self.drain()
|
|
reads, writes = [], []
|
|
read, write = continuation._read, continuation._write_json
|
|
|
|
def other_writer(label):
|
|
with closing(sqlite3.connect(paths.STATE / "jobs.sqlite", timeout=.05)) as con:
|
|
con.execute("INSERT INTO annotations(ts,note) VALUES(0,?)", (label,))
|
|
con.commit()
|
|
|
|
def read_file(path):
|
|
if path.name == "job.json":
|
|
other_writer("during file read")
|
|
reads.append(path)
|
|
return read(path)
|
|
|
|
def prepare_file(path, value, **kwargs):
|
|
other_writer("during file preparation")
|
|
writes.append(path)
|
|
return write(path, value, **kwargs)
|
|
|
|
with patch.object(continuation, "_read", side_effect=read_file), \
|
|
patch.object(continuation, "_write_json", side_effect=prepare_file):
|
|
handoff = self.plan()
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
self.assertTrue(reads)
|
|
self.assertTrue(writes)
|
|
|
|
def test_database_change_during_file_preparation_rejects_handoff(self):
|
|
jid = self.job()
|
|
self.drain()
|
|
handoff = self.plan()
|
|
write = continuation._write_json
|
|
|
|
def prepare_file(path, value, **kwargs):
|
|
write(path, value, **kwargs)
|
|
if path.name == "job.json":
|
|
with closing(sqlite3.connect(paths.STATE / "jobs.sqlite", timeout=.05)) as con:
|
|
con.execute("UPDATE jobs SET note='concurrent change' WHERE run_id=?", (self.source,))
|
|
con.commit()
|
|
|
|
with patch.object(continuation, "_write_json", side_effect=prepare_file):
|
|
with self.assertRaisesRegex(ValueError, "source changed during apply"):
|
|
continuation.apply(handoff)
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
|
|
def test_different_sources_cannot_prepare_the_same_target_concurrently(self):
|
|
first_source = self.source
|
|
self.job()
|
|
self.drain()
|
|
first = self.plan()
|
|
self.source = "other-source"
|
|
with db.connect() as con:
|
|
db.create_run(con, self.source, self.manifest)
|
|
self.job("other-model")
|
|
self.drain()
|
|
second = self.plan()
|
|
write = continuation._write_json
|
|
competed = []
|
|
|
|
def prepare_file(path, value, **kwargs):
|
|
if path.name == "continuation.json" and not competed:
|
|
competed.append(True)
|
|
with self.assertRaisesRegex(ValueError, "target preparation busy"):
|
|
continuation.apply(second)
|
|
write(path, value, **kwargs)
|
|
|
|
with patch.object(continuation, "_write_json", side_effect=prepare_file):
|
|
self.assertTrue(continuation.apply(first)["identities_match"])
|
|
self.assertTrue(competed)
|
|
self.assertEqual(continuation.receipt(self.target), first)
|
|
with db.readonly() as con:
|
|
self.assertEqual(json.loads(db.get_run(con, first_source)["manifest"])["continued_to"], self.target)
|
|
self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"]))
|
|
|
|
def test_source_file_change_during_preparation_rejects_handoff(self):
|
|
jid = self.job()
|
|
self.drain()
|
|
handoff = self.plan()
|
|
write = continuation._write_json
|
|
|
|
def prepare_file(path, value, **kwargs):
|
|
write(path, value, **kwargs)
|
|
if path.name == "job.json":
|
|
source = paths.RUNS / self.source / jid / "job.json"
|
|
source.write_text(json.dumps(dict(json.loads(source.read_text()), priority="B")))
|
|
|
|
with patch.object(continuation, "_write_json", side_effect=prepare_file):
|
|
with self.assertRaisesRegex(ValueError, "source changed during apply"):
|
|
continuation.apply(handoff)
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
|
|
def test_no_due_or_conditional_repairs_even_with_legacy_config(self):
|
|
self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2,
|
|
"policies": ["teacher", "student"], "error_classes": ["compile_fail"]}
|
|
jid = self.job()
|
|
self.rollout(jid, 0, "FAIL")
|
|
self.rollout(jid, 1)
|
|
failed = self.job("failed", state="FAIL", error_class="compile_fail")
|
|
self.rollout(failed, 4, "FAIL", mode="repair")
|
|
self.rollout(failed, 7, mode="repair")
|
|
# Neither missing nor corrupt historical Oracle evidence blocks E2E handoff.
|
|
(paths.RUNS / self.source / failed / "oracle/result.json").write_text("corrupt JSON")
|
|
self.drain()
|
|
handoff = self.plan()
|
|
self.assertEqual(handoff["counts"]["conditional_repairs"], 0)
|
|
self.assertEqual(handoff["counts"]["agent_due_repairs"], 0)
|
|
self.assertEqual(handoff["counts"]["agent_queued"], 2)
|
|
self.assertEqual(handoff["conditional_repairs"], [])
|
|
continuation.apply(handoff)
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
self.assertEqual(continuation.check(self.target)["activated_repairs"], 0)
|
|
with db.readonly() as con:
|
|
moved = db.list_rollouts(con, self.target, failed)
|
|
original = db.list_rollouts(con, self.source, failed)
|
|
self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in moved],
|
|
[(7, "repair", "teacher", "QUEUED")])
|
|
self.assertEqual([(r["kk"], r["state"]) for r in original], [(4, "FAIL"), (7, "TRANSFERRED")])
|
|
ready, _ = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertEqual([(r["job_id"], r["kk"]) for r in ready], [(jid, 1)])
|
|
|
|
def test_legacy_conditional_receipt_is_checked_but_never_activated(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 1)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
continuation.apply(handoff)
|
|
# Model a receipt written by the previous implementation, including a
|
|
# repair which may already have been activated before the upgrade.
|
|
handoff["conditional_repairs"] = [{"job_id": jid, "error_classes": ["compile_fail"], "rollouts": [
|
|
{"kk": 2, "mode": "repair", "policy": "teacher", "source_exists": False},
|
|
{"kk": 3, "mode": "repair", "policy": "teacher", "source_exists": False}]}]
|
|
handoff["counts"]["conditional_repairs"] = 2
|
|
handoff["plan_id"] = continuation._sha({k: v for k, v in handoff.items() if k != "plan_id"})
|
|
(paths.RUNS / self.target / "continuation.json").write_text(json.dumps(handoff))
|
|
with db.connect() as con:
|
|
db.set_job_state(con, self.target, jid, "FAIL", error_class="compile_fail")
|
|
db.rollout_upsert(con, self.target, jid, 2, "repair", "teacher")
|
|
self.assertEqual(continuation.enqueue_repairs(self.target, handoff), 0)
|
|
self.assertEqual(pipeline.enqueue_repairs(self.target), 0)
|
|
checked = continuation.check(self.target)
|
|
self.assertTrue(checked["identities_match"])
|
|
self.assertEqual(checked["activated_repairs"], 1)
|
|
ready, _ = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in ready], [(1, "e2e")])
|
|
with db.readonly() as con:
|
|
self.assertEqual([(r["kk"], r["mode"]) for r in db.list_rollouts(con, self.target, jid)],
|
|
[(1, "e2e"), (2, "repair")])
|
|
|
|
def test_agent_only_coverage_counts_queue_and_exports_without_fake_oracle(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 0, "FAIL")
|
|
for kk in (1, 2, 3):
|
|
self.rollout(jid, kk)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
with db.connect() as con:
|
|
db.rollout_set(con, self.target, jid, 1, state="PASS", started_at=3.0, finished_at=4.0)
|
|
models = [{"model_id": "model-a", "task": "cls", "priority": "A", "status": "ready"}]
|
|
with patch.object(coverage, "models", return_value=models), \
|
|
patch.object(coverage, "exported_keys", return_value={(self.target, jid, 1), (self.source, jid, 0)}):
|
|
actual = coverage.collect(self.target)["model-a"]
|
|
self.assertEqual(actual["jobs"], [])
|
|
self.assertEqual(dict(actual["oracle"]), {})
|
|
self.assertEqual(actual["planned_rollouts"], 3)
|
|
self.assertEqual(dict(actual["rollout"]), {"PASS": 1, "QUEUED": 2})
|
|
self.assertEqual(actual["exported"], 1)
|
|
self.assertTrue(continuation.check(self.target)["identities_match"])
|
|
|
|
def test_interrupted_filesystem_preparation_retries_without_losing_old_queue(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
self.drain()
|
|
handoff = self.plan()
|
|
with patch.object(db, "create_run", side_effect=RuntimeError("injected failure before commit")):
|
|
with self.assertRaisesRegex(RuntimeError, "injected failure"):
|
|
continuation.apply(handoff)
|
|
self.assertEqual(continuation.receipt(self.target), handoff)
|
|
self.assertTrue((paths.RUNS / self.target / jid / "job.json").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.list_jobs(con, self.target), [])
|
|
self.assertEqual(db.list_rollouts(con, self.target, jid), [])
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
self.assertEqual(db.list_rollouts(con, self.source, jid)[0]["state"], "QUEUED")
|
|
self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"]))
|
|
report = continuation.apply(handoff)
|
|
self.assertTrue(report["identities_match"])
|
|
self.assertEqual(report["oracle_states"], {"PENDING": 1})
|
|
self.assertEqual(report["agent_states"], {"QUEUED": 1})
|
|
|
|
def test_interrupted_readonly_job_temp_file_can_be_replaced_on_retry(self):
|
|
jid = self.job()
|
|
self.drain()
|
|
handoff = self.plan()
|
|
replace = Path.replace
|
|
|
|
def interrupt_job_replace(source, target):
|
|
if source.name == ".job.json.tmp":
|
|
raise OSError("interrupted after chmod before replace")
|
|
return replace(source, target)
|
|
|
|
with patch.object(Path, "replace", autospec=True, side_effect=interrupt_job_replace):
|
|
with self.assertRaisesRegex(OSError, "after chmod"):
|
|
continuation.apply(handoff)
|
|
temporary = paths.RUNS / self.target / jid / ".job.json.tmp"
|
|
self.assertEqual(temporary.stat().st_mode & 0o777, 0o444)
|
|
self.assertFalse(temporary.with_name("job.json").exists())
|
|
with db.readonly() as con:
|
|
self.assertIsNone(db.get_run(con, self.target))
|
|
self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING")
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
self.assertFalse(temporary.exists())
|
|
self.assertEqual(temporary.with_name("job.json").stat().st_mode & 0o777, 0o444)
|
|
|
|
def test_independent_e2e_handoff_preserves_history_without_inventing_oracle(self):
|
|
jid = self.job()
|
|
with db.connect() as con:
|
|
con.execute("UPDATE jobs SET mode='e2e',policy='teacher' WHERE run_id=? AND job_id=?",
|
|
(self.source, jid))
|
|
self.rollout(jid, 0, "INFRA_ERROR")
|
|
self.rollout(jid, 1, "PASS")
|
|
self.rollout(jid, 2)
|
|
self.rollout(jid, 3)
|
|
before = self.file_snapshot()
|
|
handoff = self.plan()
|
|
self.assertEqual(self.file_snapshot(), before)
|
|
self.assertEqual(handoff["blockers"], [])
|
|
self.assertEqual(handoff["counts"]["oracle"], 0)
|
|
self.assertEqual(handoff["counts"]["agent_queued"], 2)
|
|
self.assertNotIn("oracle_ref", handoff["jobs"][0]["job"])
|
|
self.drain()
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
self.assertTrue(continuation.apply(handoff)["identities_match"])
|
|
with db.readonly() as con:
|
|
original = {r["kk"]: r["state"] for r in db.list_rollouts(con, self.source, jid)}
|
|
self.assertIsNone(db.get_job(con, self.target, jid))
|
|
self.assertEqual(original, {0: "INFRA_ERROR", 1: "PASS", 2: "TRANSFERRED", 3: "TRANSFERRED"})
|
|
ready, pending = pipeline._collection_snapshot(self.target, None, None)
|
|
self.assertFalse(pending)
|
|
self.assertEqual([(r["kk"], r["job_state"], r["started"]) for r in ready],
|
|
[(2, None, 2), (3, None, 2)])
|
|
|
|
def test_independent_e2e_does_not_allow_repair_without_oracle(self):
|
|
jid = self.job()
|
|
with db.connect() as con:
|
|
con.execute("UPDATE jobs SET mode='e2e' WHERE run_id=? AND job_id=?", (self.source, jid))
|
|
self.rollout(jid, 0, mode="repair")
|
|
self.assertTrue(self.plan()["blockers"])
|
|
|
|
def test_orphan_queued_rollout_blocks_plan_instead_of_being_omitted(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 3)
|
|
with db.connect() as con:
|
|
con.execute("DELETE FROM jobs WHERE run_id=? AND job_id=?", (self.source, jid))
|
|
handoff = self.plan()
|
|
self.assertTrue(any("no source Oracle" in item for item in handoff["blockers"]))
|
|
self.drain()
|
|
with self.assertRaisesRegex(ValueError, "blocked"):
|
|
continuation.apply(handoff)
|
|
|
|
def test_direct_legacy_rollout_claims_before_preparation_and_closes_failed_attempt(self):
|
|
jid = self.job(state="PASS")
|
|
for kk, error in ((4, SystemExit("missing preparation inputs")), (5, RuntimeError("preparation failed"))):
|
|
with self.subTest(error=type(error).__name__):
|
|
def fail_preparation(*args, **kwargs):
|
|
with db.readonly() as con:
|
|
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
|
|
self.assertEqual(current["state"], "RUNNING")
|
|
self.assertIsNotNone(current["started_at"])
|
|
self.assertTrue(any("still active" in item for item in self.plan()["blockers"]))
|
|
raise error
|
|
|
|
with patch.object(paths, "config", side_effect=self.collection_config), \
|
|
patch("harness.collection_runtime.preflight"), \
|
|
patch.object(rollout, "_execute_rollout", side_effect=fail_preparation) as execute:
|
|
with self.assertRaises(type(error)):
|
|
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
|
|
with self.assertRaisesRegex(ValueError, "claimed or finished"):
|
|
rollout.run_rollout(self.source, jid, kk, "e2e", "teacher")
|
|
self.assertEqual(execute.call_count, 1)
|
|
with db.readonly() as con:
|
|
current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk)
|
|
self.assertEqual(current["state"], "UNVERIFIED")
|
|
self.assertIsNotNone(current["finished_at"])
|
|
|
|
def test_closed_source_and_target_full_expansion_are_rejected(self):
|
|
jid = self.job()
|
|
self.rollout(jid, 0)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
source_calls = [lambda: pipeline.expand(self.source), lambda: pipeline.enqueue_repairs(self.source),
|
|
lambda: continuation.enqueue_repairs(self.source, {}),
|
|
lambda: pipeline.enqueue_e2e_coverage(self.source),
|
|
lambda: pipeline.collect_rollouts(self.source, dry_run=True),
|
|
lambda: pipeline.run_oracle(self.source, dry_run=True)]
|
|
for call in source_calls:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
call()
|
|
with db.connect() as con:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
db.claim_job(con, self.source, jid, force=True)
|
|
with db.connect() as con:
|
|
with self.assertRaisesRegex(ValueError, "execution is closed"):
|
|
db.claim_rollout(con, self.source, jid, 0)
|
|
for call in (lambda: pipeline.expand(self.target), lambda: pipeline.enqueue_e2e_coverage(self.target),
|
|
lambda: pipeline.collect_rollouts(self.target, coverage="all", dry_run=True)):
|
|
with self.assertRaisesRegex(ValueError, "continuation run"):
|
|
call()
|
|
|
|
def test_identity_check_catches_substitution_even_when_counts_match(self):
|
|
jid = self.job(state="PASS")
|
|
self.rollout(jid, 1)
|
|
self.rollout(jid, 2)
|
|
self.drain()
|
|
continuation.apply(self.plan())
|
|
with db.connect() as con:
|
|
con.execute("UPDATE rollouts SET kk=3 WHERE run_id=? AND job_id=? AND kk=2", (self.target, jid))
|
|
with self.assertRaisesRegex(ValueError, "identity mismatch"):
|
|
continuation.check(self.target)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|