data-gen-bpu/scripts/test_plan_publication.py

269 lines
13 KiB
Python

"""Isolated plan/publication tests: never call a runner, board or toolchain."""
from contextlib import contextmanager, ExitStack
import json
import os
from pathlib import Path
import sqlite3
import tempfile
import threading
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import db, pipeline, runners, rollout, verify
class PlanPublication(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.root = Path(self.tmp.name)
self.db_path = self.root / "jobs.sqlite"
self.stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root)}))
self.specs = []
self.gate = 5
readonly = db.readonly
self.stack.enter_context(patch.object(db, "readonly", lambda: readonly(self.db_path)))
for obj, name, value in ((db, "connect", self.connect), (pipeline.paths, "RUNS", self.root / "runs"),
(pipeline.paths, "DATA", self.root / "data"),
(pipeline, "expand_matrix", lambda *a, **kw: self.specs),
(pipeline, "model_row", lambda _: {"dataset": "test"}),
(pipeline, "build_job_json", self.snapshot),
(pipeline.scheduler, "estimate_job", lambda *a: 1)):
self.stack.enter_context(patch.object(obj, name, value))
with self.connect() as con:
db.create_run(con, "r", {})
def collection_config(self):
# Validate the real collector contract without reading installation
# credentials or opening its shared quota ledger.
from harness.execution import PHASES, POLICY_FIELDS
policy = {field: 10 for field in POLICY_FIELDS}
return {"pools": {"agent": {"workers": 3}},
"rollouts": {"private_home": True},
"endpoints": {"teacher": {}, "student": {}},
"collection_v1": {"version": 1, "quota_ledger": str(self.root / "quota.sqlite"),
"quota_groups": {"fixture": {"max_concurrency": 1, "rpm": 10, "tpm": 1000}},
"credential_groups": {"teacher": "fixture"},
"lifecycle": {"version": 1, **policy,
"phases": {phase: dict(policy) for phase in PHASES}}}}
@contextmanager
def connect(self):
con = db._open(self.db_path)
con.executescript(db.SCHEMA)
try:
yield con
con.commit()
finally:
con.close()
def spec(self, mode="e2e", policy="teacher", runner="pi", k=2):
row = dict(job_id="j", task="cls", model="m", march="nash-m", backend="ptq", qcfg="int8",
priority="A", mode=mode, policy=policy, runner=runner, k=k)
self.specs.append(SimpleNamespace(**row, to_row=lambda: row.copy()))
def snapshot(self, spec, run_id, version, plans, target=None):
return {"run_id": run_id, "job_id": spec.job_id, "task": spec.task, "model": spec.model,
"rollout_plans": plans or [], "gates": {"tol": self.gate}, "data": {"dataset": "test", "t1": {}},
"model_spec": {"status": "ready"}, "target": target}
def test_json_recipe_representation_is_idempotent_but_values_stay_frozen(self):
requested = {"model_spec": {"preprocess": {"mean": (.485, .456, .406)}}}
job = self.root / "recipe.json"
job.write_text(json.dumps(requested))
before = job.read_bytes()
pipeline._check_frozen_job(job, requested)
self.assertIsInstance(requested["model_spec"]["preprocess"]["mean"], tuple)
self.assertEqual(job.read_bytes(), before)
requested["model_spec"]["preprocess"]["mean"] = (.5, .456, .406)
with self.assertRaisesRegex(ValueError, "model_spec"):
pipeline._check_frozen_job(job, requested)
self.assertEqual(job.read_bytes(), before)
def test_expand_preserves_full_frozen_run_target(self):
target = {'board_type': 's100p', 'march': 'nash-m', 'revision': 'fixture-frozen'}
with self.connect() as con:
con.execute('UPDATE runs SET manifest=? WHERE run_id=?', (json.dumps({'target': target}), 'r'))
self.spec()
pipeline.expand('r')
job = json.loads((self.root / 'runs/r/j/job.json').read_text())
self.assertEqual(job['target'], target)
def rows(self):
with self.connect() as con:
return db.list_rollouts(con, "r", "j")
def test_modes_policies_runners_share_one_id_space_and_repeat_is_idempotent(self):
for mode, policy, runner in (("e2e", "teacher", "pi"), ("e2e", "student", "pi"),
("e2e", "teacher", "opencode"), ("repair", "teacher", "pi")):
self.spec(mode, policy, runner)
pipeline.expand("r")
before = self.rows()
self.assertEqual([r["kk"] for r in before], list(range(8)))
self.assertEqual({(r["mode"], r["policy"], r["runner"]) for r in before},
{(s.mode, s.policy, s.runner) for s in self.specs})
pipeline.expand("r")
self.assertEqual(before, self.rows())
def test_unavailable_runner_is_rejected_before_claim_or_pi_setup(self):
self.spec(runner="opencode")
pipeline.expand("r")
def refuse(policy, *, execute=False):
self.assertTrue(execute)
raise ValueError("runner disabled")
with patch.object(pipeline.paths, "STATE", self.root / "state"), \
patch.object(pipeline.paths, "config", side_effect=self.collection_config), \
patch.object(runners, "get", return_value=SimpleNamespace(check=refuse)), \
patch.object(rollout, "setup_models_json", side_effect=AssertionError("must not write Pi HOME")), \
patch.object(rollout, "run_rollout", side_effect=AssertionError("must not execute")):
with self.assertRaisesRegex(ValueError, "runner disabled"):
pipeline.collect_rollouts("r", log=lambda _: None)
self.assertTrue(all(r["state"] == "QUEUED" and r["started_at"] is None for r in self.rows()))
def test_collection_dry_run_keeps_legacy_db_bytes_schema_and_tree(self):
self.spec()
pipeline.expand("r")
with self.connect() as con:
con.execute("DROP TABLE rollout_context")
before = {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
with patch.object(pipeline.paths, "STATE", self.root / "state"), \
patch.object(pipeline.paths, "config", return_value={"pools": {"agent": {"workers": 1}}}), \
patch.object(runners, "get", return_value=SimpleNamespace(check=lambda *a, **kw: None)), \
patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
patch.object(pipeline.scheduler, "acquire", side_effect=AssertionError("dry run must not acquire a pool")), \
patch.object(db, "connect", side_effect=AssertionError("dry run must not create/migrate DB")):
result = pipeline.collect_rollouts("r", dry_run=True, log=lambda _: None)
after = {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
self.assertEqual(before, after)
self.assertEqual(result, {"DRY_RUN": 2})
with db.readonly() as con:
self.assertIsNone(con.execute("SELECT 1 FROM sqlite_master WHERE name='rollout_context'").fetchone())
self.assertFalse((self.root / "state").exists())
def test_verify_writes_real_db_without_changing_opencode_identity(self):
self.spec(runner="opencode")
pipeline.expand("r")
jd = self.root / "runs/r/j"
job = json.loads((jd / "job.json").read_text())
job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}}
(jd / "job.json").chmod(0o644)
(jd / "job.json").write_text(json.dumps(job))
rd = jd / "rollouts/00"
rd.mkdir(parents=True)
(rd / "rollout.json").write_text(json.dumps({"mode": "e2e", "policy": "teacher", "runner": "opencode", "state": "DONE"}))
with self.connect() as con:
db.rollout_set(con, "r", "j", 0, state="VERIFYING")
result = verify.verify_rollout(rd)
self.assertTrue((rd / "verdict.json").is_file())
with self.connect() as con:
self.assertEqual(db.rollout_runner(con, "r", "j", 0), "opencode")
row = db.list_rollouts(con, "r", "j")[0]
self.assertEqual(row["state"], result["terminal"])
self.assertIsNotNone(row["verdict"])
def test_conflicting_identity_is_rejected(self):
with self.connect() as con:
db.rollout_upsert(con, "r", "j", 0, "e2e", "teacher")
with self.assertRaisesRegex(ValueError, "identity conflict"):
db.rollout_upsert(con, "r", "j", 0, "e2e", "teacher", runner="opencode")
def test_interruption_after_file_publish_is_retryable(self):
self.spec()
with patch.object(db, "upsert_job", side_effect=RuntimeError("crash before DB commit")):
with self.assertRaisesRegex(RuntimeError, "crash"):
pipeline.expand("r")
jj = self.root / "runs/r/j/job.json"
before = jj.read_bytes()
self.assertEqual(self.rows(), [])
pipeline.expand("r")
self.assertEqual(jj.read_bytes(), before)
self.assertEqual(len(self.rows()), 2)
self.assertFalse(list(jj.parent.glob(".job-*")))
def test_interrupted_different_content_refuses_without_rows(self):
self.spec()
with patch.object(db, "upsert_job", side_effect=RuntimeError):
with self.assertRaises(RuntimeError):
pipeline.expand("r")
jj = self.root / "runs/r/j/job.json"
before = jj.read_bytes()
self.gate = 9
with self.assertRaisesRegex(ValueError, "content conflict"):
pipeline.expand("r")
self.assertEqual(jj.read_bytes(), before)
self.assertEqual(self.rows(), [])
def test_missing_snapshot_for_existing_db_is_not_reconstructed(self):
self.spec()
with self.connect() as con:
db.upsert_job(con, "r", self.specs[0].to_row(), 1)
with self.assertRaisesRegex(ValueError, "no frozen job"):
pipeline.expand("r")
self.assertFalse((self.root / "runs/r/j/job.json").exists())
def test_concurrent_expands_publish_once(self):
self.spec(); self.spec(runner="opencode")
barrier = threading.Barrier(2)
errors = []
def expand():
try:
barrier.wait(timeout=5)
pipeline.expand("r")
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=expand) for _ in range(2)]
for t in threads: t.start()
for t in threads: t.join(timeout=20)
self.assertFalse(any(t.is_alive() for t in threads))
self.assertEqual(errors, [])
self.assertEqual(len(self.rows()), 4)
def test_legacy_pi_plan_and_old_rows_remain_compatible(self):
self.spec()
jd = self.root / "runs/r/j"
jd.mkdir(parents=True)
old = self.snapshot(self.specs[0], "r", "unused", [{"mode": "e2e", "policy": "teacher", "k": 2}])
(jd / "job.json").write_text(json.dumps(old))
with self.connect() as con:
con.execute("INSERT INTO rollouts(run_id,job_id,kk,mode,policy,state) VALUES('r','j',7,'e2e','teacher','PASS')")
pipeline.expand("r")
self.assertEqual([r["kk"] for r in self.rows()], [7, 8])
self.assertEqual(json.loads((jd / "job.json").read_text()), old)
self.assertTrue(all(r["runner"] == "pi" for r in self.rows()))
def test_legacy_task_contract_defaults_include_shared_coco_module(self):
p = self.root / "legacy.json"
old = {"task": "pose", "rollout_plans": []}
p.write_text(json.dumps(old))
new = {**old, "task_contract": {"module": "harness.eval.coco_extra"}, "model_runtime": None,
"toolchain": {"profile": None, "version": "legacy-config"}}
pipeline._check_frozen_job(p, new)
new["toolchain"]["profile"] = "explicit-new-profile"
with self.assertRaisesRegex(ValueError, "content conflict"):
pipeline._check_frozen_job(p, new)
def test_legacy_runtime_equivalent_snapshot_is_idempotent_but_override_is_not(self):
p = self.root / "legacy-runtime.json"
old = {"task": "cls", "model_spec": {"runtime_python": "/old/venv/bin/python"}, "rollout_plans": []}
p.write_text(json.dumps(old))
new = {**old, "model_runtime": {"configured": "/old/venv/bin/python", "python": "/old/venv/bin/python"}}
pipeline._check_frozen_job(p, new)
new["model_runtime"] = {"configured": "/different/venv/bin/python", "python": "/different/venv/bin/python"}
with self.assertRaisesRegex(ValueError, "content conflict"):
pipeline._check_frozen_job(p, new)
self.assertEqual(json.loads(p.read_text()), old)
def test_new_plan_on_frozen_job_requires_new_run(self):
self.spec(); pipeline.expand("r")
self.spec(runner="opencode")
with self.assertRaisesRegex(ValueError, "plan conflict"):
pipeline.expand("r")
self.assertEqual(len(self.rows()), 2)
if __name__ == "__main__":
unittest.main()