269 lines
13 KiB
Python
269 lines
13 KiB
Python
"""Isolated plan/publication tests: never call a runner, board or toolchain."""
|
|
from contextlib import contextmanager, ExitStack
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
import threading
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, pipeline, runners, rollout, verify
|
|
|
|
|
|
class PlanPublication(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
self.root = Path(self.tmp.name)
|
|
self.db_path = self.root / "jobs.sqlite"
|
|
self.stack.enter_context(patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root)}))
|
|
self.specs = []
|
|
self.gate = 5
|
|
readonly = db.readonly
|
|
self.stack.enter_context(patch.object(db, "readonly", lambda: readonly(self.db_path)))
|
|
for obj, name, value in ((db, "connect", self.connect), (pipeline.paths, "RUNS", self.root / "runs"),
|
|
(pipeline.paths, "DATA", self.root / "data"),
|
|
(pipeline, "expand_matrix", lambda *a, **kw: self.specs),
|
|
(pipeline, "model_row", lambda _: {"dataset": "test"}),
|
|
(pipeline, "build_job_json", self.snapshot),
|
|
(pipeline.scheduler, "estimate_job", lambda *a: 1)):
|
|
self.stack.enter_context(patch.object(obj, name, value))
|
|
with self.connect() as con:
|
|
db.create_run(con, "r", {})
|
|
|
|
def collection_config(self):
|
|
# Validate the real collector contract without reading installation
|
|
# credentials or opening its shared quota ledger.
|
|
from harness.execution import PHASES, POLICY_FIELDS
|
|
policy = {field: 10 for field in POLICY_FIELDS}
|
|
return {"pools": {"agent": {"workers": 3}},
|
|
"rollouts": {"private_home": True},
|
|
"endpoints": {"teacher": {}, "student": {}},
|
|
"collection_v1": {"version": 1, "quota_ledger": str(self.root / "quota.sqlite"),
|
|
"quota_groups": {"fixture": {"max_concurrency": 1, "rpm": 10, "tpm": 1000}},
|
|
"credential_groups": {"teacher": "fixture"},
|
|
"lifecycle": {"version": 1, **policy,
|
|
"phases": {phase: dict(policy) for phase in PHASES}}}}
|
|
|
|
@contextmanager
|
|
def connect(self):
|
|
con = db._open(self.db_path)
|
|
con.executescript(db.SCHEMA)
|
|
try:
|
|
yield con
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
|
|
def spec(self, mode="e2e", policy="teacher", runner="pi", k=2):
|
|
row = dict(job_id="j", task="cls", model="m", march="nash-m", backend="ptq", qcfg="int8",
|
|
priority="A", mode=mode, policy=policy, runner=runner, k=k)
|
|
self.specs.append(SimpleNamespace(**row, to_row=lambda: row.copy()))
|
|
|
|
def snapshot(self, spec, run_id, version, plans, target=None):
|
|
return {"run_id": run_id, "job_id": spec.job_id, "task": spec.task, "model": spec.model,
|
|
"rollout_plans": plans or [], "gates": {"tol": self.gate}, "data": {"dataset": "test", "t1": {}},
|
|
"model_spec": {"status": "ready"}, "target": target}
|
|
|
|
def test_json_recipe_representation_is_idempotent_but_values_stay_frozen(self):
|
|
requested = {"model_spec": {"preprocess": {"mean": (.485, .456, .406)}}}
|
|
job = self.root / "recipe.json"
|
|
job.write_text(json.dumps(requested))
|
|
before = job.read_bytes()
|
|
pipeline._check_frozen_job(job, requested)
|
|
self.assertIsInstance(requested["model_spec"]["preprocess"]["mean"], tuple)
|
|
self.assertEqual(job.read_bytes(), before)
|
|
requested["model_spec"]["preprocess"]["mean"] = (.5, .456, .406)
|
|
with self.assertRaisesRegex(ValueError, "model_spec"):
|
|
pipeline._check_frozen_job(job, requested)
|
|
self.assertEqual(job.read_bytes(), before)
|
|
|
|
def test_expand_preserves_full_frozen_run_target(self):
|
|
target = {'board_type': 's100p', 'march': 'nash-m', 'revision': 'fixture-frozen'}
|
|
with self.connect() as con:
|
|
con.execute('UPDATE runs SET manifest=? WHERE run_id=?', (json.dumps({'target': target}), 'r'))
|
|
self.spec()
|
|
pipeline.expand('r')
|
|
job = json.loads((self.root / 'runs/r/j/job.json').read_text())
|
|
self.assertEqual(job['target'], target)
|
|
|
|
def rows(self):
|
|
with self.connect() as con:
|
|
return db.list_rollouts(con, "r", "j")
|
|
|
|
def test_modes_policies_runners_share_one_id_space_and_repeat_is_idempotent(self):
|
|
for mode, policy, runner in (("e2e", "teacher", "pi"), ("e2e", "student", "pi"),
|
|
("e2e", "teacher", "opencode"), ("repair", "teacher", "pi")):
|
|
self.spec(mode, policy, runner)
|
|
pipeline.expand("r")
|
|
before = self.rows()
|
|
self.assertEqual([r["kk"] for r in before], list(range(8)))
|
|
self.assertEqual({(r["mode"], r["policy"], r["runner"]) for r in before},
|
|
{(s.mode, s.policy, s.runner) for s in self.specs})
|
|
pipeline.expand("r")
|
|
self.assertEqual(before, self.rows())
|
|
|
|
def test_unavailable_runner_is_rejected_before_claim_or_pi_setup(self):
|
|
self.spec(runner="opencode")
|
|
pipeline.expand("r")
|
|
def refuse(policy, *, execute=False):
|
|
self.assertTrue(execute)
|
|
raise ValueError("runner disabled")
|
|
with patch.object(pipeline.paths, "STATE", self.root / "state"), \
|
|
patch.object(pipeline.paths, "config", side_effect=self.collection_config), \
|
|
patch.object(runners, "get", return_value=SimpleNamespace(check=refuse)), \
|
|
patch.object(rollout, "setup_models_json", side_effect=AssertionError("must not write Pi HOME")), \
|
|
patch.object(rollout, "run_rollout", side_effect=AssertionError("must not execute")):
|
|
with self.assertRaisesRegex(ValueError, "runner disabled"):
|
|
pipeline.collect_rollouts("r", log=lambda _: None)
|
|
self.assertTrue(all(r["state"] == "QUEUED" and r["started_at"] is None for r in self.rows()))
|
|
|
|
def test_collection_dry_run_keeps_legacy_db_bytes_schema_and_tree(self):
|
|
self.spec()
|
|
pipeline.expand("r")
|
|
with self.connect() as con:
|
|
con.execute("DROP TABLE rollout_context")
|
|
before = {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
|
with patch.object(pipeline.paths, "STATE", self.root / "state"), \
|
|
patch.object(pipeline.paths, "config", return_value={"pools": {"agent": {"workers": 1}}}), \
|
|
patch.object(runners, "get", return_value=SimpleNamespace(check=lambda *a, **kw: None)), \
|
|
patch.object(rollout, "run_rollout", return_value={"dry_run": True}), \
|
|
patch.object(pipeline.scheduler, "acquire", side_effect=AssertionError("dry run must not acquire a pool")), \
|
|
patch.object(db, "connect", side_effect=AssertionError("dry run must not create/migrate DB")):
|
|
result = pipeline.collect_rollouts("r", dry_run=True, log=lambda _: None)
|
|
after = {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()}
|
|
self.assertEqual(before, after)
|
|
self.assertEqual(result, {"DRY_RUN": 2})
|
|
with db.readonly() as con:
|
|
self.assertIsNone(con.execute("SELECT 1 FROM sqlite_master WHERE name='rollout_context'").fetchone())
|
|
self.assertFalse((self.root / "state").exists())
|
|
|
|
def test_verify_writes_real_db_without_changing_opencode_identity(self):
|
|
self.spec(runner="opencode")
|
|
pipeline.expand("r")
|
|
jd = self.root / "runs/r/j"
|
|
job = json.loads((jd / "job.json").read_text())
|
|
job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}}
|
|
(jd / "job.json").chmod(0o644)
|
|
(jd / "job.json").write_text(json.dumps(job))
|
|
rd = jd / "rollouts/00"
|
|
rd.mkdir(parents=True)
|
|
(rd / "rollout.json").write_text(json.dumps({"mode": "e2e", "policy": "teacher", "runner": "opencode", "state": "DONE"}))
|
|
with self.connect() as con:
|
|
db.rollout_set(con, "r", "j", 0, state="VERIFYING")
|
|
result = verify.verify_rollout(rd)
|
|
self.assertTrue((rd / "verdict.json").is_file())
|
|
with self.connect() as con:
|
|
self.assertEqual(db.rollout_runner(con, "r", "j", 0), "opencode")
|
|
row = db.list_rollouts(con, "r", "j")[0]
|
|
self.assertEqual(row["state"], result["terminal"])
|
|
self.assertIsNotNone(row["verdict"])
|
|
|
|
def test_conflicting_identity_is_rejected(self):
|
|
with self.connect() as con:
|
|
db.rollout_upsert(con, "r", "j", 0, "e2e", "teacher")
|
|
with self.assertRaisesRegex(ValueError, "identity conflict"):
|
|
db.rollout_upsert(con, "r", "j", 0, "e2e", "teacher", runner="opencode")
|
|
|
|
def test_interruption_after_file_publish_is_retryable(self):
|
|
self.spec()
|
|
with patch.object(db, "upsert_job", side_effect=RuntimeError("crash before DB commit")):
|
|
with self.assertRaisesRegex(RuntimeError, "crash"):
|
|
pipeline.expand("r")
|
|
jj = self.root / "runs/r/j/job.json"
|
|
before = jj.read_bytes()
|
|
self.assertEqual(self.rows(), [])
|
|
pipeline.expand("r")
|
|
self.assertEqual(jj.read_bytes(), before)
|
|
self.assertEqual(len(self.rows()), 2)
|
|
self.assertFalse(list(jj.parent.glob(".job-*")))
|
|
|
|
def test_interrupted_different_content_refuses_without_rows(self):
|
|
self.spec()
|
|
with patch.object(db, "upsert_job", side_effect=RuntimeError):
|
|
with self.assertRaises(RuntimeError):
|
|
pipeline.expand("r")
|
|
jj = self.root / "runs/r/j/job.json"
|
|
before = jj.read_bytes()
|
|
self.gate = 9
|
|
with self.assertRaisesRegex(ValueError, "content conflict"):
|
|
pipeline.expand("r")
|
|
self.assertEqual(jj.read_bytes(), before)
|
|
self.assertEqual(self.rows(), [])
|
|
|
|
def test_missing_snapshot_for_existing_db_is_not_reconstructed(self):
|
|
self.spec()
|
|
with self.connect() as con:
|
|
db.upsert_job(con, "r", self.specs[0].to_row(), 1)
|
|
with self.assertRaisesRegex(ValueError, "no frozen job"):
|
|
pipeline.expand("r")
|
|
self.assertFalse((self.root / "runs/r/j/job.json").exists())
|
|
|
|
def test_concurrent_expands_publish_once(self):
|
|
self.spec(); self.spec(runner="opencode")
|
|
barrier = threading.Barrier(2)
|
|
errors = []
|
|
def expand():
|
|
try:
|
|
barrier.wait(timeout=5)
|
|
pipeline.expand("r")
|
|
except Exception as exc:
|
|
errors.append(exc)
|
|
threads = [threading.Thread(target=expand) for _ in range(2)]
|
|
for t in threads: t.start()
|
|
for t in threads: t.join(timeout=20)
|
|
self.assertFalse(any(t.is_alive() for t in threads))
|
|
self.assertEqual(errors, [])
|
|
self.assertEqual(len(self.rows()), 4)
|
|
|
|
def test_legacy_pi_plan_and_old_rows_remain_compatible(self):
|
|
self.spec()
|
|
jd = self.root / "runs/r/j"
|
|
jd.mkdir(parents=True)
|
|
old = self.snapshot(self.specs[0], "r", "unused", [{"mode": "e2e", "policy": "teacher", "k": 2}])
|
|
(jd / "job.json").write_text(json.dumps(old))
|
|
with self.connect() as con:
|
|
con.execute("INSERT INTO rollouts(run_id,job_id,kk,mode,policy,state) VALUES('r','j',7,'e2e','teacher','PASS')")
|
|
pipeline.expand("r")
|
|
self.assertEqual([r["kk"] for r in self.rows()], [7, 8])
|
|
self.assertEqual(json.loads((jd / "job.json").read_text()), old)
|
|
self.assertTrue(all(r["runner"] == "pi" for r in self.rows()))
|
|
|
|
def test_legacy_task_contract_defaults_include_shared_coco_module(self):
|
|
p = self.root / "legacy.json"
|
|
old = {"task": "pose", "rollout_plans": []}
|
|
p.write_text(json.dumps(old))
|
|
new = {**old, "task_contract": {"module": "harness.eval.coco_extra"}, "model_runtime": None,
|
|
"toolchain": {"profile": None, "version": "legacy-config"}}
|
|
pipeline._check_frozen_job(p, new)
|
|
new["toolchain"]["profile"] = "explicit-new-profile"
|
|
with self.assertRaisesRegex(ValueError, "content conflict"):
|
|
pipeline._check_frozen_job(p, new)
|
|
|
|
def test_legacy_runtime_equivalent_snapshot_is_idempotent_but_override_is_not(self):
|
|
p = self.root / "legacy-runtime.json"
|
|
old = {"task": "cls", "model_spec": {"runtime_python": "/old/venv/bin/python"}, "rollout_plans": []}
|
|
p.write_text(json.dumps(old))
|
|
new = {**old, "model_runtime": {"configured": "/old/venv/bin/python", "python": "/old/venv/bin/python"}}
|
|
pipeline._check_frozen_job(p, new)
|
|
new["model_runtime"] = {"configured": "/different/venv/bin/python", "python": "/different/venv/bin/python"}
|
|
with self.assertRaisesRegex(ValueError, "content conflict"):
|
|
pipeline._check_frozen_job(p, new)
|
|
self.assertEqual(json.loads(p.read_text()), old)
|
|
|
|
def test_new_plan_on_frozen_job_requires_new_run(self):
|
|
self.spec(); pipeline.expand("r")
|
|
self.spec(runner="opencode")
|
|
with self.assertRaisesRegex(ValueError, "plan conflict"):
|
|
pipeline.expand("r")
|
|
self.assertEqual(len(self.rows()), 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|