data-gen-bpu/scripts/test_agent_first.py

164 lines
8.8 KiB
Python

"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
from contextlib import ExitStack, nullcontext
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import cli, db, paths, pipeline, rollout
from harness.stages import Stage, StageOutcome
from harness.stages import s01_fetch
class AgentFirst(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("STATE", "RUNS", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
connect = db.connect
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
self.jd = paths.RUNS / "run/model"
(self.jd / "oracle").mkdir(parents=True)
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
self.save_job()
with db.connect() as con:
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
def save_job(self):
(self.jd / "job.json").write_text(json.dumps(self.job))
def execute(self, **kw):
self.called = []
def run(stage, ctx):
self.called.append(stage.name)
return StageOutcome("OK")
with patch.object(Stage, "run", run):
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
result = self.execute(prepare_only=True)
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
self.assertEqual(result["terminal"], "PREPARED")
self.assertNotIn("gates", result)
self.assertFalse((self.jd / "oracle/result.json").exists())
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
self.assertFalse(rollout.inputs_ready("run", "model")) # State alone never admits an Agent.
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
original = b'{"terminal":"PASS","evidence":"original"}'
(self.jd / "oracle/result.json").write_bytes(original)
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
with self.subTest(state=state):
with db.connect() as con:
db.set_job_state(con, "run", "model", state, result={"saved": True})
before = db.get_job(con, "run", "model")
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
self.assertEqual(self.called, [])
with db.connect() as con:
self.assertEqual(db.get_job(con, "run", "model"), before)
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
def test_dry_preparation_is_not_prepared(self):
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
model_spec={'input_spec': '1x3x224x224'},
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
self.job['data']['t1']['n'] = 100
self.job['data']['calib']['n'] = 128
self.save_job()
with patch.object(rollout.Baseline, 'run', side_effect=AssertionError('implicit model execution')):
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
self.assertIn('bench stage run export', prompt)
self.assertIn('bench stage run baseline', prompt)
self.assertFalse((self.jd/'preparation').exists())
self.assertFalse((self.jd/'rollouts').exists())
def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
_, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"}
self.save_job()
for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"):
result.parent.mkdir(parents=True, exist_ok=True)
result.write_text("deliberately invalid historical JSON")
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.assertEqual(prompt, expected)
def test_repair_rejected_before_any_claim_or_file_write(self):
with db.connect() as con:
db.rollout_upsert(con, "run", "model", 7, "repair", "teacher")
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
for function in (rollout.prepare, rollout.run_rollout):
for dry_run in (False, True):
with self.subTest(function=function.__name__, dry_run=dry_run), \
patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")):
with self.assertRaisesRegex(ValueError, "repair collection is retired"):
function("run", "model", 7, "repair", "teacher", dry_run=dry_run)
with db.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
self.assertFalse((self.jd / "rollouts").exists())
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
self.execute(prepare_only=True)
with patch.object(pipeline.gates, "evaluate", return_value={
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
self.execute()
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
self.assertTrue((self.jd / "oracle/result.json").exists())
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
with patch.object(pipeline, "run_oracle", return_value={}) as run:
cli.main(["run", "--run", "run"])
self.assertTrue(run.call_args.kwargs["prepare_only"])
cli.main(["run", "--run", "run", "--oracle"])
self.assertFalse(run.call_args.kwargs["prepare_only"])
def test_readiness_requires_versioned_inputs_not_oracle_result(self):
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
onnx = rollout.onnx_path(self.job)
onnx.parent.mkdir(parents=True)
onnx.touch()
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
baseline = rollout.baseline_dir(self.job) / "metrics.json"
baseline.parent.mkdir(parents=True)
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
baseline.write_text(json.dumps(meta))
Path(self.job["data"]["t1"]["path"]).touch()
calib = self.root / "calib/manifest.json"
calib.parent.mkdir()
calib.write_text('{}')
self.assertTrue(rollout.inputs_ready("run", "model"))
self.assertFalse((self.jd / "oracle/result.json").exists())
for file in (onnx, baseline, calib, Path(self.job["data"]["t1"]["path"])):
content = file.read_bytes()
file.unlink()
self.assertFalse(rollout.inputs_ready("run", "model"))
file.write_bytes(content)
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
self.assertFalse(rollout.inputs_ready("run", "model"))
baseline.write_text(json.dumps(meta))
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
self.assertFalse(rollout.inputs_ready("run", "model"))
if __name__ == "__main__":
unittest.main()