data-gen-bpu/scripts/test_agent_first.py

139 lines
7.2 KiB
Python

"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
from contextlib import ExitStack, nullcontext
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import cli, db, paths, pipeline, rollout
from harness.stages import Stage, StageOutcome
from harness.stages import s01_fetch
class AgentFirst(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("STATE", "RUNS", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
connect = db.connect
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
self.jd = paths.RUNS / "run/model"
(self.jd / "oracle").mkdir(parents=True)
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
self.save_job()
with db.connect() as con:
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
def save_job(self):
(self.jd / "job.json").write_text(json.dumps(self.job))
def execute(self, **kw):
self.called = []
def run(stage, ctx):
self.called.append(stage.name)
return StageOutcome("OK")
with patch.object(Stage, "run", run):
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
result = self.execute(prepare_only=True)
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
self.assertEqual(result["terminal"], "PREPARED")
self.assertNotIn("gates", result)
self.assertFalse((self.jd / "oracle/result.json").exists())
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
self.assertFalse(rollout.inputs_ready("run", "model")) # State alone never admits an Agent.
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
original = b'{"terminal":"PASS","evidence":"original"}'
(self.jd / "oracle/result.json").write_bytes(original)
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
with self.subTest(state=state):
with db.connect() as con:
db.set_job_state(con, "run", "model", state, result={"saved": True})
before = db.get_job(con, "run", "model")
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
self.assertEqual(self.called, [])
with db.connect() as con:
self.assertEqual(db.get_job(con, "run", "model"), before)
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
def test_dry_preparation_is_not_prepared(self):
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
model_spec={'input_spec': '1x3x224x224'},
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
self.job['data']['t1']['n'] = 100
self.job['data']['calib']['n'] = 128
self.save_job()
with patch.object(rollout.Baseline, 'run', side_effect=AssertionError('implicit model execution')):
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
self.assertIn('bench stage run export', prompt)
self.assertIn('bench stage run baseline', prompt)
self.assertFalse((self.jd/'preparation').exists())
self.assertFalse((self.jd/'rollouts').exists())
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
self.execute(prepare_only=True)
with patch.object(pipeline.gates, "evaluate", return_value={
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
self.execute()
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
self.assertTrue((self.jd / "oracle/result.json").exists())
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
with patch.object(pipeline, "run_oracle", return_value={}) as run:
cli.main(["run", "--run", "run"])
self.assertTrue(run.call_args.kwargs["prepare_only"])
cli.main(["run", "--run", "run", "--oracle"])
self.assertFalse(run.call_args.kwargs["prepare_only"])
def test_readiness_requires_versioned_inputs_not_oracle_result(self):
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
onnx = rollout.onnx_path(self.job)
onnx.parent.mkdir(parents=True)
onnx.touch()
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
baseline = rollout.baseline_dir(self.job) / "metrics.json"
baseline.parent.mkdir(parents=True)
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
baseline.write_text(json.dumps(meta))
Path(self.job["data"]["t1"]["path"]).touch()
calib = self.root / "calib/manifest.json"
calib.parent.mkdir()
calib.write_text('{}')
self.assertTrue(rollout.inputs_ready("run", "model"))
self.assertFalse((self.jd / "oracle/result.json").exists())
for file in (onnx, baseline, calib, Path(self.job["data"]["t1"]["path"])):
content = file.read_bytes()
file.unlink()
self.assertFalse(rollout.inputs_ready("run", "model"))
file.write_bytes(content)
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
self.assertFalse(rollout.inputs_ready("run", "model"))
baseline.write_text(json.dumps(meta))
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
self.assertFalse(rollout.inputs_ready("run", "model"))
if __name__ == "__main__":
unittest.main()