"""Preparation is reusable input evidence, never a synthetic Oracle verdict.""" from contextlib import ExitStack, nullcontext import json from pathlib import Path import tempfile import unittest from unittest.mock import patch from harness import cli, db, paths, pipeline, rollout from harness.stages import Stage, StageOutcome from harness.stages import s01_fetch class AgentFirst(unittest.TestCase): def setUp(self): tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) self.root = Path(tmp.name) self.stack = ExitStack() self.addCleanup(self.stack.close) for name in ("STATE", "RUNS", "CACHE"): directory = self.root / name.lower() directory.mkdir() self.stack.enter_context(patch.object(paths, name, directory)) self.stack.enter_context(patch.object(db, "STATE", paths.STATE)) connect = db.connect self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite"))) self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json")) self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext())) self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext())) self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext())) self.jd = paths.RUNS / "run/model" (self.jd / "oracle").mkdir(parents=True) self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls", "evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k", "t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"}, "calib": {"path": str(self.root / "calib")}}, "gates": {}} self.save_job() with db.connect() as con: con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')") def save_job(self): (self.jd / "job.json").write_text(json.dumps(self.job)) def execute(self, **kw): self.called = [] def run(stage, ctx): self.called.append(stage.name) return StageOutcome("OK") with patch.object(Stage, "run", run): return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw) def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self): result = self.execute(prepare_only=True) self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"]) self.assertEqual(result["terminal"], "PREPARED") self.assertNotIn("gates", result) self.assertFalse((self.jd / "oracle/result.json").exists()) self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result) self.assertFalse(rollout.inputs_ready("run", "model")) # State alone never admits an Agent. def test_default_preparation_preserves_all_finished_states_even_when_forced(self): original = b'{"terminal":"PASS","evidence":"original"}' (self.jd / "oracle/result.json").write_bytes(original) for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"): with self.subTest(state=state): with db.connect() as con: db.set_job_state(con, "run", "model", state, result={"saved": True}) before = db.get_job(con, "run", "model") self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED") self.assertEqual(self.called, []) with db.connect() as con: self.assertEqual(db.get_job(con, "run", "model"), before) self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original) def test_dry_preparation_is_not_prepared(self): self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED") def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self): self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8', model_spec={'input_spec': '1x3x224x224'}, gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}}) self.job['data']['t1']['n'] = 100 self.job['data']['calib']['n'] = 128 self.save_job() with patch.object(rollout.Baseline, 'run', side_effect=AssertionError('implicit model execution')): _, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True) self.assertIn('bench stage run export', prompt) self.assertIn('bench stage run baseline', prompt) self.assertFalse((self.jd/'preparation').exists()) self.assertFalse((self.jd/'rollouts').exists()) def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self): self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them() _, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True) self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"} self.save_job() for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"): result.parent.mkdir(parents=True, exist_ok=True) result.write_text("deliberately invalid historical JSON") _, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True) self.assertEqual(prompt, expected) def test_repair_rejected_before_any_claim_or_file_write(self): with db.connect() as con: db.rollout_upsert(con, "run", "model", 7, "repair", "teacher") before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()) for function in (rollout.prepare, rollout.run_rollout): for dry_run in (False, True): with self.subTest(function=function.__name__, dry_run=dry_run), \ patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")): with self.assertRaisesRegex(ValueError, "repair collection is retired"): function("run", "model", 7, "repair", "teacher", dry_run=dry_run) with db.connect() as con: self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before) self.assertFalse((self.jd / "rollouts").exists()) def test_explicit_oracle_can_run_all_stages_after_preparation(self): self.execute(prepare_only=True) with patch.object(pipeline.gates, "evaluate", return_value={ "terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}): self.execute() self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER]) self.assertTrue((self.jd / "oracle/result.json").exists()) def test_cli_defaults_to_preparation_and_oracle_is_explicit(self): with patch.object(pipeline, "run_oracle", return_value={}) as run: cli.main(["run", "--run", "run"]) self.assertTrue(run.call_args.kwargs["prepare_only"]) cli.main(["run", "--run", "run", "--oracle"]) self.assertFalse(run.call_args.kwargs["prepare_only"]) def test_readiness_requires_versioned_inputs_not_oracle_result(self): s01_fetch.INDEX.parent.mkdir(parents=True) s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}') onnx = rollout.onnx_path(self.job) onnx.parent.mkdir(parents=True) onnx.touch() (onnx.parent / "meta.json").write_text('{"exporter_version":"2"}') baseline = rollout.baseline_dir(self.job) / "metrics.json" baseline.parent.mkdir(parents=True) meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}} baseline.write_text(json.dumps(meta)) Path(self.job["data"]["t1"]["path"]).touch() calib = self.root / "calib/manifest.json" calib.parent.mkdir() calib.write_text('{}') self.assertTrue(rollout.inputs_ready("run", "model")) self.assertFalse((self.jd / "oracle/result.json").exists()) for file in (onnx, baseline, calib, Path(self.job["data"]["t1"]["path"])): content = file.read_bytes() file.unlink() self.assertFalse(rollout.inputs_ready("run", "model")) file.write_bytes(content) baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"})) self.assertFalse(rollout.inputs_ready("run", "model")) baseline.write_text(json.dumps(meta)) (onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}') self.assertFalse(rollout.inputs_ready("run", "model")) if __name__ == "__main__": unittest.main()