139 lines
7.2 KiB
Python
139 lines
7.2 KiB
Python
"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
|
|
from contextlib import ExitStack, nullcontext
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import cli, db, paths, pipeline, rollout
|
|
from harness.stages import Stage, StageOutcome
|
|
from harness.stages import s01_fetch
|
|
|
|
|
|
class AgentFirst(unittest.TestCase):
|
|
def setUp(self):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
self.root = Path(tmp.name)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
for name in ("STATE", "RUNS", "CACHE"):
|
|
directory = self.root / name.lower()
|
|
directory.mkdir()
|
|
self.stack.enter_context(patch.object(paths, name, directory))
|
|
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
|
|
connect = db.connect
|
|
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
|
|
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
|
|
self.jd = paths.RUNS / "run/model"
|
|
(self.jd / "oracle").mkdir(parents=True)
|
|
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
|
|
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
|
|
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
|
|
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
|
|
self.save_job()
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
|
|
|
|
def save_job(self):
|
|
(self.jd / "job.json").write_text(json.dumps(self.job))
|
|
|
|
def execute(self, **kw):
|
|
self.called = []
|
|
def run(stage, ctx):
|
|
self.called.append(stage.name)
|
|
return StageOutcome("OK")
|
|
with patch.object(Stage, "run", run):
|
|
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
|
|
|
|
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
|
|
result = self.execute(prepare_only=True)
|
|
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
|
|
self.assertEqual(result["terminal"], "PREPARED")
|
|
self.assertNotIn("gates", result)
|
|
self.assertFalse((self.jd / "oracle/result.json").exists())
|
|
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
|
|
self.assertFalse(rollout.inputs_ready("run", "model")) # State alone never admits an Agent.
|
|
|
|
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
|
|
original = b'{"terminal":"PASS","evidence":"original"}'
|
|
(self.jd / "oracle/result.json").write_bytes(original)
|
|
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
|
|
with self.subTest(state=state):
|
|
with db.connect() as con:
|
|
db.set_job_state(con, "run", "model", state, result={"saved": True})
|
|
before = db.get_job(con, "run", "model")
|
|
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
|
|
self.assertEqual(self.called, [])
|
|
with db.connect() as con:
|
|
self.assertEqual(db.get_job(con, "run", "model"), before)
|
|
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
|
|
|
|
def test_dry_preparation_is_not_prepared(self):
|
|
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
|
|
|
|
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
|
|
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
|
|
model_spec={'input_spec': '1x3x224x224'},
|
|
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
|
|
self.job['data']['t1']['n'] = 100
|
|
self.job['data']['calib']['n'] = 128
|
|
self.save_job()
|
|
with patch.object(rollout.Baseline, 'run', side_effect=AssertionError('implicit model execution')):
|
|
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
|
|
self.assertIn('bench stage run export', prompt)
|
|
self.assertIn('bench stage run baseline', prompt)
|
|
self.assertFalse((self.jd/'preparation').exists())
|
|
self.assertFalse((self.jd/'rollouts').exists())
|
|
|
|
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
|
|
self.execute(prepare_only=True)
|
|
with patch.object(pipeline.gates, "evaluate", return_value={
|
|
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
|
|
self.execute()
|
|
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
|
|
self.assertTrue((self.jd / "oracle/result.json").exists())
|
|
|
|
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
|
|
with patch.object(pipeline, "run_oracle", return_value={}) as run:
|
|
cli.main(["run", "--run", "run"])
|
|
self.assertTrue(run.call_args.kwargs["prepare_only"])
|
|
cli.main(["run", "--run", "run", "--oracle"])
|
|
self.assertFalse(run.call_args.kwargs["prepare_only"])
|
|
|
|
def test_readiness_requires_versioned_inputs_not_oracle_result(self):
|
|
s01_fetch.INDEX.parent.mkdir(parents=True)
|
|
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
|
|
onnx = rollout.onnx_path(self.job)
|
|
onnx.parent.mkdir(parents=True)
|
|
onnx.touch()
|
|
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
|
|
baseline = rollout.baseline_dir(self.job) / "metrics.json"
|
|
baseline.parent.mkdir(parents=True)
|
|
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
|
|
baseline.write_text(json.dumps(meta))
|
|
Path(self.job["data"]["t1"]["path"]).touch()
|
|
calib = self.root / "calib/manifest.json"
|
|
calib.parent.mkdir()
|
|
calib.write_text('{}')
|
|
self.assertTrue(rollout.inputs_ready("run", "model"))
|
|
self.assertFalse((self.jd / "oracle/result.json").exists())
|
|
for file in (onnx, baseline, calib, Path(self.job["data"]["t1"]["path"])):
|
|
content = file.read_bytes()
|
|
file.unlink()
|
|
self.assertFalse(rollout.inputs_ready("run", "model"))
|
|
file.write_bytes(content)
|
|
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
|
|
self.assertFalse(rollout.inputs_ready("run", "model"))
|
|
baseline.write_text(json.dumps(meta))
|
|
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
|
|
self.assertFalse(rollout.inputs_ready("run", "model"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|