data-gen-bpu/scripts/test_agent_first.py

225 lines
13 KiB
Python

"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
from contextlib import ExitStack, nullcontext
from copy import deepcopy
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import cli, db, paths, pipeline, rollout
from harness.stages import Baseline, Export, Stage, StageContext, StageOutcome
from harness.stages import s01_fetch
class AgentFirst(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("STATE", "RUNS", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
connect = db.connect
readonly = db.readonly
self.stack.enter_context(patch.object(db, "readonly", lambda: readonly(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
self.jd = paths.RUNS / "run/model"
(self.jd / "oracle").mkdir(parents=True)
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
self.ctx = StageContext("run", self.job, self.jd, self.jd / "oracle/stages", self.jd / "oracle/artifacts")
self.save_job()
with db.connect() as con:
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
def save_job(self):
(self.jd / "job.json").write_text(json.dumps(self.job))
def execute(self, **kw):
self.called = []
def run(stage, ctx):
self.called.append(stage.name)
return StageOutcome("OK")
with patch.object(Stage, "run", run):
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
result = self.execute(prepare_only=True)
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
self.assertEqual(result["terminal"], "PREPARED")
self.assertNotIn("gates", result)
self.assertFalse((self.jd / "oracle/result.json").exists())
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
# Preparation state alone is not cache evidence.
self.assertIsNone(Baseline().cached(self.ctx))
self.assertIsNone(Export().cached(self.ctx))
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
original = b'{"terminal":"PASS","evidence":"original"}'
(self.jd / "oracle/result.json").write_bytes(original)
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
with self.subTest(state=state):
with db.connect() as con:
db.set_job_state(con, "run", "model", state, result={"saved": True})
before = db.get_job(con, "run", "model")
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
self.assertEqual(self.called, [])
with db.connect() as con:
self.assertEqual(db.get_job(con, "run", "model"), before)
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
def test_dry_preparation_is_not_prepared(self):
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
model_spec={'input_spec': '1x3x224x224'},
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
self.job['data']['t1']['n'] = 100
self.job['data']['calib']['n'] = 128
self.save_job()
with patch.object(Stage, 'run', side_effect=AssertionError('implicit model execution')):
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
self.assertIn('bench stage run export', prompt)
self.assertIn('bench stage run baseline', prompt)
self.assertFalse((self.jd/'preparation').exists())
self.assertFalse((self.jd/'rollouts').exists())
def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
_, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"}
self.save_job()
for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"):
result.parent.mkdir(parents=True, exist_ok=True)
result.write_text("deliberately invalid historical JSON")
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.assertEqual(prompt, expected)
def test_recipe_prompt_uses_explicit_submission_and_fixed_reference(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
self.job["model_recipe"] = {"reference": {"entrypoint": "official/model.py",
"files": {"official/model.py": "1" * 64},
"weights_sha256": "2" * 64}}
self.save_job()
with patch.object(rollout, "onnx_path", side_effect=AssertionError("legacy cache lookup")), \
patch.object(rollout, "baseline_dir", side_effect=AssertionError("legacy cache lookup")):
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
for stage in ("baseline", "compile", "hbm", "board", "eval"):
self.assertIn(f"bench stage run {stage} --submission SUBMISSION_HANDLE", prompt)
self.assertIn("bench stage run export --recipe recipe.json", prompt)
self.assertIn("不是所有提交必经步骤", prompt)
self.assertIn("无需先跑基线", prompt)
self.assertIn("official/model.py", prompt)
self.assertIn("2" * 64, prompt)
self.assertIn("recipe_outputs", prompt)
self.assertIn("不是相对清单所在子目录", prompt)
self.assertIn("新的提交 handle", prompt)
self.assertIn("由 decode 恢复完整任务预测", prompt)
self.assertNotIn("ONNX 缓存位置", prompt)
self.assertNotIn("FP32 基线缓存位置", prompt)
self.assertNotIn(str(paths.CACHE), prompt)
self.assertFalse((self.jd / "rollouts").exists())
def test_recipe_system_snapshot_specializes_only_opted_in_task(self):
# Exercise the recipe-only opt-in independently of the installation's
# global private-home default, which managed collection enables.
config = deepcopy(paths.config())
config["rollouts"]["private_home"] = False
self.stack.enter_context(patch.object(paths, "config", return_value=config))
self.test_recipe_prompt_uses_explicit_submission_and_fixed_reference()
original = (paths.AGENT / "SYSTEM.md").read_bytes()
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
rd, _, _ = rollout.prepare("run", "model", 0, "e2e", "teacher")
system = (rd / "agent/system.md").read_text()
self.assertIn("探索次序由你决定", system)
self.assertIn("--submission SUBMISSION_HANDLE", system)
self.assertIn("workspace 内可 cd", system)
self.assertNotIn("最多重试", system)
self.assertNotIn("三次不同方法", system)
self.assertNotIn("legacy 沙箱", system)
self.assertNotIn("bench stage run export #", system)
self.assertNotIn("权重获取用 `bench stage run fetch`", system)
self.assertEqual((paths.AGENT / "SYSTEM.md").read_bytes(), original)
self.assertEqual(json.loads((rd / "rollout.json").read_text())["jail_policy"], "private-home")
provider = json.loads((rd / "workspace/.agent-home/.pi/agent/models.json").read_text())["providers"]
self.assertEqual(list(provider), ["dr-gateway"])
self.assertEqual(provider["dr-gateway"]["apiKey"], "BENCH_RUNNER_API_KEY")
self.job.pop("model_recipe")
self.save_job()
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
legacy, _, _ = rollout.prepare("run", "model", 1, "e2e", "teacher")
self.assertEqual((legacy / "agent/system.md").read_bytes(), original)
self.assertNotIn("jail_policy", json.loads((legacy / "rollout.json").read_text()))
self.assertFalse((legacy / "workspace/.agent-home").exists())
def test_repair_rejected_before_any_claim_or_file_write(self):
with db.connect() as con:
db.rollout_upsert(con, "run", "model", 7, "repair", "teacher")
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
for function in (rollout.prepare, rollout.run_rollout):
for dry_run in (False, True):
with self.subTest(function=function.__name__, dry_run=dry_run), \
patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")):
with self.assertRaisesRegex(ValueError, "repair collection is retired"):
function("run", "model", 7, "repair", "teacher", dry_run=dry_run)
with db.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
self.assertFalse((self.jd / "rollouts").exists())
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
self.execute(prepare_only=True)
with patch.object(pipeline.gates, "evaluate", return_value={
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
self.execute()
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
self.assertTrue((self.jd / "oracle/result.json").exists())
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
with patch.object(pipeline, "run_oracle", return_value={}) as run:
cli.main(["run", "--run", "run"])
self.assertTrue(run.call_args.kwargs["prepare_only"])
cli.main(["run", "--run", "run", "--oracle"])
self.assertFalse(run.call_args.kwargs["prepare_only"])
def test_stage_caches_require_versioned_artifacts_not_oracle_result(self):
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
onnx = rollout.onnx_path(self.job)
onnx.parent.mkdir(parents=True)
onnx.touch()
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
baseline = rollout.baseline_dir(self.job) / "metrics.json"
baseline.parent.mkdir(parents=True)
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
baseline.write_text(json.dumps(meta))
self.assertIsNotNone(Baseline().cached(self.ctx))
self.assertIsNotNone(Export().cached(self.ctx))
self.assertFalse((self.jd / "oracle/result.json").exists())
for stage, file in ((Export(), onnx), (Baseline(), baseline)):
content = file.read_bytes()
file.unlink()
self.assertIsNone(stage.cached(self.ctx))
file.write_bytes(content)
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
self.assertIsNone(Baseline().cached(self.ctx))
baseline.write_text(json.dumps(meta))
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
self.assertIsNone(Export().cached(self.ctx))
if __name__ == "__main__":
unittest.main()