225 lines
13 KiB
Python
225 lines
13 KiB
Python
"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
|
|
from contextlib import ExitStack, nullcontext
|
|
from copy import deepcopy
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import cli, db, paths, pipeline, rollout
|
|
from harness.stages import Baseline, Export, Stage, StageContext, StageOutcome
|
|
from harness.stages import s01_fetch
|
|
|
|
|
|
class AgentFirst(unittest.TestCase):
|
|
def setUp(self):
|
|
tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(tmp.cleanup)
|
|
self.root = Path(tmp.name)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
for name in ("STATE", "RUNS", "CACHE"):
|
|
directory = self.root / name.lower()
|
|
directory.mkdir()
|
|
self.stack.enter_context(patch.object(paths, name, directory))
|
|
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
|
|
connect = db.connect
|
|
readonly = db.readonly
|
|
self.stack.enter_context(patch.object(db, "readonly", lambda: readonly(paths.STATE / "jobs.sqlite")))
|
|
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
|
|
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
|
|
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
|
|
self.jd = paths.RUNS / "run/model"
|
|
(self.jd / "oracle").mkdir(parents=True)
|
|
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
|
|
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
|
|
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
|
|
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
|
|
self.ctx = StageContext("run", self.job, self.jd, self.jd / "oracle/stages", self.jd / "oracle/artifacts")
|
|
self.save_job()
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
|
|
|
|
def save_job(self):
|
|
(self.jd / "job.json").write_text(json.dumps(self.job))
|
|
|
|
def execute(self, **kw):
|
|
self.called = []
|
|
def run(stage, ctx):
|
|
self.called.append(stage.name)
|
|
return StageOutcome("OK")
|
|
with patch.object(Stage, "run", run):
|
|
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
|
|
|
|
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
|
|
result = self.execute(prepare_only=True)
|
|
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
|
|
self.assertEqual(result["terminal"], "PREPARED")
|
|
self.assertNotIn("gates", result)
|
|
self.assertFalse((self.jd / "oracle/result.json").exists())
|
|
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
|
|
# Preparation state alone is not cache evidence.
|
|
self.assertIsNone(Baseline().cached(self.ctx))
|
|
self.assertIsNone(Export().cached(self.ctx))
|
|
|
|
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
|
|
original = b'{"terminal":"PASS","evidence":"original"}'
|
|
(self.jd / "oracle/result.json").write_bytes(original)
|
|
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
|
|
with self.subTest(state=state):
|
|
with db.connect() as con:
|
|
db.set_job_state(con, "run", "model", state, result={"saved": True})
|
|
before = db.get_job(con, "run", "model")
|
|
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
|
|
self.assertEqual(self.called, [])
|
|
with db.connect() as con:
|
|
self.assertEqual(db.get_job(con, "run", "model"), before)
|
|
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
|
|
|
|
def test_dry_preparation_is_not_prepared(self):
|
|
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
|
|
|
|
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
|
|
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
|
|
model_spec={'input_spec': '1x3x224x224'},
|
|
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
|
|
self.job['data']['t1']['n'] = 100
|
|
self.job['data']['calib']['n'] = 128
|
|
self.save_job()
|
|
with patch.object(Stage, 'run', side_effect=AssertionError('implicit model execution')):
|
|
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
|
|
self.assertIn('bench stage run export', prompt)
|
|
self.assertIn('bench stage run baseline', prompt)
|
|
self.assertFalse((self.jd/'preparation').exists())
|
|
self.assertFalse((self.jd/'rollouts').exists())
|
|
|
|
def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self):
|
|
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
|
|
_, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
|
|
self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"}
|
|
self.save_job()
|
|
for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"):
|
|
result.parent.mkdir(parents=True, exist_ok=True)
|
|
result.write_text("deliberately invalid historical JSON")
|
|
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
|
|
self.assertEqual(prompt, expected)
|
|
|
|
def test_recipe_prompt_uses_explicit_submission_and_fixed_reference(self):
|
|
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
|
|
self.job["model_recipe"] = {"reference": {"entrypoint": "official/model.py",
|
|
"files": {"official/model.py": "1" * 64},
|
|
"weights_sha256": "2" * 64}}
|
|
self.save_job()
|
|
with patch.object(rollout, "onnx_path", side_effect=AssertionError("legacy cache lookup")), \
|
|
patch.object(rollout, "baseline_dir", side_effect=AssertionError("legacy cache lookup")):
|
|
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
|
|
for stage in ("baseline", "compile", "hbm", "board", "eval"):
|
|
self.assertIn(f"bench stage run {stage} --submission SUBMISSION_HANDLE", prompt)
|
|
self.assertIn("bench stage run export --recipe recipe.json", prompt)
|
|
self.assertIn("不是所有提交必经步骤", prompt)
|
|
self.assertIn("无需先跑基线", prompt)
|
|
self.assertIn("official/model.py", prompt)
|
|
self.assertIn("2" * 64, prompt)
|
|
self.assertIn("recipe_outputs", prompt)
|
|
self.assertIn("不是相对清单所在子目录", prompt)
|
|
self.assertIn("新的提交 handle", prompt)
|
|
self.assertIn("由 decode 恢复完整任务预测", prompt)
|
|
self.assertNotIn("ONNX 缓存位置", prompt)
|
|
self.assertNotIn("FP32 基线缓存位置", prompt)
|
|
self.assertNotIn(str(paths.CACHE), prompt)
|
|
self.assertFalse((self.jd / "rollouts").exists())
|
|
|
|
def test_recipe_system_snapshot_specializes_only_opted_in_task(self):
|
|
# Exercise the recipe-only opt-in independently of the installation's
|
|
# global private-home default, which managed collection enables.
|
|
config = deepcopy(paths.config())
|
|
config["rollouts"]["private_home"] = False
|
|
self.stack.enter_context(patch.object(paths, "config", return_value=config))
|
|
self.test_recipe_prompt_uses_explicit_submission_and_fixed_reference()
|
|
original = (paths.AGENT / "SYSTEM.md").read_bytes()
|
|
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
|
|
rd, _, _ = rollout.prepare("run", "model", 0, "e2e", "teacher")
|
|
system = (rd / "agent/system.md").read_text()
|
|
self.assertIn("探索次序由你决定", system)
|
|
self.assertIn("--submission SUBMISSION_HANDLE", system)
|
|
self.assertIn("workspace 内可 cd", system)
|
|
self.assertNotIn("最多重试", system)
|
|
self.assertNotIn("三次不同方法", system)
|
|
self.assertNotIn("legacy 沙箱", system)
|
|
self.assertNotIn("bench stage run export #", system)
|
|
self.assertNotIn("权重获取用 `bench stage run fetch`", system)
|
|
self.assertEqual((paths.AGENT / "SYSTEM.md").read_bytes(), original)
|
|
self.assertEqual(json.loads((rd / "rollout.json").read_text())["jail_policy"], "private-home")
|
|
provider = json.loads((rd / "workspace/.agent-home/.pi/agent/models.json").read_text())["providers"]
|
|
self.assertEqual(list(provider), ["dr-gateway"])
|
|
self.assertEqual(provider["dr-gateway"]["apiKey"], "BENCH_RUNNER_API_KEY")
|
|
self.job.pop("model_recipe")
|
|
self.save_job()
|
|
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
|
|
legacy, _, _ = rollout.prepare("run", "model", 1, "e2e", "teacher")
|
|
self.assertEqual((legacy / "agent/system.md").read_bytes(), original)
|
|
self.assertNotIn("jail_policy", json.loads((legacy / "rollout.json").read_text()))
|
|
self.assertFalse((legacy / "workspace/.agent-home").exists())
|
|
|
|
def test_repair_rejected_before_any_claim_or_file_write(self):
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, "run", "model", 7, "repair", "teacher")
|
|
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
|
|
for function in (rollout.prepare, rollout.run_rollout):
|
|
for dry_run in (False, True):
|
|
with self.subTest(function=function.__name__, dry_run=dry_run), \
|
|
patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")):
|
|
with self.assertRaisesRegex(ValueError, "repair collection is retired"):
|
|
function("run", "model", 7, "repair", "teacher", dry_run=dry_run)
|
|
with db.connect() as con:
|
|
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
|
|
self.assertFalse((self.jd / "rollouts").exists())
|
|
|
|
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
|
|
self.execute(prepare_only=True)
|
|
with patch.object(pipeline.gates, "evaluate", return_value={
|
|
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
|
|
self.execute()
|
|
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
|
|
self.assertTrue((self.jd / "oracle/result.json").exists())
|
|
|
|
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
|
|
with patch.object(pipeline, "run_oracle", return_value={}) as run:
|
|
cli.main(["run", "--run", "run"])
|
|
self.assertTrue(run.call_args.kwargs["prepare_only"])
|
|
cli.main(["run", "--run", "run", "--oracle"])
|
|
self.assertFalse(run.call_args.kwargs["prepare_only"])
|
|
|
|
def test_stage_caches_require_versioned_artifacts_not_oracle_result(self):
|
|
s01_fetch.INDEX.parent.mkdir(parents=True)
|
|
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
|
|
onnx = rollout.onnx_path(self.job)
|
|
onnx.parent.mkdir(parents=True)
|
|
onnx.touch()
|
|
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
|
|
baseline = rollout.baseline_dir(self.job) / "metrics.json"
|
|
baseline.parent.mkdir(parents=True)
|
|
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
|
|
baseline.write_text(json.dumps(meta))
|
|
self.assertIsNotNone(Baseline().cached(self.ctx))
|
|
self.assertIsNotNone(Export().cached(self.ctx))
|
|
self.assertFalse((self.jd / "oracle/result.json").exists())
|
|
for stage, file in ((Export(), onnx), (Baseline(), baseline)):
|
|
content = file.read_bytes()
|
|
file.unlink()
|
|
self.assertIsNone(stage.cached(self.ctx))
|
|
file.write_bytes(content)
|
|
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
|
|
self.assertIsNone(Baseline().cached(self.ctx))
|
|
baseline.write_text(json.dumps(meta))
|
|
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
|
|
self.assertIsNone(Export().cached(self.ctx))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|