data-gen-bpu/scripts/test_agent_first.py
bench 8fb9349907 Agent no longer gets the platform's model code; each task states its I/O contract
The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every
bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates
included. That did the model-specific half of the deployment for it.

- guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site
- the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the
  graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract
- SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is
  replaced by the facts that differ from the docs
- e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass
- agent/VERSION 24

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 21:51:17 +08:00

235 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Preparation is reusable input evidence, never a synthetic Oracle verdict."""
from contextlib import ExitStack, nullcontext
from copy import deepcopy
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import cli, db, paths, pipeline, rollout
from harness.stages import Baseline, Export, Stage, StageContext, StageOutcome
from harness.stages import s01_fetch
class AgentFirst(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
for name in ("STATE", "RUNS", "CACHE"):
directory = self.root / name.lower()
directory.mkdir()
self.stack.enter_context(patch.object(paths, name, directory))
self.stack.enter_context(patch.object(db, "STATE", paths.STATE))
connect = db.connect
readonly = db.readonly
self.stack.enter_context(patch.object(db, "readonly", lambda: readonly(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite")))
self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json"))
self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()))
self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext()))
self.jd = paths.RUNS / "run/model"
(self.jd / "oracle").mkdir(parents=True)
self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls",
"evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k",
"t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"},
"calib": {"path": str(self.root / "calib")}}, "gates": {}}
self.ctx = StageContext("run", self.job, self.jd, self.jd / "oracle/stages", self.jd / "oracle/artifacts")
self.save_job()
with db.connect() as con:
con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')")
def save_job(self):
(self.jd / "job.json").write_text(json.dumps(self.job))
def execute(self, **kw):
self.called = []
def run(stage, ctx):
self.called.append(stage.name)
return StageOutcome("OK")
with patch.object(Stage, "run", run):
return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw)
def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self):
result = self.execute(prepare_only=True)
self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"])
self.assertEqual(result["terminal"], "PREPARED")
self.assertNotIn("gates", result)
self.assertFalse((self.jd / "oracle/result.json").exists())
self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result)
# Preparation state alone is not cache evidence.
self.assertIsNone(Baseline().cached(self.ctx))
self.assertIsNone(Export().cached(self.ctx))
def test_default_preparation_preserves_all_finished_states_even_when_forced(self):
original = b'{"terminal":"PASS","evidence":"original"}'
(self.jd / "oracle/result.json").write_bytes(original)
for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"):
with self.subTest(state=state):
with db.connect() as con:
db.set_job_state(con, "run", "model", state, result={"saved": True})
before = db.get_job(con, "run", "model")
self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED")
self.assertEqual(self.called, [])
with db.connect() as con:
self.assertEqual(db.get_job(con, "run", "model"), before)
self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original)
def test_dry_preparation_is_not_prepared(self):
self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED")
def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self):
self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8',
model_spec={'input_spec': '1x3x224x224'},
gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}})
self.job['data']['t1']['n'] = 100
self.job['data']['calib']['n'] = 128
self.save_job()
with patch.object(Stage, 'run', side_effect=AssertionError('implicit model execution')):
_, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True)
self.assertIn('bench stage run export', prompt)
self.assertIn('bench stage run baseline', prompt)
self.assertFalse((self.jd/'preparation').exists())
self.assertFalse((self.jd/'rollouts').exists())
def test_e2e_prompt_ends_with_the_model_familys_io_contract(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.assertNotIn("输入输出约定", prompt) # no contract written for this model
self.job["model"] = "yolox.s"
self.save_job()
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.assertIn("## 输入输出约定(YOLOX)", prompt)
self.assertIn("decode_in_inference = True", prompt)
def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
_, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"}
self.save_job()
for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"):
result.parent.mkdir(parents=True, exist_ok=True)
result.write_text("deliberately invalid historical JSON")
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
self.assertEqual(prompt, expected)
def test_recipe_prompt_uses_explicit_submission_and_fixed_reference(self):
self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them()
self.job["model_recipe"] = {"reference": {"entrypoint": "official/model.py",
"files": {"official/model.py": "1" * 64},
"weights_sha256": "2" * 64}}
self.save_job()
with patch.object(rollout, "onnx_path", side_effect=AssertionError("legacy cache lookup")), \
patch.object(rollout, "baseline_dir", side_effect=AssertionError("legacy cache lookup")):
_, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True)
for stage in ("baseline", "compile", "hbm", "board", "eval"):
self.assertIn(f"bench stage run {stage} --submission SUBMISSION_HANDLE", prompt)
self.assertIn("bench stage run export --recipe recipe.json", prompt)
self.assertIn("不是所有提交必经步骤", prompt)
self.assertIn("无需先跑基线", prompt)
self.assertIn("official/model.py", prompt)
self.assertIn("2" * 64, prompt)
self.assertIn("recipe_outputs", prompt)
self.assertIn("不是相对清单所在子目录", prompt)
self.assertIn("新的提交 handle", prompt)
self.assertIn("由 decode 恢复完整任务预测", prompt)
self.assertNotIn("ONNX 缓存位置", prompt)
self.assertNotIn("FP32 基线缓存位置", prompt)
self.assertNotIn(str(paths.CACHE), prompt)
self.assertFalse((self.jd / "rollouts").exists())
def test_recipe_system_snapshot_specializes_only_opted_in_task(self):
# Exercise the recipe-only opt-in independently of the installation's
# global private-home default, which managed collection enables.
config = deepcopy(paths.config())
config["rollouts"]["private_home"] = False
self.stack.enter_context(patch.object(paths, "config", return_value=config))
self.test_recipe_prompt_uses_explicit_submission_and_fixed_reference()
original = (paths.AGENT / "SYSTEM.md").read_bytes()
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
rd, _, _ = rollout.prepare("run", "model", 0, "e2e", "teacher")
system = (rd / "agent/system.md").read_text()
self.assertIn("探索次序由你决定", system)
self.assertIn("--submission SUBMISSION_HANDLE", system)
self.assertIn("workspace 内可 cd", system)
self.assertNotIn("最多重试", system)
self.assertNotIn("三次不同方法", system)
self.assertNotIn("legacy 沙箱", system)
self.assertNotIn("bench stage run export #", system)
self.assertNotIn("权重获取用 `bench stage run fetch`", system)
self.assertEqual((paths.AGENT / "SYSTEM.md").read_bytes(), original)
self.assertEqual(json.loads((rd / "rollout.json").read_text())["jail_policy"], "private-home")
provider = json.loads((rd / "workspace/.agent-home/.pi/agent/models.json").read_text())["providers"]
self.assertEqual(list(provider), ["dr-gateway"])
self.assertEqual(provider["dr-gateway"]["apiKey"], "BENCH_RUNNER_API_KEY")
self.job.pop("model_recipe")
self.save_job()
with patch("subprocess.run", return_value=SimpleNamespace(returncode=0, stdout="")):
legacy, _, _ = rollout.prepare("run", "model", 1, "e2e", "teacher")
self.assertEqual((legacy / "agent/system.md").read_bytes(), original)
self.assertNotIn("jail_policy", json.loads((legacy / "rollout.json").read_text()))
self.assertFalse((legacy / "workspace/.agent-home").exists())
def test_repair_rejected_before_any_claim_or_file_write(self):
with db.connect() as con:
db.rollout_upsert(con, "run", "model", 7, "repair", "teacher")
before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone())
for function in (rollout.prepare, rollout.run_rollout):
for dry_run in (False, True):
with self.subTest(function=function.__name__, dry_run=dry_run), \
patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")):
with self.assertRaisesRegex(ValueError, "repair collection is retired"):
function("run", "model", 7, "repair", "teacher", dry_run=dry_run)
with db.connect() as con:
self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before)
self.assertFalse((self.jd / "rollouts").exists())
def test_explicit_oracle_can_run_all_stages_after_preparation(self):
self.execute(prepare_only=True)
with patch.object(pipeline.gates, "evaluate", return_value={
"terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}):
self.execute()
self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER])
self.assertTrue((self.jd / "oracle/result.json").exists())
def test_cli_defaults_to_preparation_and_oracle_is_explicit(self):
with patch.object(pipeline, "run_oracle", return_value={}) as run:
cli.main(["run", "--run", "run"])
self.assertTrue(run.call_args.kwargs["prepare_only"])
cli.main(["run", "--run", "run", "--oracle"])
self.assertFalse(run.call_args.kwargs["prepare_only"])
def test_stage_caches_require_versioned_artifacts_not_oracle_result(self):
s01_fetch.INDEX.parent.mkdir(parents=True)
s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}')
onnx = rollout.onnx_path(self.job)
onnx.parent.mkdir(parents=True)
onnx.touch()
(onnx.parent / "meta.json").write_text('{"exporter_version":"2"}')
baseline = rollout.baseline_dir(self.job) / "metrics.json"
baseline.parent.mkdir(parents=True)
meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}}
baseline.write_text(json.dumps(meta))
self.assertIsNotNone(Baseline().cached(self.ctx))
self.assertIsNotNone(Export().cached(self.ctx))
self.assertFalse((self.jd / "oracle/result.json").exists())
for stage, file in ((Export(), onnx), (Baseline(), baseline)):
content = file.read_bytes()
file.unlink()
self.assertIsNone(stage.cached(self.ctx))
file.write_bytes(content)
baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"}))
self.assertIsNone(Baseline().cached(self.ctx))
baseline.write_text(json.dumps(meta))
(onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}')
self.assertIsNone(Export().cached(self.ctx))
if __name__ == "__main__":
unittest.main()