data-gen-bpu/scripts/test_preflight_extensions.py

122 lines
6.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Runner/policy/profile admission is checked without compilation or model work."""
from contextlib import ExitStack
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import Mock, patch
from harness import db, pipeline, registry, runners
class PreflightExtensions(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.root = Path(self.tmp.name)
self.python = self.root / "profile/bin/python"
self.python.parent.mkdir(parents=True)
self.python.write_text("fixture only; never executed")
(self.root / "horizon").mkdir()
for directory in (self.root / "data/pool/fixture/v1", self.root / "data/calib/cls/input"):
directory.mkdir(parents=True)
(directory / "manifest.json").write_text("{}")
self.cfg = {"paths": {"venv": {"ptq": str(self.root / "profile")},
"horizon": str(self.root / "horizon"), "pi_bin": str(self.root / "missing-pi")},
"endpoints": {"teacher": {"api_key_env": "TEACHER_KEY"},
"student": {"api_key_env": "STUDENT_KEY"}}}
self.secrets = {"BOARD_SSH_KEY": "fixture", "STUDENT_KEY": "fixture"}
self.spec = SimpleNamespace(mode="e2e", runner="opencode", policy="student", model="m", task="cls",
march="nash-m", backend="ptq", qcfg="int8", overrides={})
row = {"model_id": "m", "dataset": "fixture", "input_spec": "input"}
for obj, name, value in ((pipeline.paths, "config", lambda: self.cfg),
(pipeline.paths, "secret", lambda key: self.secrets.get(key)),
(pipeline.paths, "PROJECT_ROOT", self.root),
(pipeline.paths, "DATA", self.root / "data"),
(pipeline, "models", lambda: [row]),
(pipeline, "model_row", lambda _: row),
(pipeline, "expand_matrix", lambda *a, **kw: [self.spec]),
(pipeline, "datasets", lambda: {"datasets": {"fixture": {"version": "v1", "t1": {"n": 0}}}})):
self.stack.enter_context(patch.object(obj, name, value))
self.check = Mock()
self.get_runner = self.stack.enter_context(patch.object(runners, "get", return_value=SimpleNamespace(check=self.check)))
def test_opencode_student_does_not_require_pi_or_teacher_credentials(self):
errors, _ = pipeline.preflight()
self.assertEqual(errors, [])
self.get_runner.assert_called_once_with("opencode")
self.check.assert_called_once_with("student", execute=True)
def test_pi_still_requires_its_binary(self):
self.spec.runner = "pi"
errors, _ = pipeline.preflight()
self.assertEqual(len(errors), 1)
self.assertIn("pi 不在 PATH", errors[0])
def test_runner_disabled_or_wrong_version_is_an_error(self):
self.check.side_effect = ValueError("opencode requires pinned version")
errors, _ = pipeline.preflight()
self.assertEqual(errors, ["runner[opencode/student]: opencode requires pinned version"])
def test_only_selected_policy_key_is_required(self):
self.secrets = {"TEACHER_KEY": "fixture"}
errors, _ = pipeline.preflight()
self.assertEqual(errors, ["STUDENT_KEY 未设:policy=student 的轨迹无法采集"])
def test_oracle_only_has_no_runner_or_llm_dependency(self):
self.spec.mode = "oracle"
self.cfg["endpoints"] = {}
self.get_runner.side_effect = AssertionError("Oracle must not touch a runner")
errors, _ = pipeline.preflight(oracle_only=True)
self.assertEqual(errors, [])
self.get_runner.assert_not_called()
def test_selected_profile_replaces_missing_legacy_venv(self):
self.cfg["paths"]["venv"]["ptq"] = str(self.root / "missing-legacy")
self.cfg["toolchains"] = {"selected": {"backend": "ptq", "marches": ["nash-m"],
"qcfgs": ["int8"], "version": "fixture-v1", "venv": str(self.root / "profile")}}
self.spec.overrides = {"toolchain": "selected"}
self.assertEqual(pipeline.preflight()[0], [])
self.python.unlink()
errors, _ = pipeline.preflight()
self.assertEqual(len(errors), 1)
self.assertIn("toolchain[selected] Python 缺", errors[0])
def test_profile_target_mismatch_is_rejected_before_execution(self):
self.cfg["toolchains"] = {"other": {"backend": "ptq", "marches": ["nash-p"],
"qcfgs": ["int8"], "version": "fixture-v1", "venv": str(self.root / "profile")}}
self.spec.overrides = {"toolchain": "other"}
errors, _ = pipeline.preflight()
self.assertEqual(len(errors), 1)
self.assertIn("marches does not allow nash-m", errors[0])
def test_empty_target_resources_are_explicit_and_do_not_open_database(self):
self.spec.march = 'nash-p'
config = {'march_by_board': {'s600': 'nash-p'}, 'boards': {}}
with patch.object(registry, 'boards', return_value=config), \
patch.object(db, 'connect', side_effect=AssertionError('preflight must not open DB')):
errors, warnings = pipeline.preflight(oracle_only=True, target='s600')
self.assertEqual(errors, [])
message = next(w for w in warnings if 'board resources' in w)
self.assertIn('enabled_matching=0', message)
self.assertIn('authentication/inference not checked', message)
def test_per_board_credentials_replace_global_only_warning(self):
self.secrets = {'HOST_A': 'private-host', 'KEY_A': 'private-key'}
config = {'march_by_board': {'s100p': 'nash-m'}, 'boards': {
'board-a': {'board_type': 's100p', 'host_env': 'HOST_A', 'ssh_key_env': 'KEY_A'}}}
with patch.object(registry, 'boards', return_value=config):
errors, warnings = pipeline.preflight(oracle_only=True)
self.assertEqual(errors, [])
message = next(w for w in warnings if 'board resources' in w)
self.assertIn('credentials_ready=1', message)
self.assertIn('missing={}', message)
self.assertNotIn('private-host', message)
self.assertNotIn('private-key', message)
if __name__ == "__main__":
unittest.main()