122 lines
6.4 KiB
Python
122 lines
6.4 KiB
Python
"""Runner/policy/profile admission is checked without compilation or model work."""
|
||
from contextlib import ExitStack
|
||
from pathlib import Path
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
import unittest
|
||
from unittest.mock import Mock, patch
|
||
|
||
from harness import db, pipeline, registry, runners
|
||
|
||
|
||
class PreflightExtensions(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(self.tmp.cleanup)
|
||
self.stack = ExitStack()
|
||
self.addCleanup(self.stack.close)
|
||
self.root = Path(self.tmp.name)
|
||
self.python = self.root / "profile/bin/python"
|
||
self.python.parent.mkdir(parents=True)
|
||
self.python.write_text("fixture only; never executed")
|
||
(self.root / "horizon").mkdir()
|
||
for directory in (self.root / "data/pool/fixture/v1", self.root / "data/calib/cls/input"):
|
||
directory.mkdir(parents=True)
|
||
(directory / "manifest.json").write_text("{}")
|
||
self.cfg = {"paths": {"venv": {"ptq": str(self.root / "profile")},
|
||
"horizon": str(self.root / "horizon"), "pi_bin": str(self.root / "missing-pi")},
|
||
"endpoints": {"teacher": {"api_key_env": "TEACHER_KEY"},
|
||
"student": {"api_key_env": "STUDENT_KEY"}}}
|
||
self.secrets = {"BOARD_SSH_KEY": "fixture", "STUDENT_KEY": "fixture"}
|
||
self.spec = SimpleNamespace(mode="e2e", runner="opencode", policy="student", model="m", task="cls",
|
||
march="nash-m", backend="ptq", qcfg="int8", overrides={})
|
||
row = {"model_id": "m", "dataset": "fixture", "input_spec": "input"}
|
||
for obj, name, value in ((pipeline.paths, "config", lambda: self.cfg),
|
||
(pipeline.paths, "secret", lambda key: self.secrets.get(key)),
|
||
(pipeline.paths, "PROJECT_ROOT", self.root),
|
||
(pipeline.paths, "DATA", self.root / "data"),
|
||
(pipeline, "models", lambda: [row]),
|
||
(pipeline, "model_row", lambda _: row),
|
||
(pipeline, "expand_matrix", lambda *a, **kw: [self.spec]),
|
||
(pipeline, "datasets", lambda: {"datasets": {"fixture": {"version": "v1", "t1": {"n": 0}}}})):
|
||
self.stack.enter_context(patch.object(obj, name, value))
|
||
self.check = Mock()
|
||
self.get_runner = self.stack.enter_context(patch.object(runners, "get", return_value=SimpleNamespace(check=self.check)))
|
||
|
||
def test_opencode_student_does_not_require_pi_or_teacher_credentials(self):
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(errors, [])
|
||
self.get_runner.assert_called_once_with("opencode")
|
||
self.check.assert_called_once_with("student", execute=True)
|
||
|
||
def test_pi_still_requires_its_binary(self):
|
||
self.spec.runner = "pi"
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(len(errors), 1)
|
||
self.assertIn("pi 不在 PATH", errors[0])
|
||
|
||
def test_runner_disabled_or_wrong_version_is_an_error(self):
|
||
self.check.side_effect = ValueError("opencode requires pinned version")
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(errors, ["runner[opencode/student]: opencode requires pinned version"])
|
||
|
||
def test_only_selected_policy_key_is_required(self):
|
||
self.secrets = {"TEACHER_KEY": "fixture"}
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(errors, ["STUDENT_KEY 未设:policy=student 的轨迹无法采集"])
|
||
|
||
def test_oracle_only_has_no_runner_or_llm_dependency(self):
|
||
self.spec.mode = "oracle"
|
||
self.cfg["endpoints"] = {}
|
||
self.get_runner.side_effect = AssertionError("Oracle must not touch a runner")
|
||
errors, _ = pipeline.preflight(oracle_only=True)
|
||
self.assertEqual(errors, [])
|
||
self.get_runner.assert_not_called()
|
||
|
||
def test_selected_profile_replaces_missing_legacy_venv(self):
|
||
self.cfg["paths"]["venv"]["ptq"] = str(self.root / "missing-legacy")
|
||
self.cfg["toolchains"] = {"selected": {"backend": "ptq", "marches": ["nash-m"],
|
||
"qcfgs": ["int8"], "version": "fixture-v1", "venv": str(self.root / "profile")}}
|
||
self.spec.overrides = {"toolchain": "selected"}
|
||
self.assertEqual(pipeline.preflight()[0], [])
|
||
self.python.unlink()
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(len(errors), 1)
|
||
self.assertIn("toolchain[selected] Python 缺", errors[0])
|
||
|
||
def test_profile_target_mismatch_is_rejected_before_execution(self):
|
||
self.cfg["toolchains"] = {"other": {"backend": "ptq", "marches": ["nash-p"],
|
||
"qcfgs": ["int8"], "version": "fixture-v1", "venv": str(self.root / "profile")}}
|
||
self.spec.overrides = {"toolchain": "other"}
|
||
errors, _ = pipeline.preflight()
|
||
self.assertEqual(len(errors), 1)
|
||
self.assertIn("marches does not allow nash-m", errors[0])
|
||
|
||
def test_empty_target_resources_are_explicit_and_do_not_open_database(self):
|
||
self.spec.march = 'nash-p'
|
||
config = {'march_by_board': {'s600': 'nash-p'}, 'boards': {}}
|
||
with patch.object(registry, 'boards', return_value=config), \
|
||
patch.object(db, 'connect', side_effect=AssertionError('preflight must not open DB')):
|
||
errors, warnings = pipeline.preflight(oracle_only=True, target='s600')
|
||
self.assertEqual(errors, [])
|
||
message = next(w for w in warnings if 'board resources' in w)
|
||
self.assertIn('enabled_matching=0', message)
|
||
self.assertIn('authentication/inference not checked', message)
|
||
|
||
def test_per_board_credentials_replace_global_only_warning(self):
|
||
self.secrets = {'HOST_A': 'private-host', 'KEY_A': 'private-key'}
|
||
config = {'march_by_board': {'s100p': 'nash-m'}, 'boards': {
|
||
'board-a': {'board_type': 's100p', 'host_env': 'HOST_A', 'ssh_key_env': 'KEY_A'}}}
|
||
with patch.object(registry, 'boards', return_value=config):
|
||
errors, warnings = pipeline.preflight(oracle_only=True)
|
||
self.assertEqual(errors, [])
|
||
message = next(w for w in warnings if 'board resources' in w)
|
||
self.assertIn('credentials_ready=1', message)
|
||
self.assertIn('missing={}', message)
|
||
self.assertNotIn('private-host', message)
|
||
self.assertNotIn('private-key', message)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|