data-gen-bpu/scripts/test_multiboard.py

258 lines
12 KiB
Python

"""Physical board locks must remain exclusive while independent boards run in parallel."""
from contextlib import ExitStack, nullcontext
from copy import deepcopy
import fcntl
import hashlib
import multiprocessing
from pathlib import Path
from queue import Empty
import tempfile
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import yaml
from harness import paths, registry, scheduler
def _hold_board(registry_dir, state_dir, config, secrets, march, lane, acquired, release):
"""Spawn-safe real flock worker; test configuration contains no credentials."""
registry.REGISTRY = Path(registry_dir)
paths.STATE = Path(state_dir)
paths.config = lambda: config
paths.secret = secrets.get
with scheduler.acquire("board", poll_s=0.02, march=march, lane=lane) as slot:
acquired.put(slot)
release.wait(timeout=8)
class MultiBoard(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.registry_dir = self.root / "registry"
self.registry_dir.mkdir()
self.config = {"pools": {"board": {"workers": 99, "reserve_for_agent": 0}, "cpu": {"workers": 2}}}
self.secrets = {"A_HOST": "test-host", "A_PORT": "40001", "B_HOST": "test-host", "B_PORT": "40002"}
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.stack.enter_context(patch.object(registry, "REGISTRY", self.registry_dir))
self.stack.enter_context(patch.object(paths, "STATE", self.root / "state"))
self.stack.enter_context(patch.object(paths, "config", lambda: self.config))
self.secret = self.stack.enter_context(patch.object(paths, "secret", side_effect=self.secrets.get))
self.boards = {
"a": {"slot": 0, "march": "nash-m", "host_env": "A_HOST", "port_env": "A_PORT", "user_env": "A_USER"},
"b": {"slot": 7, "march": "nash-m", "host_env": "B_HOST", "port_env": "B_PORT", "user_env": "B_USER"},
}
self.write()
def write(self):
(self.registry_dir / "boards.yaml").write_text(yaml.safe_dump({"boards": self.boards}))
def context(self, march="nash-m", board=None):
return SimpleNamespace(job={"march": march, "board": board}, extra={})
def worker(self, march="nash-m", lane="normal"):
mp = multiprocessing.get_context("spawn")
acquired, release = mp.Queue(), mp.Event()
process = mp.Process(target=_hold_board, args=(str(self.registry_dir), str(paths.STATE), self.config,
self.secrets, march, lane, acquired, release))
process.start()
def cleanup():
release.set()
process.join(timeout=3)
if process.is_alive():
process.terminate()
process.join(timeout=3)
acquired.close()
acquired.join_thread()
self.addCleanup(cleanup)
return process, acquired, release
def test_slots_are_sparse_enabled_and_refreshed_without_cache(self):
self.assertEqual(list(registry.board_slots()), [0, 7])
self.assertEqual(scheduler.pool_size("board"), 2)
self.boards["b"]["enabled"] = False
self.write()
self.assertEqual(list(registry.board_slots()), [0])
self.assertEqual(scheduler.pool_size("board"), 1)
self.assertEqual(registry.board_for_march("nash-m")["name"], "a")
self.assertIsNone(registry.board_for_march("nash-p"))
def test_legacy_single_board_uses_original_board_zero_lock(self):
self.boards = {"legacy": {"march": "nash-m"}}
self.write()
self.assertEqual(list(registry.board_slots()), [0])
with scheduler.acquire("board", march="nash-m") as slot:
self.assertEqual(slot, 0)
with open(paths.STATE / "locks" / "board-0.lock", "r") as handle:
with self.assertRaises(BlockingIOError):
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
def test_multi_board_requires_explicit_unique_non_negative_integer_slots(self):
initial = deepcopy(self.boards)
for invalid in (None, -1, 1.5, "1", True, 0):
with self.subTest(slot=invalid):
self.boards = deepcopy(initial)
if invalid is None:
self.boards["b"].pop("slot")
else:
self.boards["b"]["slot"] = invalid
self.write()
with self.assertRaises(ValueError):
registry.board_slots()
def test_disabled_slots_cannot_be_reused(self):
self.boards["a"]["enabled"] = False
self.boards["b"]["slot"] = 0
self.write()
with self.assertRaisesRegex(ValueError, "重复 slot"):
registry.board_slots()
def test_duplicate_endpoints_rejected_without_reading_other_credentials(self):
self.secrets["B_PORT"] = self.secrets["A_PORT"]
with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"):
registry.board_slots()
self.assertLessEqual({call.args[0] for call in self.secret.call_args_list},
{"A_HOST", "A_PORT", "B_HOST", "B_PORT"})
self.secrets.clear()
self.assertEqual(len(registry.board_slots()), 2)
def test_default_ssh_port_and_host_case_do_not_allow_duplicate_endpoint(self):
self.secrets.update(A_HOST="TEST-HOST.", B_HOST="test-host", A_PORT="22")
self.secrets.pop("B_PORT")
with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"):
registry.board_slots()
def test_invalid_march_is_rejected_even_on_disabled_board(self):
self.boards["a"].update(enabled=False, march="nash-z")
self.write()
with self.assertRaisesRegex(ValueError, "march"):
registry.board_slots("nash-m")
with self.assertRaisesRegex(ValueError, "march"):
registry.board_slots("wrong")
def test_different_boards_run_in_parallel_across_processes(self):
first, first_acquired, first_release = self.worker()
self.assertEqual(first_acquired.get(timeout=5), 0)
second, second_acquired, second_release = self.worker()
self.assertEqual(second_acquired.get(timeout=5), 7)
self.assertTrue(first.is_alive())
self.assertTrue(second.is_alive())
self.assertEqual(scheduler.pool_usage()["board"], 2)
first_release.set()
second_release.set()
def test_same_board_is_exclusive_across_processes(self):
self.boards["b"]["enabled"] = False
self.write()
first, first_acquired, first_release = self.worker()
self.assertEqual(first_acquired.get(timeout=5), 0)
second, second_acquired, second_release = self.worker()
with self.assertRaises(Empty):
second_acquired.get(timeout=0.4)
first_release.set()
self.assertEqual(second_acquired.get(timeout=5), 0)
second_release.set()
def test_march_filter_uses_only_matching_board_slot(self):
self.boards["b"]["march"] = "nash-p"
self.write()
self.assertEqual(list(registry.board_slots("nash-p")), [7])
with scheduler.acquire("board", march="nash-p") as slot:
self.assertEqual(slot, 7)
def test_agent_reservation_is_applied_within_compatible_boards(self):
self.config["pools"]["board"]["reserve_for_agent"] = 1
with scheduler.acquire("board", march="nash-m", lane="normal") as oracle_slot:
self.assertEqual(oracle_slot, 7)
with scheduler.acquire("board", march="nash-m", lane="agent") as agent_slot:
self.assertEqual(agent_slot, 0)
self.boards["b"]["march"] = "nash-p"
self.write()
# A globally reserved slot must not starve the only matching board.
with scheduler.acquire("board", march="nash-m", lane="normal") as slot:
self.assertEqual(slot, 0)
def test_no_compatible_board_returns_without_waiting(self):
ctx = self.context("nash-e")
with patch.object(scheduler.time, "sleep") as sleep:
with scheduler.acquire_stage(ctx, "board") as slot:
self.assertIsNone(slot)
self.assertIsNone(ctx.extra["board"])
sleep.assert_not_called()
self.boards["a"]["enabled"] = self.boards["b"]["enabled"] = False
self.write()
with scheduler.acquire("board") as slot:
self.assertIsNone(slot)
self.assertEqual(scheduler.pool_size("board"), 0)
def test_stage_uses_locked_board_snapshot_even_if_registry_changes(self):
self.config["pools"]["board"]["reserve_for_agent"] = 1
ctx = self.context(board={"name": "a", "march": "nash-m"})
with scheduler.acquire_stage(ctx, "board") as slot:
self.assertEqual(slot, 7)
self.assertEqual(ctx.extra["board"]["name"], "b")
self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST")
self.assertEqual(ctx.extra["slot"], 7)
self.boards["b"].update(enabled=False, host_env="NEW_HOST")
self.write()
self.assertEqual(list(registry.board_slots()), [0])
self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST")
self.assertEqual(ctx.extra["board"]["name"], "b")
def test_stage_rejects_incompatible_frozen_board(self):
with self.assertRaisesRegex(ValueError, "board march"):
with scheduler.acquire_stage(self.context(board={"march": "nash-p"}), "board"):
self.fail("incompatible board must not be acquired")
def test_non_board_stage_delegates_original_pool_and_lane(self):
ctx = self.context()
with patch.object(scheduler, "acquire", return_value=nullcontext(1)) as acquire:
with scheduler.acquire_stage(ctx, "cpu", lane="agent") as slot:
self.assertEqual(slot, 1)
self.assertEqual(ctx.extra["slot"], 1)
acquire.assert_called_once_with("cpu", lane="agent")
self.assertNotIn("board", ctx.extra)
self.assertEqual(scheduler.pool_size("cpu"), 2)
def test_default_fingerprint_preserves_legacy_bytes_and_accepts_board_snapshot(self):
(self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n")
expected = hashlib.sha256()
for path in sorted(self.registry_dir.iterdir()):
expected.update(path.name.encode())
expected.update(path.read_bytes())
self.assertEqual(registry.registry_fingerprint(), expected.hexdigest())
original_boards = (self.registry_dir / "boards.yaml").read_bytes()
self.boards["b"]["slot"] = 10
self.write()
self.assertNotEqual(registry.registry_fingerprint(), expected.hexdigest())
self.assertEqual(registry.registry_fingerprint(board_bytes=original_boards), expected.hexdigest())
def test_device_changes_do_not_change_task_fingerprint(self):
original = registry.registry_fingerprint(include_board_resources=False)
self.boards["b"].update(slot=20, enabled=False, host_env="NEW_HOST")
self.boards["new"] = {"slot": 30, "march": "nash-p", "host_env": "THIRD_HOST"}
self.write()
self.assertEqual(registry.registry_fingerprint(include_board_resources=False), original)
def test_static_board_maps_and_other_registry_content_still_change_task_fingerprint(self):
original = registry.registry_fingerprint(include_board_resources=False)
path = self.registry_dir / "boards.yaml"
path.write_text(yaml.safe_dump({"boards": self.boards, "march_by_board": {"s100p": "nash-m"}}))
with_mapping = registry.registry_fingerprint(include_board_resources=False)
self.assertNotEqual(with_mapping, original)
path.write_text(yaml.safe_dump({"march_by_board": {"s100p": "nash-m"}, "boards": self.boards}, sort_keys=False))
self.assertEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping)
(self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n")
self.assertNotEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping)
if __name__ == "__main__":
unittest.main()