"""Physical board locks must remain exclusive while independent boards run in parallel.""" from contextlib import ExitStack, nullcontext from copy import deepcopy import fcntl import hashlib import multiprocessing from pathlib import Path from queue import Empty import tempfile import time from types import SimpleNamespace import unittest from unittest.mock import patch import yaml from harness import paths, registry, scheduler def _hold_board(registry_dir, state_dir, config, secrets, march, lane, acquired, release): """Spawn-safe real flock worker; test configuration contains no credentials.""" registry.REGISTRY = Path(registry_dir) paths.STATE = Path(state_dir) paths.config = lambda: config paths.secret = secrets.get with scheduler.acquire("board", poll_s=0.02, march=march, lane=lane) as slot: acquired.put(slot) release.wait(timeout=8) class MultiBoard(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.root = Path(self.tmp.name) self.registry_dir = self.root / "registry" self.registry_dir.mkdir() self.config = {"pools": {"board": {"workers": 99, "reserve_for_agent": 0}, "cpu": {"workers": 2}}} self.secrets = {"A_HOST": "test-host", "A_PORT": "40001", "B_HOST": "test-host", "B_PORT": "40002"} self.stack = ExitStack() self.addCleanup(self.stack.close) self.stack.enter_context(patch.object(registry, "REGISTRY", self.registry_dir)) self.stack.enter_context(patch.object(paths, "STATE", self.root / "state")) self.stack.enter_context(patch.object(paths, "config", lambda: self.config)) self.secret = self.stack.enter_context(patch.object(paths, "secret", side_effect=self.secrets.get)) self.boards = { "a": {"slot": 0, "march": "nash-m", "host_env": "A_HOST", "port_env": "A_PORT", "user_env": "A_USER"}, "b": {"slot": 7, "march": "nash-m", "host_env": "B_HOST", "port_env": "B_PORT", "user_env": "B_USER"}, } self.write() def write(self): (self.registry_dir / "boards.yaml").write_text(yaml.safe_dump({"boards": self.boards})) def context(self, march="nash-m", board=None): return SimpleNamespace(job={"march": march, "board": board}, extra={}) def worker(self, march="nash-m", lane="normal"): mp = multiprocessing.get_context("spawn") acquired, release = mp.Queue(), mp.Event() process = mp.Process(target=_hold_board, args=(str(self.registry_dir), str(paths.STATE), self.config, self.secrets, march, lane, acquired, release)) process.start() def cleanup(): release.set() process.join(timeout=3) if process.is_alive(): process.terminate() process.join(timeout=3) acquired.close() acquired.join_thread() self.addCleanup(cleanup) return process, acquired, release def test_slots_are_sparse_enabled_and_refreshed_without_cache(self): self.assertEqual(list(registry.board_slots()), [0, 7]) self.assertEqual(scheduler.pool_size("board"), 2) self.boards["b"]["enabled"] = False self.write() self.assertEqual(list(registry.board_slots()), [0]) self.assertEqual(scheduler.pool_size("board"), 1) self.assertEqual(registry.board_for_march("nash-m")["name"], "a") self.assertIsNone(registry.board_for_march("nash-p")) def test_legacy_single_board_uses_original_board_zero_lock(self): self.boards = {"legacy": {"march": "nash-m"}} self.write() self.assertEqual(list(registry.board_slots()), [0]) with scheduler.acquire("board", march="nash-m") as slot: self.assertEqual(slot, 0) with open(paths.STATE / "locks" / "board-0.lock", "r") as handle: with self.assertRaises(BlockingIOError): fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) def test_multi_board_requires_explicit_unique_non_negative_integer_slots(self): initial = deepcopy(self.boards) for invalid in (None, -1, 1.5, "1", True, 0): with self.subTest(slot=invalid): self.boards = deepcopy(initial) if invalid is None: self.boards["b"].pop("slot") else: self.boards["b"]["slot"] = invalid self.write() with self.assertRaises(ValueError): registry.board_slots() def test_disabled_slots_cannot_be_reused(self): self.boards["a"]["enabled"] = False self.boards["b"]["slot"] = 0 self.write() with self.assertRaisesRegex(ValueError, "重复 slot"): registry.board_slots() def test_duplicate_endpoints_rejected_without_reading_other_credentials(self): self.secrets["B_PORT"] = self.secrets["A_PORT"] with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"): registry.board_slots() self.assertLessEqual({call.args[0] for call in self.secret.call_args_list}, {"A_HOST", "A_PORT", "B_HOST", "B_PORT"}) self.secrets.clear() self.assertEqual(len(registry.board_slots()), 2) def test_default_ssh_port_and_host_case_do_not_allow_duplicate_endpoint(self): self.secrets.update(A_HOST="TEST-HOST.", B_HOST="test-host", A_PORT="22") self.secrets.pop("B_PORT") with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"): registry.board_slots() def test_invalid_march_is_rejected_even_on_disabled_board(self): self.boards["a"].update(enabled=False, march="nash-z") self.write() with self.assertRaisesRegex(ValueError, "march"): registry.board_slots("nash-m") with self.assertRaisesRegex(ValueError, "march"): registry.board_slots("wrong") def test_different_boards_run_in_parallel_across_processes(self): first, first_acquired, first_release = self.worker() self.assertEqual(first_acquired.get(timeout=5), 0) second, second_acquired, second_release = self.worker() self.assertEqual(second_acquired.get(timeout=5), 7) self.assertTrue(first.is_alive()) self.assertTrue(second.is_alive()) self.assertEqual(scheduler.pool_usage()["board"], 2) first_release.set() second_release.set() def test_same_board_is_exclusive_across_processes(self): self.boards["b"]["enabled"] = False self.write() first, first_acquired, first_release = self.worker() self.assertEqual(first_acquired.get(timeout=5), 0) second, second_acquired, second_release = self.worker() with self.assertRaises(Empty): second_acquired.get(timeout=0.4) first_release.set() self.assertEqual(second_acquired.get(timeout=5), 0) second_release.set() def test_march_filter_uses_only_matching_board_slot(self): self.boards["b"]["march"] = "nash-p" self.write() self.assertEqual(list(registry.board_slots("nash-p")), [7]) with scheduler.acquire("board", march="nash-p") as slot: self.assertEqual(slot, 7) def test_agent_reservation_is_applied_within_compatible_boards(self): self.config["pools"]["board"]["reserve_for_agent"] = 1 with scheduler.acquire("board", march="nash-m", lane="normal") as oracle_slot: self.assertEqual(oracle_slot, 7) with scheduler.acquire("board", march="nash-m", lane="agent") as agent_slot: self.assertEqual(agent_slot, 0) self.boards["b"]["march"] = "nash-p" self.write() # A globally reserved slot must not starve the only matching board. with scheduler.acquire("board", march="nash-m", lane="normal") as slot: self.assertEqual(slot, 0) def test_no_compatible_board_returns_without_waiting(self): ctx = self.context("nash-e") with patch.object(scheduler.time, "sleep") as sleep: with scheduler.acquire_stage(ctx, "board") as slot: self.assertIsNone(slot) self.assertIsNone(ctx.extra["board"]) sleep.assert_not_called() self.boards["a"]["enabled"] = self.boards["b"]["enabled"] = False self.write() with scheduler.acquire("board") as slot: self.assertIsNone(slot) self.assertEqual(scheduler.pool_size("board"), 0) def test_stage_uses_locked_board_snapshot_even_if_registry_changes(self): self.config["pools"]["board"]["reserve_for_agent"] = 1 ctx = self.context(board={"name": "a", "march": "nash-m"}) with scheduler.acquire_stage(ctx, "board") as slot: self.assertEqual(slot, 7) self.assertEqual(ctx.extra["board"]["name"], "b") self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST") self.assertEqual(ctx.extra["slot"], 7) self.boards["b"].update(enabled=False, host_env="NEW_HOST") self.write() self.assertEqual(list(registry.board_slots()), [0]) self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST") self.assertEqual(ctx.extra["board"]["name"], "b") def test_stage_rejects_incompatible_frozen_board(self): with self.assertRaisesRegex(ValueError, "board march"): with scheduler.acquire_stage(self.context(board={"march": "nash-p"}), "board"): self.fail("incompatible board must not be acquired") def test_non_board_stage_delegates_original_pool_and_lane(self): ctx = self.context() with patch.object(scheduler, "acquire", return_value=nullcontext(1)) as acquire: with scheduler.acquire_stage(ctx, "cpu", lane="agent") as slot: self.assertEqual(slot, 1) self.assertEqual(ctx.extra["slot"], 1) acquire.assert_called_once_with("cpu", lane="agent") self.assertNotIn("board", ctx.extra) self.assertEqual(scheduler.pool_size("cpu"), 2) def test_default_fingerprint_preserves_legacy_bytes_and_accepts_board_snapshot(self): (self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n") expected = hashlib.sha256() for path in sorted(self.registry_dir.iterdir()): expected.update(path.name.encode()) expected.update(path.read_bytes()) self.assertEqual(registry.registry_fingerprint(), expected.hexdigest()) original_boards = (self.registry_dir / "boards.yaml").read_bytes() self.boards["b"]["slot"] = 10 self.write() self.assertNotEqual(registry.registry_fingerprint(), expected.hexdigest()) self.assertEqual(registry.registry_fingerprint(board_bytes=original_boards), expected.hexdigest()) def test_device_changes_do_not_change_task_fingerprint(self): original = registry.registry_fingerprint(include_board_resources=False) self.boards["b"].update(slot=20, enabled=False, host_env="NEW_HOST") self.boards["new"] = {"slot": 30, "march": "nash-p", "host_env": "THIRD_HOST"} self.write() self.assertEqual(registry.registry_fingerprint(include_board_resources=False), original) def test_static_board_maps_and_other_registry_content_still_change_task_fingerprint(self): original = registry.registry_fingerprint(include_board_resources=False) path = self.registry_dir / "boards.yaml" path.write_text(yaml.safe_dump({"boards": self.boards, "march_by_board": {"s100p": "nash-m"}})) with_mapping = registry.registry_fingerprint(include_board_resources=False) self.assertNotEqual(with_mapping, original) path.write_text(yaml.safe_dump({"march_by_board": {"s100p": "nash-m"}, "boards": self.boards}, sort_keys=False)) self.assertEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping) (self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n") self.assertNotEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping) if __name__ == "__main__": unittest.main()