258 lines
12 KiB
Python
258 lines
12 KiB
Python
"""Physical board locks must remain exclusive while independent boards run in parallel."""
|
|
from contextlib import ExitStack, nullcontext
|
|
from copy import deepcopy
|
|
import fcntl
|
|
import hashlib
|
|
import multiprocessing
|
|
from pathlib import Path
|
|
from queue import Empty
|
|
import tempfile
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import yaml
|
|
|
|
from harness import paths, registry, scheduler
|
|
|
|
|
|
def _hold_board(registry_dir, state_dir, config, secrets, march, lane, acquired, release):
|
|
"""Spawn-safe real flock worker; test configuration contains no credentials."""
|
|
registry.REGISTRY = Path(registry_dir)
|
|
paths.STATE = Path(state_dir)
|
|
paths.config = lambda: config
|
|
paths.secret = secrets.get
|
|
with scheduler.acquire("board", poll_s=0.02, march=march, lane=lane) as slot:
|
|
acquired.put(slot)
|
|
release.wait(timeout=8)
|
|
|
|
|
|
class MultiBoard(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.root = Path(self.tmp.name)
|
|
self.registry_dir = self.root / "registry"
|
|
self.registry_dir.mkdir()
|
|
self.config = {"pools": {"board": {"workers": 99, "reserve_for_agent": 0}, "cpu": {"workers": 2}}}
|
|
self.secrets = {"A_HOST": "test-host", "A_PORT": "40001", "B_HOST": "test-host", "B_PORT": "40002"}
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
self.stack.enter_context(patch.object(registry, "REGISTRY", self.registry_dir))
|
|
self.stack.enter_context(patch.object(paths, "STATE", self.root / "state"))
|
|
self.stack.enter_context(patch.object(paths, "config", lambda: self.config))
|
|
self.secret = self.stack.enter_context(patch.object(paths, "secret", side_effect=self.secrets.get))
|
|
self.boards = {
|
|
"a": {"slot": 0, "march": "nash-m", "host_env": "A_HOST", "port_env": "A_PORT", "user_env": "A_USER"},
|
|
"b": {"slot": 7, "march": "nash-m", "host_env": "B_HOST", "port_env": "B_PORT", "user_env": "B_USER"},
|
|
}
|
|
self.write()
|
|
|
|
def write(self):
|
|
(self.registry_dir / "boards.yaml").write_text(yaml.safe_dump({"boards": self.boards}))
|
|
|
|
def context(self, march="nash-m", board=None):
|
|
return SimpleNamespace(job={"march": march, "board": board}, extra={})
|
|
|
|
def worker(self, march="nash-m", lane="normal"):
|
|
mp = multiprocessing.get_context("spawn")
|
|
acquired, release = mp.Queue(), mp.Event()
|
|
process = mp.Process(target=_hold_board, args=(str(self.registry_dir), str(paths.STATE), self.config,
|
|
self.secrets, march, lane, acquired, release))
|
|
process.start()
|
|
|
|
def cleanup():
|
|
release.set()
|
|
process.join(timeout=3)
|
|
if process.is_alive():
|
|
process.terminate()
|
|
process.join(timeout=3)
|
|
acquired.close()
|
|
acquired.join_thread()
|
|
|
|
self.addCleanup(cleanup)
|
|
return process, acquired, release
|
|
|
|
def test_slots_are_sparse_enabled_and_refreshed_without_cache(self):
|
|
self.assertEqual(list(registry.board_slots()), [0, 7])
|
|
self.assertEqual(scheduler.pool_size("board"), 2)
|
|
self.boards["b"]["enabled"] = False
|
|
self.write()
|
|
self.assertEqual(list(registry.board_slots()), [0])
|
|
self.assertEqual(scheduler.pool_size("board"), 1)
|
|
self.assertEqual(registry.board_for_march("nash-m")["name"], "a")
|
|
self.assertIsNone(registry.board_for_march("nash-p"))
|
|
|
|
def test_legacy_single_board_uses_original_board_zero_lock(self):
|
|
self.boards = {"legacy": {"march": "nash-m"}}
|
|
self.write()
|
|
self.assertEqual(list(registry.board_slots()), [0])
|
|
with scheduler.acquire("board", march="nash-m") as slot:
|
|
self.assertEqual(slot, 0)
|
|
with open(paths.STATE / "locks" / "board-0.lock", "r") as handle:
|
|
with self.assertRaises(BlockingIOError):
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
|
|
def test_multi_board_requires_explicit_unique_non_negative_integer_slots(self):
|
|
initial = deepcopy(self.boards)
|
|
for invalid in (None, -1, 1.5, "1", True, 0):
|
|
with self.subTest(slot=invalid):
|
|
self.boards = deepcopy(initial)
|
|
if invalid is None:
|
|
self.boards["b"].pop("slot")
|
|
else:
|
|
self.boards["b"]["slot"] = invalid
|
|
self.write()
|
|
with self.assertRaises(ValueError):
|
|
registry.board_slots()
|
|
|
|
def test_disabled_slots_cannot_be_reused(self):
|
|
self.boards["a"]["enabled"] = False
|
|
self.boards["b"]["slot"] = 0
|
|
self.write()
|
|
with self.assertRaisesRegex(ValueError, "重复 slot"):
|
|
registry.board_slots()
|
|
|
|
def test_duplicate_endpoints_rejected_without_reading_other_credentials(self):
|
|
self.secrets["B_PORT"] = self.secrets["A_PORT"]
|
|
with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"):
|
|
registry.board_slots()
|
|
self.assertLessEqual({call.args[0] for call in self.secret.call_args_list},
|
|
{"A_HOST", "A_PORT", "B_HOST", "B_PORT"})
|
|
self.secrets.clear()
|
|
self.assertEqual(len(registry.board_slots()), 2)
|
|
|
|
def test_default_ssh_port_and_host_case_do_not_allow_duplicate_endpoint(self):
|
|
self.secrets.update(A_HOST="TEST-HOST.", B_HOST="test-host", A_PORT="22")
|
|
self.secrets.pop("B_PORT")
|
|
with self.assertRaisesRegex(ValueError, "重复 SSH endpoint"):
|
|
registry.board_slots()
|
|
|
|
def test_invalid_march_is_rejected_even_on_disabled_board(self):
|
|
self.boards["a"].update(enabled=False, march="nash-z")
|
|
self.write()
|
|
with self.assertRaisesRegex(ValueError, "march"):
|
|
registry.board_slots("nash-m")
|
|
with self.assertRaisesRegex(ValueError, "march"):
|
|
registry.board_slots("wrong")
|
|
|
|
def test_different_boards_run_in_parallel_across_processes(self):
|
|
first, first_acquired, first_release = self.worker()
|
|
self.assertEqual(first_acquired.get(timeout=5), 0)
|
|
second, second_acquired, second_release = self.worker()
|
|
self.assertEqual(second_acquired.get(timeout=5), 7)
|
|
self.assertTrue(first.is_alive())
|
|
self.assertTrue(second.is_alive())
|
|
self.assertEqual(scheduler.pool_usage()["board"], 2)
|
|
first_release.set()
|
|
second_release.set()
|
|
|
|
def test_same_board_is_exclusive_across_processes(self):
|
|
self.boards["b"]["enabled"] = False
|
|
self.write()
|
|
first, first_acquired, first_release = self.worker()
|
|
self.assertEqual(first_acquired.get(timeout=5), 0)
|
|
second, second_acquired, second_release = self.worker()
|
|
with self.assertRaises(Empty):
|
|
second_acquired.get(timeout=0.4)
|
|
first_release.set()
|
|
self.assertEqual(second_acquired.get(timeout=5), 0)
|
|
second_release.set()
|
|
|
|
def test_march_filter_uses_only_matching_board_slot(self):
|
|
self.boards["b"]["march"] = "nash-p"
|
|
self.write()
|
|
self.assertEqual(list(registry.board_slots("nash-p")), [7])
|
|
with scheduler.acquire("board", march="nash-p") as slot:
|
|
self.assertEqual(slot, 7)
|
|
|
|
def test_agent_reservation_is_applied_within_compatible_boards(self):
|
|
self.config["pools"]["board"]["reserve_for_agent"] = 1
|
|
with scheduler.acquire("board", march="nash-m", lane="normal") as oracle_slot:
|
|
self.assertEqual(oracle_slot, 7)
|
|
with scheduler.acquire("board", march="nash-m", lane="agent") as agent_slot:
|
|
self.assertEqual(agent_slot, 0)
|
|
self.boards["b"]["march"] = "nash-p"
|
|
self.write()
|
|
# A globally reserved slot must not starve the only matching board.
|
|
with scheduler.acquire("board", march="nash-m", lane="normal") as slot:
|
|
self.assertEqual(slot, 0)
|
|
|
|
def test_no_compatible_board_returns_without_waiting(self):
|
|
ctx = self.context("nash-e")
|
|
with patch.object(scheduler.time, "sleep") as sleep:
|
|
with scheduler.acquire_stage(ctx, "board") as slot:
|
|
self.assertIsNone(slot)
|
|
self.assertIsNone(ctx.extra["board"])
|
|
sleep.assert_not_called()
|
|
self.boards["a"]["enabled"] = self.boards["b"]["enabled"] = False
|
|
self.write()
|
|
with scheduler.acquire("board") as slot:
|
|
self.assertIsNone(slot)
|
|
self.assertEqual(scheduler.pool_size("board"), 0)
|
|
|
|
def test_stage_uses_locked_board_snapshot_even_if_registry_changes(self):
|
|
self.config["pools"]["board"]["reserve_for_agent"] = 1
|
|
ctx = self.context(board={"name": "a", "march": "nash-m"})
|
|
with scheduler.acquire_stage(ctx, "board") as slot:
|
|
self.assertEqual(slot, 7)
|
|
self.assertEqual(ctx.extra["board"]["name"], "b")
|
|
self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST")
|
|
self.assertEqual(ctx.extra["slot"], 7)
|
|
self.boards["b"].update(enabled=False, host_env="NEW_HOST")
|
|
self.write()
|
|
self.assertEqual(list(registry.board_slots()), [0])
|
|
self.assertEqual(ctx.extra["board"]["host_env"], "B_HOST")
|
|
self.assertEqual(ctx.extra["board"]["name"], "b")
|
|
|
|
def test_stage_rejects_incompatible_frozen_board(self):
|
|
with self.assertRaisesRegex(ValueError, "board march"):
|
|
with scheduler.acquire_stage(self.context(board={"march": "nash-p"}), "board"):
|
|
self.fail("incompatible board must not be acquired")
|
|
|
|
def test_non_board_stage_delegates_original_pool_and_lane(self):
|
|
ctx = self.context()
|
|
with patch.object(scheduler, "acquire", return_value=nullcontext(1)) as acquire:
|
|
with scheduler.acquire_stage(ctx, "cpu", lane="agent") as slot:
|
|
self.assertEqual(slot, 1)
|
|
self.assertEqual(ctx.extra["slot"], 1)
|
|
acquire.assert_called_once_with("cpu", lane="agent")
|
|
self.assertNotIn("board", ctx.extra)
|
|
self.assertEqual(scheduler.pool_size("cpu"), 2)
|
|
|
|
def test_default_fingerprint_preserves_legacy_bytes_and_accepts_board_snapshot(self):
|
|
(self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n")
|
|
expected = hashlib.sha256()
|
|
for path in sorted(self.registry_dir.iterdir()):
|
|
expected.update(path.name.encode())
|
|
expected.update(path.read_bytes())
|
|
self.assertEqual(registry.registry_fingerprint(), expected.hexdigest())
|
|
original_boards = (self.registry_dir / "boards.yaml").read_bytes()
|
|
self.boards["b"]["slot"] = 10
|
|
self.write()
|
|
self.assertNotEqual(registry.registry_fingerprint(), expected.hexdigest())
|
|
self.assertEqual(registry.registry_fingerprint(board_bytes=original_boards), expected.hexdigest())
|
|
|
|
def test_device_changes_do_not_change_task_fingerprint(self):
|
|
original = registry.registry_fingerprint(include_board_resources=False)
|
|
self.boards["b"].update(slot=20, enabled=False, host_env="NEW_HOST")
|
|
self.boards["new"] = {"slot": 30, "march": "nash-p", "host_env": "THIRD_HOST"}
|
|
self.write()
|
|
self.assertEqual(registry.registry_fingerprint(include_board_resources=False), original)
|
|
|
|
def test_static_board_maps_and_other_registry_content_still_change_task_fingerprint(self):
|
|
original = registry.registry_fingerprint(include_board_resources=False)
|
|
path = self.registry_dir / "boards.yaml"
|
|
path.write_text(yaml.safe_dump({"boards": self.boards, "march_by_board": {"s100p": "nash-m"}}))
|
|
with_mapping = registry.registry_fingerprint(include_board_resources=False)
|
|
self.assertNotEqual(with_mapping, original)
|
|
path.write_text(yaml.safe_dump({"march_by_board": {"s100p": "nash-m"}, "boards": self.boards}, sort_keys=False))
|
|
self.assertEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping)
|
|
(self.registry_dir / "models.csv").write_bytes(b"model_id,task\nexample,cls\n")
|
|
self.assertNotEqual(registry.registry_fingerprint(include_board_resources=False), with_mapping)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|