- board: RemoteSession.read_receipt (3 tries, 60 s, 10 s apart) shared by 07-board and the terminal path, which read once under a 15 s cap. When it still cannot be read, an idle board probed while the lease is held counts as cleanup proof (proof=board_idle_ps). 10-27/28: 13 rollouts lost, 7 already clean. - execution: a *CLEANUP_UNCONFIRMED stop is INFRA_ERROR, not AGENT_ERROR. - scheduler: a board quarantine marker without a lease is released when the board is idle (10-24..26: board-6 locked 47 hours). - quota: a generation paused on HTTP_ATTEMPT_BUDGET_EXHAUSTED whose every attempt was a 429 resumes with fresh attempts once its group is OPEN/RECOVERING. Until then it holds no slot, so new claims keep probing the account (10-28 account e: 6 rollouts paused 9 hours after the re-key). - salty_offload: the export runs in a background thread; archiving and freeing go on every 10 minutes (exports take 4-6 hours since 10-25). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 68028a2dfac4860d9c930b79ecaf3f53fb15494b)
684 lines
38 KiB
Python
684 lines
38 KiB
Python
"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。"""
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import shlex
|
||
import tarfile
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from contextlib import nullcontext
|
||
from types import SimpleNamespace
|
||
from unittest.mock import Mock, patch
|
||
|
||
import numpy as np
|
||
|
||
from harness.eval.board import BoardRunner
|
||
from harness.tools import board_bench, board_perf
|
||
|
||
|
||
class Runtime:
|
||
model_names = ["model"]
|
||
input_names = {"model": ["data"]}
|
||
input_shapes = {"model": {"data": [1, 3, 2, 2]}}
|
||
input_types = {"model": {"data": "FLOAT32"}}
|
||
output_names = {"model": [f"output{i}" for i in range(12)]}
|
||
|
||
def __init__(self):
|
||
self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]}
|
||
|
||
def run(self, feed):
|
||
for i, name in enumerate(self.output_names["model"]):
|
||
self.buffers[name][:] = feed["model"]["data"].mean() + i
|
||
# 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。
|
||
return {"model": dict(reversed(list(self.buffers.items())))}
|
||
|
||
|
||
class Session:
|
||
instances = []
|
||
fail = False
|
||
corrupt = False
|
||
mismatch = False
|
||
|
||
def __init__(self, hbm, runtime_dir):
|
||
self.hbm = Path(hbm)
|
||
self.local = tempfile.TemporaryDirectory()
|
||
self.directory = Path(self.local.name)
|
||
self.closed = False
|
||
self.uploads = 0
|
||
self.instances.append(self)
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def path(self, name):
|
||
return str(self.directory / name)
|
||
|
||
def upload(self, source, name):
|
||
shutil.copyfile(source, self.path(name))
|
||
self.uploads += 1
|
||
|
||
def download(self, name, target):
|
||
shutil.copyfile(self.path(name), target)
|
||
|
||
def bench(self, args):
|
||
if self.fail:
|
||
raise SystemExit(17)
|
||
n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3])
|
||
if self.corrupt:
|
||
np.savez(args[3], **{"s0000/out0": np.ones(1)})
|
||
return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board",
|
||
"hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(),
|
||
"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}
|
||
|
||
def remove(self, *names):
|
||
for name in names:
|
||
Path(self.path(name)).unlink()
|
||
|
||
def close(self):
|
||
self.closed = True
|
||
self.local.cleanup()
|
||
|
||
|
||
def sample(value=1, dtype=np.float32):
|
||
return np.full((1, 3, 2, 2), value, dtype=dtype)
|
||
|
||
|
||
class BoardRuntimeTests(unittest.TestCase):
|
||
def setUp(self):
|
||
Session.instances = []
|
||
Session.fail = Session.corrupt = Session.mismatch = False
|
||
|
||
def test_bulk_copies_reused_buffers_in_declared_output_order(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0001=sample(3), s0000=sample(1))
|
||
self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2)
|
||
with np.load(fout) as outputs:
|
||
self.assertEqual(outputs["s0000/out0"].item(), 1)
|
||
self.assertEqual(outputs["s0000/out10"].item(), 11)
|
||
self.assertEqual(outputs["s0001/out0"].item(), 3)
|
||
|
||
def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)):
|
||
np.savez(fin, s0000=x)
|
||
with self.assertRaises(ValueError):
|
||
board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout))
|
||
self.assertFalse(fout.exists())
|
||
runtime = Runtime()
|
||
runtime.input_names = {"model": ["first", "second"]}
|
||
with self.assertRaisesRegex(ValueError, "one input"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0000=sample())
|
||
runtime = Runtime()
|
||
runtime.input_types = {"model": {"data": "TensorType.F32"}}
|
||
self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1)
|
||
for dtype in ("FLOAT16", "INT8", "UINT8"):
|
||
runtime.input_types = {"model": {"data": dtype}}
|
||
with self.assertRaisesRegex(ValueError, "FLOAT32"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_runner_chunks_numeric_outputs_and_cleanup(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
with BoardRunner(str(hbm)) as runner:
|
||
runner.CHUNK = 2
|
||
outs = runner.batch([sample(v) for v in range(5)])
|
||
self.assertEqual([o[0].item() for o in outs], list(range(5)))
|
||
self.assertEqual(outs[0][10].item(), 10)
|
||
self.assertEqual(runner(sample(6))[0].item(), 6)
|
||
self.assertEqual(runner.metadata["source"], "board")
|
||
self.assertIn("S100P", runner.metadata["board_type"])
|
||
self.assertEqual(list(runner.tmp.iterdir()), [])
|
||
self.assertEqual(len(Session.instances), 1)
|
||
self.assertEqual(Session.instances[0].uploads, 4)
|
||
self.assertEqual(list(Session.instances[0].directory.iterdir()), [])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[0].closed)
|
||
|
||
def test_board_stage_failing_before_dispatch_leaves_not_started_receipt(self):
|
||
# 10-15: a harness check failed before any board command; no receipt read as BOARD_CLEANUP_UNCONFIRMED
|
||
# and the quarantine ended the rollout.
|
||
from harness.stages.base import Stage, StageContext
|
||
|
||
def stage(pool, write_receipt):
|
||
class Fake(Stage):
|
||
name = "07-fake"
|
||
def native(self, ctx, stage_dir):
|
||
if write_receipt:
|
||
from harness.execution import atomic_json
|
||
atomic_json(Path(os.environ["BENCH_BOARD_RECEIPT_DIR"]) / "op.json", {"cleanup_state": "UNKNOWN"})
|
||
raise ValueError("failed before the board")
|
||
Fake.pool = pool
|
||
return Fake()
|
||
|
||
for pool, write_receipt, expected in (("board", False, [("VERIFIED", "not_started")]),
|
||
("board", True, [("UNKNOWN", None)]), ("cpu", False, [])):
|
||
with self.subTest(pool=pool, write_receipt=write_receipt), tempfile.TemporaryDirectory() as td, \
|
||
patch.dict(os.environ, {"BENCH_BOARD_RECEIPT_DIR": str(Path(td) / "receipts")}), \
|
||
patch("harness.salty_remote.selected", return_value=False), \
|
||
patch("harness.salty_stage.selected", return_value=False):
|
||
ctx = StageContext(run_id="r", job={"task": "seg"}, job_dir=Path(td), stage_root=Path(td) / "stages",
|
||
artifacts=Path(td) / "artifacts", rollout=0)
|
||
self.assertEqual(stage(pool, write_receipt).run(ctx).state, "FAIL")
|
||
receipts = [json.loads(p.read_text()) for p in (Path(td) / "receipts").glob("*.json")]
|
||
self.assertEqual([(r["cleanup_state"], r.get("proof")) for r in receipts], expected)
|
||
|
||
def test_runner_sizes_board_calls_and_evaluator_groups_by_measured_bytes(self):
|
||
# 10-15: one 32-sample solo.r50 call was 12 GB; segmentation went one image per call.
|
||
from harness.eval.common import groups
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
with BoardRunner(str(hbm)) as runner:
|
||
self.assertEqual(runner.group_size(), 1) # unmeasured: one sample first
|
||
runner.batch([sample(0)])
|
||
runner.MAX_BYTES = 3 * runner._sample_bytes
|
||
self.assertEqual(runner.group_size(), 3)
|
||
self.assertEqual([len(g) for g in groups(runner, list(range(7)))], [3, 3, 1])
|
||
outs = runner.batch([sample(v) for v in range(7)])
|
||
self.assertEqual([o[0].item() for o in outs], list(range(7)))
|
||
self.assertEqual(Session.instances[0].uploads, 1 + 3)
|
||
self.assertEqual([len(g) for g in groups(object(), list(range(7)), 4)], [4, 3])
|
||
|
||
def test_runner_failure_and_invalid_outputs_cleanup(self):
|
||
for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)):
|
||
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
setattr(Session, mode, True)
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(error):
|
||
runner.batch([sample()])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[-1].closed)
|
||
setattr(Session, mode, False)
|
||
|
||
def test_runner_setup_failure_cleans_local_temp(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(SystemExit):
|
||
runner.__enter__()
|
||
self.assertFalse(runner.tmp.exists())
|
||
|
||
def test_ssh_error_is_not_hidden_by_valid_json(self):
|
||
result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n')
|
||
with patch.object(board_perf.subprocess, "run", return_value=result):
|
||
with self.assertRaises(SystemExit) as raised:
|
||
board_perf.run(["ssh", "board", "python3 board_bench.py"])
|
||
self.assertEqual(raised.exception.code, 17)
|
||
|
||
def test_remote_session_uploads_once_and_quotes_shell_paths(self):
|
||
calls = []
|
||
receipt = {}
|
||
|
||
def fake_run(command, timeout=1800, **kwargs):
|
||
calls.append(command)
|
||
if "python3 " in command[-1]:
|
||
args = shlex.split(command[-1])
|
||
receipt.update(operation_id=args[args.index('--operation-id') + 1],
|
||
guard_version=4, cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
|
||
return 'runtime startup\n{"samples": 1}\n'
|
||
if command[-1].startswith('cat -- '):
|
||
return json.dumps(receipt)
|
||
return ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), \
|
||
patch.object(board_perf, "run", side_effect=fake_run), \
|
||
patch.object(board_perf, "guarded_run", side_effect=fake_run):
|
||
with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session:
|
||
session.__enter__()
|
||
self.assertEqual(session.bench(["--runs", "1"])["samples"], 1)
|
||
scps = [c for c in calls if c[0] == "scp"]
|
||
self.assertEqual(len(scps), 6)
|
||
self.assertTrue(any("board_probe.py" in " ".join(cmd) for cmd in scps))
|
||
self.assertTrue(any("board_identity.py" in " ".join(cmd) for cmd in scps))
|
||
self.assertEqual(scps[0][1:3], ["-P", "2222"])
|
||
self.assertTrue(calls[0][-1].startswith("find '/userdata/bench space' -maxdepth 1 -name 'eval-*' -mmin +360 -print "))
|
||
self.assertIn("'/userdata/bench space/", calls[1][-1])
|
||
self.assertTrue(calls[-1][-1].startswith("rm -rf -- "))
|
||
bench = next(c[-1] for c in calls if c[-1].startswith("python3 "))
|
||
self.assertNotIn("|", bench)
|
||
self.assertNotIn("2>/dev/null", bench)
|
||
self.assertIn('board_guard.py', bench)
|
||
|
||
def test_success_without_matching_cleanup_receipt_is_rejected(self):
|
||
for receipt in ('', '{"operation_id":"wrong","guard_version":3,"cleanup_state":"VERIFIED"}'):
|
||
with self.subTest(receipt=receipt), \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=receipt), \
|
||
patch.object(board_perf, 'guarded_run', return_value='{"samples":1}'):
|
||
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_UNCONFIRMED'):
|
||
with board_perf.RemoteSession('model.hbm') as session:
|
||
session.bench(['--runs', '1'])
|
||
|
||
def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self):
|
||
calls = []
|
||
|
||
def fake_run(command, timeout=1800, **kwargs):
|
||
calls.append(command)
|
||
if command[0] == "scp":
|
||
raise SystemExit(23)
|
||
return ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
|
||
session = board_perf.RemoteSession("/local/model.hbm")
|
||
with self.assertRaises(SystemExit) as raised:
|
||
session.__enter__()
|
||
self.assertEqual(raised.exception.code, 23)
|
||
self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory)
|
||
self.assertFalse(session.active)
|
||
|
||
def test_setup_failure_has_not_started_proof_without_dispatch(self):
|
||
for failed_command in ('ssh', 'scp'):
|
||
with self.subTest(command=failed_command), tempfile.TemporaryDirectory() as td:
|
||
def fail_setup(command, **kwargs):
|
||
if command[0] == failed_command and not kwargs.get('cleanup'):
|
||
raise SystemExit(255)
|
||
return ''
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', side_effect=fail_setup), \
|
||
patch.object(board_perf, 'guarded_run') as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
|
||
with self.assertRaises(SystemExit):
|
||
board_perf.RemoteSession('model.hbm').__enter__()
|
||
dispatch.assert_not_called()
|
||
receipts = list(Path(td).glob('*.json'))
|
||
self.assertEqual(len(receipts), 1)
|
||
receipt = json.loads(receipts[0].read_text())
|
||
self.assertEqual(receipt['operation_id'], receipts[0].stem)
|
||
self.assertEqual(receipt['guard_version'], 4)
|
||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
||
self.assertTrue(receipt['all_children_reaped'])
|
||
self.assertEqual(receipt['proof'], 'not_started')
|
||
|
||
def test_setup_retries_only_known_transport_loss_and_is_bounded(self):
|
||
for diagnostic, successes, expected_calls in (
|
||
('kex_exchange_identification: Connection closed by remote host', True, 2),
|
||
('scp: lost connection', True, 2),
|
||
('Permission denied (publickey).', True, 1),
|
||
('', True, 1),
|
||
('Connection reset by peer', False, 2)):
|
||
with self.subTest(diagnostic=diagnostic, successes=successes), \
|
||
patch('harness.execution.from_environment', return_value=None), \
|
||
patch.object(board_perf.time, 'sleep'):
|
||
failed = SimpleNamespace(returncode=255, stdout='', stderr=diagnostic)
|
||
good = SimpleNamespace(returncode=0, stdout='ready', stderr='')
|
||
with patch.object(board_perf.subprocess, 'run', side_effect=[failed, good if successes else failed]) as invoke:
|
||
action = lambda: board_perf.run(['ssh', 'board', 'mkdir -p /userdata/owned'])
|
||
if successes and expected_calls == 2:
|
||
self.assertEqual(board_perf.RemoteSession.setup(action), 'ready')
|
||
else:
|
||
with self.assertRaises(SystemExit) as error:
|
||
board_perf.RemoteSession.setup(action)
|
||
self.assertEqual(error.exception.code, 255)
|
||
self.assertEqual(invoke.call_count, expected_calls)
|
||
|
||
def test_container_caps_board_call_by_the_controller_phase_deadline(self):
|
||
# A Salty container has no execution state; without this a board hang outlived the phase (10-12/14).
|
||
seen = []
|
||
def dispatch(cmd, timeout):
|
||
seen.append(timeout)
|
||
raise SystemExit(255)
|
||
with tempfile.TemporaryDirectory() as td, \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=''), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=dispatch), \
|
||
patch('harness.execution.from_environment', return_value=None):
|
||
with patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td, 'BENCH_PHASE_DEADLINE': str(time.time() + 700)}):
|
||
with self.assertRaises(SystemExit):
|
||
board_perf.RemoteSession('model.hbm').bench([])
|
||
self.assertAlmostEqual(seen[0], 100, delta=5) # 700 s left - 540 s hand-back - 60 s cleanup
|
||
with patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td, 'BENCH_PHASE_DEADLINE': str(time.time() + 500)}):
|
||
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_BUDGET_INSUFFICIENT'):
|
||
board_perf.RemoteSession('model.hbm').bench([])
|
||
self.assertEqual(len(seen), 1) # never dispatched without the budget
|
||
|
||
def test_budget_rejection_preserves_prior_unknown_receipt(self):
|
||
owner = Mock()
|
||
owner.snapshot.return_value = {'deadline_monotonic': 59}
|
||
with tempfile.TemporaryDirectory() as td, \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=''), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(255)) as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}), \
|
||
patch('harness.execution.from_environment', return_value=None):
|
||
session = board_perf.RemoteSession('model.hbm')
|
||
with self.assertRaises(SystemExit):
|
||
session.bench([])
|
||
original = list(Path(td).glob('*.json'))[0]
|
||
original_bytes = original.read_bytes()
|
||
self.assertEqual(json.loads(original_bytes)['cleanup_state'], 'UNKNOWN')
|
||
with patch('harness.execution.from_environment', return_value=owner), \
|
||
patch.object(board_perf.time, 'monotonic', return_value=0):
|
||
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
|
||
session.bench([])
|
||
dispatch.assert_called_once()
|
||
self.assertEqual(original.read_bytes(), original_bytes)
|
||
receipts = [json.loads(p.read_text()) for p in Path(td).glob('*.json')]
|
||
self.assertEqual(len(receipts), 2)
|
||
self.assertEqual({r['cleanup_state'] for r in receipts}, {'UNKNOWN', 'VERIFIED'})
|
||
self.assertTrue(session.cleanup_unconfirmed)
|
||
|
||
def test_dispatched_failure_requires_remote_proof_even_without_output(self):
|
||
for failure in (SystemExit(255), board_perf.subprocess.TimeoutExpired('ssh', 1), None):
|
||
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td, \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=''), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=failure, return_value='') as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
|
||
session = board_perf.RemoteSession('model.hbm')
|
||
with self.assertRaises((SystemExit, RuntimeError, board_perf.subprocess.TimeoutExpired)):
|
||
session.bench([])
|
||
dispatch.assert_called_once()
|
||
receipt = json.loads(next(Path(td).glob('*.json')).read_text())
|
||
self.assertEqual(receipt['cleanup_state'], 'UNKNOWN')
|
||
self.assertNotEqual(receipt.get('proof'), 'not_started')
|
||
self.assertTrue(session.cleanup_unconfirmed)
|
||
|
||
def test_terminal_setup_failure_has_not_started_proof(self):
|
||
from harness import terminal_board
|
||
for step in ('__enter__', 'upload'):
|
||
with self.subTest(step=step), tempfile.TemporaryDirectory() as td:
|
||
hd = Path(td)
|
||
(hd / 'snapshot').mkdir(); (hd / 'input').mkdir()
|
||
current = Mock()
|
||
session = Mock()
|
||
session.__enter__ = Mock()
|
||
session.setup.side_effect = lambda action, *args: action(*args)
|
||
getattr(session, step).side_effect = SystemExit(255)
|
||
ctx = SimpleNamespace(extra={'board': {}})
|
||
# A configured board is required before capability/setup are attempted.
|
||
ctx.extra['board']['runtime_dir'] = '/userdata/fixture'
|
||
with patch.object(terminal_board, 'credentials', return_value=nullcontext()), \
|
||
patch.object(terminal_board.lifecycle, 'current', return_value=current), \
|
||
patch.object(terminal_board, 'TerminalSession', return_value=session), \
|
||
patch.object(terminal_board, 'guarded_run') as dispatch:
|
||
with self.assertRaises(SystemExit):
|
||
terminal_board.execute(ctx, hd, {}, 100)
|
||
dispatch.assert_not_called()
|
||
current.quarantine.assert_not_called()
|
||
session.close.assert_called_once()
|
||
paths = list((hd / 'board-receipts').glob('*.json'))
|
||
self.assertEqual(len(paths), 1)
|
||
receipt = json.loads(paths[0].read_text())
|
||
self.assertEqual(receipt['operation_id'], paths[0].stem)
|
||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
||
self.assertEqual(receipt['proof'], 'not_started')
|
||
|
||
def test_terminal_lost_receipt_is_clean_only_when_the_board_is_idle(self):
|
||
# 10-27/28: 13 rollouts ended because reading the receipt hit a gateway blip; 7 had already cleaned up.
|
||
from harness import terminal_board
|
||
for board_ps in (['sshd sshd: root@notty', 'ps ps -eo comm=,args='], None):
|
||
with self.subTest(idle=board_ps is not None), tempfile.TemporaryDirectory() as td:
|
||
hd = Path(td)
|
||
(hd / 'snapshot').mkdir(); (hd / 'input').mkdir()
|
||
current = Mock()
|
||
session = Mock(directory='/userdata/fixture/eval-x', opts=[], target='root@board')
|
||
session.__enter__ = Mock()
|
||
session.setup.side_effect = lambda action, *args: action(*args)
|
||
session.path.side_effect = lambda name: '/userdata/fixture/eval-x/' + name
|
||
session.remaining.return_value = 100
|
||
session.execute.return_value = ''
|
||
session.read_receipt.side_effect = SystemExit(255)
|
||
session.download.side_effect = lambda name, local: tarfile.open(local, 'w').close()
|
||
ctx = SimpleNamespace(extra={'board': {'runtime_dir': '/userdata/fixture'}, 'slot': 3})
|
||
config = {'cwd': '.', 'command': 'true'}
|
||
with patch.object(terminal_board, 'credentials', return_value=nullcontext()), \
|
||
patch.object(terminal_board.lifecycle, 'current', return_value=current), \
|
||
patch.object(terminal_board, 'TerminalSession', return_value=session), \
|
||
patch.object(terminal_board, 'guarded_run', return_value='done\n'), \
|
||
patch.object(terminal_board, '_board_idle_ps', return_value=board_ps) as probe:
|
||
if board_ps is None:
|
||
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_UNCONFIRMED'):
|
||
terminal_board.execute(ctx, hd, config, 100)
|
||
current.quarantine.assert_called_with('BOARD_CLEANUP_UNCONFIRMED')
|
||
else:
|
||
result = terminal_board.execute(ctx, hd, config, 100)
|
||
self.assertEqual((result.state, result.cleanup_state), ('OK', 'VERIFIED'))
|
||
current.quarantine.assert_not_called()
|
||
probe.assert_called_once_with(3)
|
||
receipt = json.loads(next((hd / 'board-receipts').glob('*.json')).read_text())
|
||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED' if board_ps else 'UNKNOWN')
|
||
self.assertEqual(receipt.get('proof'), 'board_idle_ps' if board_ps else None)
|
||
|
||
def test_busy_wait_retries_only_guard_rejection_without_shell_probe(self):
|
||
session = SimpleNamespace(bench=Mock(side_effect=[SystemExit(75), {"latency_ms_mean": 1.0}]))
|
||
with tempfile.TemporaryDirectory() as td:
|
||
out = Path(td) / "perf.json"
|
||
argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)]
|
||
with patch.object(board_perf.sys, "argv", argv), \
|
||
patch.object(board_perf, "run") as run, \
|
||
patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \
|
||
patch.object(board_perf.time, "sleep") as sleep:
|
||
self.assertEqual(board_perf.main(), 0)
|
||
run.assert_not_called()
|
||
sleep.assert_called_once_with(20)
|
||
self.assertEqual(session.bench.call_count, 2)
|
||
result = json.loads(out.read_text())
|
||
self.assertEqual(result["busy_waited_s"], 20)
|
||
self.assertFalse(result["contended"])
|
||
|
||
def test_busy_or_unknown_does_not_produce_performance_result(self):
|
||
for status in (75, 78):
|
||
with self.subTest(status=status), tempfile.TemporaryDirectory() as td:
|
||
out = Path(td) / 'perf.json'
|
||
argv = ['board_perf', '--hbm', 'm.hbm', '--job-id', 'j', '--out', str(out)]
|
||
if status == 75:
|
||
argv += ['--busy-wait-s', '0']
|
||
session = SimpleNamespace(bench=Mock(side_effect=SystemExit(status)))
|
||
with patch.object(board_perf.sys, 'argv', argv), \
|
||
patch.object(board_perf, 'RemoteSession', return_value=nullcontext(session)), \
|
||
patch.object(board_perf.time, 'sleep') as sleep:
|
||
with self.assertRaises(SystemExit) as error:
|
||
board_perf.main()
|
||
self.assertEqual(error.exception.code, status)
|
||
session.bench.assert_called_once()
|
||
sleep.assert_not_called()
|
||
self.assertFalse(out.exists())
|
||
|
||
def test_eval_runner_uploads_same_probe_with_real_remote_session(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
hbm = Path(td) / 'model.hbm'
|
||
hbm.write_bytes(b'fixture')
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value='') as remote:
|
||
with BoardRunner(str(hbm)):
|
||
pass
|
||
uploads = [call.args[0] for call in remote.call_args_list if call.args[0][0] == 'scp']
|
||
self.assertTrue(any('board_probe.py' in ' '.join(command) for command in uploads))
|
||
self.assertTrue(any('board_guard.py' in ' '.join(command) for command in uploads))
|
||
|
||
def test_failed_guard_retains_evidence_without_erasing_live_directory(self):
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value='') as run, \
|
||
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(124)):
|
||
with self.assertRaises(SystemExit):
|
||
with board_perf.RemoteSession('model.hbm') as session:
|
||
session.bench(['--runs', '1'])
|
||
self.assertFalse(any(c.args[0][-1].startswith('rm ') for c in run.call_args_list))
|
||
# The directory and receipt stay for the possibly live guard; the bulky files go.
|
||
emptied = [c.args[0][-1] for c in run.call_args_list if c.args[0][-1].startswith(f'find {session.directory} ')]
|
||
self.assertEqual(len(emptied), 1)
|
||
self.assertIn("'!' -name '*guard-result.json'", emptied[0])
|
||
|
||
def test_failed_inference_with_confirmed_cleanup_removes_directory(self):
|
||
# 10-15: such sessions were kept too and filled the S600s' /userdata (one solo.r50 chunk = 12 GB).
|
||
calls, receipt = [], {}
|
||
|
||
def fake_run(command, timeout=1800, **kwargs):
|
||
calls.append(command)
|
||
return json.dumps(receipt) if command[-1].startswith('cat -- ') else ''
|
||
|
||
def failing_inference(command, timeout):
|
||
args = shlex.split(command[-1])
|
||
receipt.update(operation_id=args[args.index('--operation-id') + 1], guard_version=4,
|
||
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
|
||
raise SystemExit(1)
|
||
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', side_effect=fake_run), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=failing_inference):
|
||
with self.assertRaises(SystemExit):
|
||
with board_perf.RemoteSession('model.hbm') as session:
|
||
session.bench(['--runs', '1'])
|
||
self.assertEqual(calls[-1][-1], f'rm -rf -- {session.directory}')
|
||
|
||
|
||
class PredispatchProofTests(unittest.TestCase):
|
||
def setUp(self):
|
||
temp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(temp.cleanup)
|
||
self.root = Path(temp.name)
|
||
self.rd = self.root / 'rollout'
|
||
self.receipts = self.rd / 'stages/_queue/op/board-receipts'
|
||
self.receipts.mkdir(parents=True)
|
||
self.env = patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': str(self.receipts)})
|
||
self.env.start(); self.addCleanup(self.env.stop)
|
||
|
||
def rows(self):
|
||
return [json.loads(p.read_text()) for p in self.receipts.glob('*.json')]
|
||
|
||
def resource_state(self):
|
||
# Exercise the existing release gate with a settled, empty local owner.
|
||
from harness import execution, paths, scheduler
|
||
locks = self.root / 'state/locks'
|
||
locks.mkdir(parents=True, exist_ok=True)
|
||
identity = {'execution_id': 'fixture', 'epoch': 1}
|
||
execution.atomic_json(self.rd / 'execution.json', dict(identity,
|
||
execution_state='TERMINAL', cleanup_state='VERIFIED', processes=[]))
|
||
execution.atomic_json(locks / 'board-0.lease.json', dict(identity,
|
||
pool='board', slot=0, holder={}, operation_id='op', execution_dir=str(self.rd)))
|
||
with patch.object(paths, 'STATE', locks.parent), \
|
||
patch.object(execution, 'process_status', return_value='DEAD'), \
|
||
patch.object(scheduler, '_board_idle_ps', return_value=None):
|
||
return scheduler.reconcile_leases()[0]['state']
|
||
|
||
def eval_failure(self, case):
|
||
from harness.eval import run as evaluate
|
||
baseline = self.root / 'baseline'
|
||
baseline.mkdir(exist_ok=True)
|
||
metrics = {'t1_manifest_sha': 'fixture', 'evaluator_version': evaluate.EVALUATOR_VERSION}
|
||
if case == 'baseline_mismatch':
|
||
metrics['t1_manifest_sha'] = 'different'
|
||
if case != 'missing_baseline':
|
||
(baseline / 'metrics.json').write_text(json.dumps(metrics))
|
||
argv = ['eval', '--mode', 'quant', '--runner', 'board', '--task', 'cls',
|
||
'--model', 'fixture', '--input-spec', '1x3x2x2', '--t1', 'missing.tar',
|
||
'--hbm', str(self.root / 'missing.hbm'), '--baseline', str(baseline),
|
||
'--out', str(self.root / 'eval.json')]
|
||
if case == 'version':
|
||
argv += ['--evaluator-version', 'invalid']
|
||
t1 = SimpleNamespace(manifest={'sha256': 'fixture'}, items=[])
|
||
with patch.object(evaluate.sys, 'argv', argv), \
|
||
patch.object(evaluate.T1, 'open', side_effect=FileNotFoundError('missing T1')
|
||
if case == 'missing_t1' else None, return_value=t1), \
|
||
patch.object(board_perf, 'run') as transport, \
|
||
patch.object(board_perf, 'guarded_run') as dispatch:
|
||
if case in ('version', 'baseline_mismatch'):
|
||
self.assertEqual(evaluate.main(), 66 if case == 'version' else 67)
|
||
else:
|
||
with self.assertRaises(FileNotFoundError):
|
||
evaluate.main()
|
||
transport.assert_not_called(); dispatch.assert_not_called()
|
||
self.assertFalse((self.root / 'eval.json').exists())
|
||
|
||
def test_local_eval_failures_keep_failure_and_release_without_ssh(self):
|
||
for case in ('missing_hbm', 'missing_baseline', 'missing_t1', 'baseline_mismatch', 'version'):
|
||
with self.subTest(case=case):
|
||
for path in self.receipts.glob('*.json'):
|
||
path.unlink()
|
||
(self.root / 'baseline/metrics.json').unlink(missing_ok=True)
|
||
self.eval_failure(case)
|
||
self.assertEqual(len(self.rows()), 1)
|
||
receipt = self.rows()[0]
|
||
self.assertEqual((receipt['cleanup_state'], receipt['proof']), ('VERIFIED', 'not_started'))
|
||
self.assertTrue(receipt['all_children_reaped'])
|
||
self.assertEqual(self.resource_state(), 'RELEASED')
|
||
|
||
def test_perf_configuration_failure_has_proof_without_ssh(self):
|
||
argv = ['perf', '--hbm', 'missing.hbm', '--job-id', 'fixture', '--out', str(self.root / 'perf.json')]
|
||
with patch.object(board_perf.sys, 'argv', argv), \
|
||
patch.object(board_perf, 'ssh_base', side_effect=SystemExit(4)), \
|
||
patch.object(board_perf, 'run') as transport, \
|
||
patch.object(board_perf, 'guarded_run') as dispatch:
|
||
with self.assertRaises(SystemExit) as error:
|
||
board_perf.main()
|
||
self.assertEqual(error.exception.code, 4)
|
||
transport.assert_not_called(); dispatch.assert_not_called()
|
||
self.assertEqual(self.resource_state(), 'RELEASED')
|
||
|
||
def test_prior_unknown_survives_new_local_validation_failure(self):
|
||
prior = self.receipts / 'prior.json'
|
||
prior.write_text('{"operation_id":"prior","cleanup_state":"UNKNOWN"}\n')
|
||
before = prior.read_bytes()
|
||
self.eval_failure('missing_hbm')
|
||
self.assertEqual(prior.read_bytes(), before)
|
||
self.assertEqual(len(self.rows()), 2)
|
||
self.assertEqual(self.resource_state(), 'QUARANTINED')
|
||
|
||
def test_process_exit_during_dispatch_keeps_unknown_before_finally(self):
|
||
import subprocess, sys
|
||
code = """import os
|
||
from unittest.mock import patch
|
||
from harness.tools import board_perf
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), patch.object(board_perf, 'guarded_run', side_effect=lambda *a: os._exit(77)):
|
||
session = board_perf.RemoteSession('fixture')
|
||
session.not_started('fixture')
|
||
session.bench([])
|
||
"""
|
||
result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True, timeout=10)
|
||
self.assertEqual(result.returncode, 77, result.stderr)
|
||
self.assertEqual({r['cleanup_state'] for r in self.rows()}, {'VERIFIED', 'UNKNOWN'})
|
||
self.assertEqual(self.resource_state(), 'QUARANTINED')
|
||
|
||
def test_dispatch_receipt_must_persist_before_transport(self):
|
||
from harness import execution
|
||
def fail_write(path, value):
|
||
self.assertEqual(value['cleanup_state'], 'UNKNOWN')
|
||
raise OSError('fsync failed')
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
||
patch.object(board_perf, 'guarded_run') as dispatch, \
|
||
patch.object(board_perf, 'run') as transport:
|
||
session = board_perf.RemoteSession('fixture')
|
||
session.not_started('fixture')
|
||
with patch.object(execution, 'atomic_json', side_effect=fail_write):
|
||
with self.assertRaisesRegex(OSError, 'fsync failed'):
|
||
session.bench([])
|
||
dispatch.assert_not_called(); transport.assert_not_called()
|
||
|
||
def test_real_receipt_replaces_same_pending_operation(self):
|
||
actual = {}
|
||
def dispatch(command, timeout):
|
||
args = shlex.split(command[-1]); op = args[args.index('--operation-id') + 1]
|
||
pending = json.loads((self.receipts / (op + '.json')).read_text())
|
||
self.assertEqual(pending['cleanup_state'], 'UNKNOWN')
|
||
self.assertEqual(pending['operation_id'], op)
|
||
actual.update(operation_id=op, guard_version=4, cleanup_state='VERIFIED',
|
||
all_children_reaped=True, proof='waitpid:ECHILD')
|
||
return '{"samples":1}'
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=dispatch), \
|
||
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **kw: json.dumps(actual)):
|
||
session = board_perf.RemoteSession('fixture')
|
||
session.not_started('fixture')
|
||
self.assertEqual(session.bench([])['samples'], 1)
|
||
self.assertEqual(len(self.rows()), 2)
|
||
self.assertEqual({r['cleanup_state'] for r in self.rows()}, {'VERIFIED'})
|
||
self.assertEqual(self.resource_state(), 'RELEASED')
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|