data-gen-bpu/scripts/test_compile_budget.py
bench 375ce4d1f5 stagerun: 编译超时、看门狗判卡死也不扣编译次数
和容器被杀、执行器退出一样算系统的错,每条轨迹合计最多免 3 次。
10-12 17:16 后 DETR 两次编译一小时没日志没 CPU 被看门狗杀掉、
segformer.b5 编译逼近 4 小时上限,原来都会白扣模型一次机会。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:44:24 +08:00

71 lines
4.5 KiB
Python

"""Compile attempts are the primary per-trajectory budget."""
import sqlite3
import unittest
from harness import db, stagerun
class CompileBudget(unittest.TestCase):
def test_counts_started_compiles_of_this_rollout_only(self):
con = sqlite3.connect(':memory:')
con.executescript(db.SCHEMA)
rows = [('05-compile', 0, 0, 'agent-long'), ('05-compile', 0, 1, 'agent-long'),
('05-compile', 0, 2, 'queued'), # timed out waiting for a slot: never compiled
('04-calibrate', 0, 0, 'agent'), # calibration is not a compile
('05-compile', 1, 0, 'agent-long')] # a sibling rollout has its own budget
for stage, rollout, attempt, lane in rows:
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)"
" VALUES ('r', 'j', ?, ?, ?, ?, 'FAIL', 0)", (rollout, stage, attempt, lane))
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2)
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
def test_infra_kills_are_free_up_to_a_cap(self):
con = sqlite3.connect(':memory:')
con.executescript(db.SCHEMA)
errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown',
'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles
for attempt, error in enumerate(errors):
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
" VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES)
# A compile past the phase time limit and one the watchdog killed as stalled are free too.
for attempt, error in ((0, 'PHASE_EXECUTION_DEADLINE_EXCEEDED'), (1, 'stalled'), (2, 'unknown')):
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
" VALUES ('r', 'j', 1, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
def test_compile_past_the_cap_is_refused_before_any_work(self):
import json, tempfile
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
with tempfile.TemporaryDirectory() as d:
rd = Path(d) / '00'; hd = rd / 'handles/h'; hd.mkdir(parents=True)
(hd / 'request.json').write_text('{"stage":"05-compile"}')
ctx = SimpleNamespace(run_id='r', job={'job_id': 'j', 'backend': 'ptq'}, rollout=0, lane='normal',
extra={}, agent_config=None)
stage = MagicMock(name='stage'); stage.name = '05-compile'; stage.pool = 'cpu'
for used, refused in ((stagerun.MAX_COMPILE_ATTEMPTS - 1, False), (stagerun.MAX_COMPILE_ATTEMPTS, True)):
stage.run.reset_mock()
stage.run.return_value = SimpleNamespace(state='OK', exit_code=0, error_class=None, duration_s=0, measurements={})
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
patch.dict(stagerun.BY_NAME, {'05-compile': lambda: stage}), \
patch.object(stagerun.db, 'connect', side_effect=lambda: nullcontext(MagicMock())), \
patch.object(stagerun.scheduler, 'acquire', return_value=nullcontext(1)), \
patch.object(stagerun.db, 'stage_start') as start, patch.object(stagerun.db, 'stage_finish'), \
patch.object(stagerun, 'compile_attempts_used', return_value=used), \
patch.object(Path, 'exists', lambda self: True if self.name == 'compile.yaml' else Path.is_file(self) or Path.is_dir(self)):
stagerun._exec(hd)
status = json.loads((hd / 'status.json').read_text())
if refused:
self.assertEqual((status['result'], status['error_class']), ('FAIL', 'COMPILE_BUDGET_EXHAUSTED'))
stage.run.assert_not_called(); start.assert_not_called()
else:
self.assertNotEqual(status.get('error_class'), 'COMPILE_BUDGET_EXHAUSTED')
stage.run.assert_called_once()
if __name__ == '__main__':
unittest.main()