和容器被杀、执行器退出一样算系统的错,每条轨迹合计最多免 3 次。 10-12 17:16 后 DETR 两次编译一小时没日志没 CPU 被看门狗杀掉、 segformer.b5 编译逼近 4 小时上限,原来都会白扣模型一次机会。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
71 lines
4.5 KiB
Python
71 lines
4.5 KiB
Python
"""Compile attempts are the primary per-trajectory budget."""
|
|
import sqlite3
|
|
import unittest
|
|
|
|
from harness import db, stagerun
|
|
|
|
|
|
class CompileBudget(unittest.TestCase):
|
|
def test_counts_started_compiles_of_this_rollout_only(self):
|
|
con = sqlite3.connect(':memory:')
|
|
con.executescript(db.SCHEMA)
|
|
rows = [('05-compile', 0, 0, 'agent-long'), ('05-compile', 0, 1, 'agent-long'),
|
|
('05-compile', 0, 2, 'queued'), # timed out waiting for a slot: never compiled
|
|
('04-calibrate', 0, 0, 'agent'), # calibration is not a compile
|
|
('05-compile', 1, 0, 'agent-long')] # a sibling rollout has its own budget
|
|
for stage, rollout, attempt, lane in rows:
|
|
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)"
|
|
" VALUES ('r', 'j', ?, ?, ?, ?, 'FAIL', 0)", (rollout, stage, attempt, lane))
|
|
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2)
|
|
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
|
|
|
|
def test_infra_kills_are_free_up_to_a_cap(self):
|
|
con = sqlite3.connect(':memory:')
|
|
con.executescript(db.SCHEMA)
|
|
errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown',
|
|
'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles
|
|
for attempt, error in enumerate(errors):
|
|
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
|
|
" VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
|
|
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES)
|
|
# A compile past the phase time limit and one the watchdog killed as stalled are free too.
|
|
for attempt, error in ((0, 'PHASE_EXECUTION_DEADLINE_EXCEEDED'), (1, 'stalled'), (2, 'unknown')):
|
|
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
|
|
" VALUES ('r', 'j', 1, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
|
|
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
|
|
|
|
def test_compile_past_the_cap_is_refused_before_any_work(self):
|
|
import json, tempfile
|
|
from contextlib import nullcontext
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
with tempfile.TemporaryDirectory() as d:
|
|
rd = Path(d) / '00'; hd = rd / 'handles/h'; hd.mkdir(parents=True)
|
|
(hd / 'request.json').write_text('{"stage":"05-compile"}')
|
|
ctx = SimpleNamespace(run_id='r', job={'job_id': 'j', 'backend': 'ptq'}, rollout=0, lane='normal',
|
|
extra={}, agent_config=None)
|
|
stage = MagicMock(name='stage'); stage.name = '05-compile'; stage.pool = 'cpu'
|
|
for used, refused in ((stagerun.MAX_COMPILE_ATTEMPTS - 1, False), (stagerun.MAX_COMPILE_ATTEMPTS, True)):
|
|
stage.run.reset_mock()
|
|
stage.run.return_value = SimpleNamespace(state='OK', exit_code=0, error_class=None, duration_s=0, measurements={})
|
|
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
|
|
patch.dict(stagerun.BY_NAME, {'05-compile': lambda: stage}), \
|
|
patch.object(stagerun.db, 'connect', side_effect=lambda: nullcontext(MagicMock())), \
|
|
patch.object(stagerun.scheduler, 'acquire', return_value=nullcontext(1)), \
|
|
patch.object(stagerun.db, 'stage_start') as start, patch.object(stagerun.db, 'stage_finish'), \
|
|
patch.object(stagerun, 'compile_attempts_used', return_value=used), \
|
|
patch.object(Path, 'exists', lambda self: True if self.name == 'compile.yaml' else Path.is_file(self) or Path.is_dir(self)):
|
|
stagerun._exec(hd)
|
|
status = json.loads((hd / 'status.json').read_text())
|
|
if refused:
|
|
self.assertEqual((status['result'], status['error_class']), ('FAIL', 'COMPILE_BUDGET_EXHAUSTED'))
|
|
stage.run.assert_not_called(); start.assert_not_called()
|
|
else:
|
|
self.assertNotEqual(status.get('error_class'), 'COMPILE_BUDGET_EXHAUSTED')
|
|
stage.run.assert_called_once()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|