"""Compile attempts are the primary per-trajectory budget.""" import sqlite3 import unittest from harness import db, stagerun class CompileBudget(unittest.TestCase): def test_counts_started_compiles_of_this_rollout_only(self): con = sqlite3.connect(':memory:') con.executescript(db.SCHEMA) rows = [('05-compile', 0, 0, 'agent-long'), ('05-compile', 0, 1, 'agent-long'), ('05-compile', 0, 2, 'queued'), # timed out waiting for a slot: never compiled ('04-calibrate', 0, 0, 'agent'), # calibration is not a compile ('05-compile', 1, 0, 'agent-long')] # a sibling rollout has its own budget for stage, rollout, attempt, lane in rows: con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)" " VALUES ('r', 'j', ?, ?, ?, ?, 'FAIL', 0)", (rollout, stage, attempt, lane)) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1) def test_infra_kills_are_free_up_to_a_cap(self): con = sqlite3.connect(':memory:') con.executescript(db.SCHEMA) errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown', 'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles for attempt, error in enumerate(errors): con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)" " VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error)) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES) # A compile past the phase time limit and one the watchdog killed as stalled are free too. for attempt, error in ((0, 'PHASE_EXECUTION_DEADLINE_EXCEEDED'), (1, 'stalled'), (2, 'unknown')): con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)" " VALUES ('r', 'j', 1, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error)) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1) def test_compile_past_the_cap_is_refused_before_any_work(self): import json, tempfile from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch with tempfile.TemporaryDirectory() as d: rd = Path(d) / '00'; hd = rd / 'handles/h'; hd.mkdir(parents=True) (hd / 'request.json').write_text('{"stage":"05-compile"}') ctx = SimpleNamespace(run_id='r', job={'job_id': 'j', 'backend': 'ptq'}, rollout=0, lane='normal', extra={}, agent_config=None) stage = MagicMock(name='stage'); stage.name = '05-compile'; stage.pool = 'cpu' for used, refused in ((stagerun.MAX_COMPILE_ATTEMPTS - 1, False), (stagerun.MAX_COMPILE_ATTEMPTS, True)): stage.run.reset_mock() stage.run.return_value = SimpleNamespace(state='OK', exit_code=0, error_class=None, duration_s=0, measurements={}) with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \ patch.dict(stagerun.BY_NAME, {'05-compile': lambda: stage}), \ patch.object(stagerun.db, 'connect', side_effect=lambda: nullcontext(MagicMock())), \ patch.object(stagerun.scheduler, 'acquire', return_value=nullcontext(1)), \ patch.object(stagerun.db, 'stage_start') as start, patch.object(stagerun.db, 'stage_finish'), \ patch.object(stagerun, 'compile_attempts_used', return_value=used), \ patch.object(Path, 'exists', lambda self: True if self.name == 'compile.yaml' else Path.is_file(self) or Path.is_dir(self)): stagerun._exec(hd) status = json.loads((hd / 'status.json').read_text()) if refused: self.assertEqual((status['result'], status['error_class']), ('FAIL', 'COMPILE_BUDGET_EXHAUSTED')) stage.run.assert_not_called(); start.assert_not_called() else: self.assertNotEqual(status.get('error_class'), 'COMPILE_BUDGET_EXHAUSTED') stage.run.assert_called_once() if __name__ == '__main__': unittest.main()