"""Control-plane races exercise the real execution commit lock, never a provider.""" from concurrent.futures import ThreadPoolExecutor from pathlib import Path import tempfile import threading import time from types import SimpleNamespace import unittest from unittest.mock import patch from harness import collection_runtime as runtime, db, pipeline, rollout from harness.collection_runtime import preflight as runtime_preflight from harness.execution import Execution from harness.quota import Broker, QuotaPolicy, Upstream from harness.rollout import settle_execution from scripts import test_collection as collection_fixtures from scripts.test_execution import policy class ControlTests(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory() self.addCleanup(temp.cleanup) self.execution = Execution(Path(temp.name)) self.execution.start(policy()) def test_cancel_after_completion_has_no_side_effect(self): self.execution.finish(0, protocol_complete=True) with patch('harness.collection_runtime.broker') as broker: result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True) self.assertEqual(result['execution_outcome'], 'COMPLETED') broker.assert_not_called() self.assertIsNone(self.execution.snapshot()['cancel_requested']) def test_completion_committed_between_snapshot_and_cancel_wins(self): commit_cancel = self.execution.cancel def race(reason): self.execution.finish(0, protocol_complete=True) return commit_cancel(reason) with patch.object(self.execution, 'cancel', side_effect=race), patch('harness.collection_runtime.broker') as broker: result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True) self.assertEqual(result['execution_outcome'], 'COMPLETED') broker.assert_not_called() self.assertIsNone(self.execution.snapshot()['cancel_requested']) class ClaimQuotaTests(unittest.TestCase): connect = collection_fixtures.Collection.connect job = collection_fixtures.Collection.job complete = collection_fixtures.Collection.complete collect = collection_fixtures.Collection.collect def setUp(self): collection_fixtures.Collection.setUp(self) self.ledger = self.root / 'quota.sqlite' quota_policy = QuotaPolicy(max_concurrency=1, rpm=1, tpm=1000) self.broker = Broker(self.ledger, {'fixture': quota_policy, 'other': quota_policy}, { 'teacher': Upstream('fixture', 'http://127.0.0.1:9/v1', 'fixture-secret', ('fixture',))}) self.addCleanup(self.broker.close) self.runtime_config.return_value = ({'quota_ledger': str(self.ledger), 'credential_groups': {'teacher': 'fixture', 'student': 'other'}}, {}) self.runtime_preflight.side_effect = runtime_preflight patch.object(runtime, 'broker', return_value=self.broker).start() patch.object(runtime.paths, 'secret', return_value='fixture-secret').start() patch.object(runtime.paths, 'config', return_value={ 'pools': {'agent': {'workers': 3}}, 'paths': {'node_bin': 'node'}, 'endpoints': {'teacher': {'api_key_env': 'FIXTURE_KEY'}, 'student': {'api_key_env': 'OTHER_FIXTURE_KEY'}}}).start() self.sdk = patch('subprocess.run', return_value=SimpleNamespace(returncode=0, stdout='0.73.1')).start() self.http = patch.object(self.broker, 'generate', side_effect=AssertionError('unexpected model request')).start() def group(self, state, reason=None, group='fixture'): with self.broker._db() as con: con.execute('UPDATE quota_groups SET state=?,reason=?,cooldown_until=? WHERE group_id=?', (state, reason, time.time() + 600, group)) def rollout_rows(self): with self.connect() as con: return [dict(row) for row in con.execute('SELECT * FROM rollouts ORDER BY run_id,job_id,kk')] def test_two_collectors_share_hard_block_without_claim_or_sdk(self): self.job('s100p', k=2) self.job('s600', k=2) with self.connect() as con: for table in ('jobs', 'rollouts', 'rollout_context'): con.execute(f"UPDATE {table} SET run_id='other-run' WHERE job_id='s600'") before = self.rollout_rows() for state in ('RECONCILING', 'PAUSED', 'UNRECOGNIZED', ''): with self.subTest(state=state): self.group(state, 'REQUEST_OUTCOME_UNKNOWN' if state in ('RECONCILING', 'PAUSED') else None) with patch.object(rollout, 'run_rollout') as launch, ThreadPoolExecutor(max_workers=2) as collectors: futures = [collectors.submit(pipeline.collect_rollouts, run, parallel=1, log=lambda _: None) for run in ('run', 'other-run')] self.assertEqual([future.result(timeout=5) for future in futures], [{}, {}]) launch.assert_not_called() self.assertEqual(self.rollout_rows(), before) self.sdk.assert_not_called() self.http.assert_not_called() self.assertEqual(self.broker.snapshot()['executions'], []) self.assertFalse((self.root / 'runs').exists()) def test_missing_unreadable_or_missing_group_keeps_queued(self): self.job('queued', k=2) before = self.rollout_rows() invalid = self.root / 'invalid.sqlite' invalid.write_bytes(b'not a SQLite database') for ledger, group in ((self.root / 'missing.sqlite', 'fixture'), (invalid, 'fixture'), (self.ledger, 'absent')): with self.subTest(ledger=ledger.name, group=group): self.runtime_config.return_value = ({'quota_ledger': str(ledger), 'credential_groups': {'teacher': group}}, {}) with patch.object(rollout, 'run_rollout') as launch: self.assertEqual(self.collect(), {}) launch.assert_not_called() self.assertEqual(self.rollout_rows(), before) self.sdk.assert_not_called() self.http.assert_not_called() def test_normal_cooling_and_spent_budget_still_use_existing_request_queue(self): now = time.time() with self.broker._db() as con: con.execute("INSERT INTO attempts(attempt_id,generation_id,group_id,owner,state,state_at_send," "epoch,started_at,ended_at,deadline,reserved_tokens,charged_tokens,estimator,sent) " "VALUES ('spent','spent','fixture','fixture','FINISHED','OPEN',0,?,?,?,?,?,'fixture',1)", (now, now, now + 600, 1000, 1000)) for state in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'): with self.subTest(state=state): self.group(state) self.job(state, k=1) with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch: self.assertEqual(self.collect(job_ids=[state], parallel=1, limit=1), {'PASS': 1}) launch.assert_called_once() self.assertEqual(self.sdk.call_count, 4) self.http.assert_not_called() def test_blocked_other_group_does_not_hold_selected_policy(self): self.group('PAUSED', 'UPSTREAM_AUTH_OR_HARD_QUOTA', group='other') self.job('selected', k=1) with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch: self.assertEqual(self.collect(parallel=1, limit=1), {'PASS': 1}) launch.assert_called_once() def test_watch_rechecks_without_new_recovery_path(self): self.group('RECONCILING', 'REQUEST_OUTCOME_UNKNOWN') self.job('queued', k=1) checked = threading.Event() original_snapshot = runtime.quota_snapshot def snapshot(*args, **kwargs): result = original_snapshot(*args, **kwargs) checked.set() return result with patch.object(runtime, 'quota_snapshot', side_effect=snapshot), \ patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch, \ ThreadPoolExecutor(max_workers=1) as collector: future = collector.submit(self.collect, parallel=1, limit=1, watch=True) try: self.assertTrue(checked.wait(3)) self.assertEqual(self.rollout_rows()[0]['state'], 'QUEUED') self.assertIsNone(self.rollout_rows()[0]['started_at']) launch.assert_not_called() self.group('OPEN') self.assertEqual(future.result(timeout=8), {'PASS': 1}) finally: pipeline.collect_drain_file('run').touch() launch.assert_called_once() if __name__ == '__main__': unittest.main()