data-gen-bpu/scripts/test_collection_control.py

170 lines
8.7 KiB
Python

"""Control-plane races exercise the real execution commit lock, never a provider."""
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import tempfile
import threading
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import collection_runtime as runtime, db, pipeline, rollout
from harness.collection_runtime import preflight as runtime_preflight
from harness.execution import Execution
from harness.quota import Broker, QuotaPolicy, Upstream
from harness.rollout import settle_execution
from scripts import test_collection as collection_fixtures
from scripts.test_execution import policy
class ControlTests(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.execution = Execution(Path(temp.name))
self.execution.start(policy())
def test_cancel_after_completion_has_no_side_effect(self):
self.execution.finish(0, protocol_complete=True)
with patch('harness.collection_runtime.broker') as broker:
result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True)
self.assertEqual(result['execution_outcome'], 'COMPLETED')
broker.assert_not_called()
self.assertIsNone(self.execution.snapshot()['cancel_requested'])
def test_completion_committed_between_snapshot_and_cancel_wins(self):
commit_cancel = self.execution.cancel
def race(reason):
self.execution.finish(0, protocol_complete=True)
return commit_cancel(reason)
with patch.object(self.execution, 'cancel', side_effect=race), patch('harness.collection_runtime.broker') as broker:
result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True)
self.assertEqual(result['execution_outcome'], 'COMPLETED')
broker.assert_not_called()
self.assertIsNone(self.execution.snapshot()['cancel_requested'])
class ClaimQuotaTests(unittest.TestCase):
connect = collection_fixtures.Collection.connect
job = collection_fixtures.Collection.job
complete = collection_fixtures.Collection.complete
collect = collection_fixtures.Collection.collect
def setUp(self):
collection_fixtures.Collection.setUp(self)
self.ledger = self.root / 'quota.sqlite'
quota_policy = QuotaPolicy(max_concurrency=1, rpm=1, tpm=1000)
self.broker = Broker(self.ledger, {'fixture': quota_policy, 'other': quota_policy}, {
'teacher': Upstream('fixture', 'http://127.0.0.1:9/v1', 'fixture-secret', ('fixture',))})
self.addCleanup(self.broker.close)
self.runtime_config.return_value = ({'quota_ledger': str(self.ledger),
'credential_groups': {'teacher': 'fixture', 'student': 'other'}}, {})
self.runtime_preflight.side_effect = runtime_preflight
patch.object(runtime, 'broker', return_value=self.broker).start()
patch.object(runtime.paths, 'secret', return_value='fixture-secret').start()
patch.object(runtime.paths, 'config', return_value={
'pools': {'agent': {'workers': 3}}, 'paths': {'node_bin': 'node'},
'endpoints': {'teacher': {'api_key_env': 'FIXTURE_KEY'},
'student': {'api_key_env': 'OTHER_FIXTURE_KEY'}}}).start()
self.sdk = patch('subprocess.run', return_value=SimpleNamespace(returncode=0, stdout='0.73.1')).start()
self.http = patch.object(self.broker, 'generate', side_effect=AssertionError('unexpected model request')).start()
def group(self, state, reason=None, group='fixture'):
with self.broker._db() as con:
con.execute('UPDATE quota_groups SET state=?,reason=?,cooldown_until=? WHERE group_id=?',
(state, reason, time.time() + 600, group))
def rollout_rows(self):
with self.connect() as con:
return [dict(row) for row in con.execute('SELECT * FROM rollouts ORDER BY run_id,job_id,kk')]
def test_two_collectors_share_hard_block_without_claim_or_sdk(self):
self.job('s100p', k=2)
self.job('s600', k=2)
with self.connect() as con:
for table in ('jobs', 'rollouts', 'rollout_context'):
con.execute(f"UPDATE {table} SET run_id='other-run' WHERE job_id='s600'")
before = self.rollout_rows()
for state in ('RECONCILING', 'PAUSED', 'UNRECOGNIZED', ''):
with self.subTest(state=state):
self.group(state, 'REQUEST_OUTCOME_UNKNOWN' if state in ('RECONCILING', 'PAUSED') else None)
with patch.object(rollout, 'run_rollout') as launch, ThreadPoolExecutor(max_workers=2) as collectors:
futures = [collectors.submit(pipeline.collect_rollouts, run, parallel=1, log=lambda _: None)
for run in ('run', 'other-run')]
self.assertEqual([future.result(timeout=5) for future in futures], [{}, {}])
launch.assert_not_called()
self.assertEqual(self.rollout_rows(), before)
self.sdk.assert_not_called()
self.http.assert_not_called()
self.assertEqual(self.broker.snapshot()['executions'], [])
self.assertFalse((self.root / 'runs').exists())
def test_missing_unreadable_or_missing_group_keeps_queued(self):
self.job('queued', k=2)
before = self.rollout_rows()
invalid = self.root / 'invalid.sqlite'
invalid.write_bytes(b'not a SQLite database')
for ledger, group in ((self.root / 'missing.sqlite', 'fixture'), (invalid, 'fixture'),
(self.ledger, 'absent')):
with self.subTest(ledger=ledger.name, group=group):
self.runtime_config.return_value = ({'quota_ledger': str(ledger),
'credential_groups': {'teacher': group}}, {})
with patch.object(rollout, 'run_rollout') as launch:
self.assertEqual(self.collect(), {})
launch.assert_not_called()
self.assertEqual(self.rollout_rows(), before)
self.sdk.assert_not_called()
self.http.assert_not_called()
def test_normal_cooling_and_spent_budget_still_use_existing_request_queue(self):
now = time.time()
with self.broker._db() as con:
con.execute("INSERT INTO attempts(attempt_id,generation_id,group_id,owner,state,state_at_send,"
"epoch,started_at,ended_at,deadline,reserved_tokens,charged_tokens,estimator,sent) "
"VALUES ('spent','spent','fixture','fixture','FINISHED','OPEN',0,?,?,?,?,?,'fixture',1)",
(now, now, now + 600, 1000, 1000))
for state in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'):
with self.subTest(state=state):
self.group(state)
self.job(state, k=1)
with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch:
self.assertEqual(self.collect(job_ids=[state], parallel=1, limit=1), {'PASS': 1})
launch.assert_called_once()
self.assertEqual(self.sdk.call_count, 4)
self.http.assert_not_called()
def test_blocked_other_group_does_not_hold_selected_policy(self):
self.group('PAUSED', 'UPSTREAM_AUTH_OR_HARD_QUOTA', group='other')
self.job('selected', k=1)
with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch:
self.assertEqual(self.collect(parallel=1, limit=1), {'PASS': 1})
launch.assert_called_once()
def test_watch_rechecks_without_new_recovery_path(self):
self.group('RECONCILING', 'REQUEST_OUTCOME_UNKNOWN')
self.job('queued', k=1)
checked = threading.Event()
original_snapshot = runtime.quota_snapshot
def snapshot(*args, **kwargs):
result = original_snapshot(*args, **kwargs)
checked.set()
return result
with patch.object(runtime, 'quota_snapshot', side_effect=snapshot), \
patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch, \
ThreadPoolExecutor(max_workers=1) as collector:
future = collector.submit(self.collect, parallel=1, limit=1, watch=True)
try:
self.assertTrue(checked.wait(3))
self.assertEqual(self.rollout_rows()[0]['state'], 'QUEUED')
self.assertIsNone(self.rollout_rows()[0]['started_at'])
launch.assert_not_called()
self.group('OPEN')
self.assertEqual(future.result(timeout=8), {'PASS': 1})
finally:
pipeline.collect_drain_file('run').touch()
launch.assert_called_once()
if __name__ == '__main__':
unittest.main()