170 lines
8.7 KiB
Python
170 lines
8.7 KiB
Python
"""Control-plane races exercise the real execution commit lock, never a provider."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import collection_runtime as runtime, db, pipeline, rollout
|
|
from harness.collection_runtime import preflight as runtime_preflight
|
|
from harness.execution import Execution
|
|
from harness.quota import Broker, QuotaPolicy, Upstream
|
|
from harness.rollout import settle_execution
|
|
from scripts import test_collection as collection_fixtures
|
|
from scripts.test_execution import policy
|
|
|
|
|
|
class ControlTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.execution = Execution(Path(temp.name))
|
|
self.execution.start(policy())
|
|
|
|
def test_cancel_after_completion_has_no_side_effect(self):
|
|
self.execution.finish(0, protocol_complete=True)
|
|
with patch('harness.collection_runtime.broker') as broker:
|
|
result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True)
|
|
self.assertEqual(result['execution_outcome'], 'COMPLETED')
|
|
broker.assert_not_called()
|
|
self.assertIsNone(self.execution.snapshot()['cancel_requested'])
|
|
|
|
def test_completion_committed_between_snapshot_and_cancel_wins(self):
|
|
commit_cancel = self.execution.cancel
|
|
def race(reason):
|
|
self.execution.finish(0, protocol_complete=True)
|
|
return commit_cancel(reason)
|
|
with patch.object(self.execution, 'cancel', side_effect=race), patch('harness.collection_runtime.broker') as broker:
|
|
result = settle_execution('r', 'j', 0, self.execution, 'late cancel', cancel=True)
|
|
self.assertEqual(result['execution_outcome'], 'COMPLETED')
|
|
broker.assert_not_called()
|
|
self.assertIsNone(self.execution.snapshot()['cancel_requested'])
|
|
|
|
|
|
class ClaimQuotaTests(unittest.TestCase):
|
|
connect = collection_fixtures.Collection.connect
|
|
job = collection_fixtures.Collection.job
|
|
complete = collection_fixtures.Collection.complete
|
|
collect = collection_fixtures.Collection.collect
|
|
|
|
def setUp(self):
|
|
collection_fixtures.Collection.setUp(self)
|
|
self.ledger = self.root / 'quota.sqlite'
|
|
quota_policy = QuotaPolicy(max_concurrency=1, rpm=1, tpm=1000)
|
|
self.broker = Broker(self.ledger, {'fixture': quota_policy, 'other': quota_policy}, {
|
|
'teacher': Upstream('fixture', 'http://127.0.0.1:9/v1', 'fixture-secret', ('fixture',))})
|
|
self.addCleanup(self.broker.close)
|
|
self.runtime_config.return_value = ({'quota_ledger': str(self.ledger),
|
|
'credential_groups': {'teacher': 'fixture', 'student': 'other'}}, {})
|
|
self.runtime_preflight.side_effect = runtime_preflight
|
|
patch.object(runtime, 'broker', return_value=self.broker).start()
|
|
patch.object(runtime.paths, 'secret', return_value='fixture-secret').start()
|
|
patch.object(runtime.paths, 'config', return_value={
|
|
'pools': {'agent': {'workers': 3}}, 'paths': {'node_bin': 'node'},
|
|
'endpoints': {'teacher': {'api_key_env': 'FIXTURE_KEY'},
|
|
'student': {'api_key_env': 'OTHER_FIXTURE_KEY'}}}).start()
|
|
self.sdk = patch('subprocess.run', return_value=SimpleNamespace(returncode=0, stdout='0.73.1')).start()
|
|
self.http = patch.object(self.broker, 'generate', side_effect=AssertionError('unexpected model request')).start()
|
|
|
|
def group(self, state, reason=None, group='fixture'):
|
|
with self.broker._db() as con:
|
|
con.execute('UPDATE quota_groups SET state=?,reason=?,cooldown_until=? WHERE group_id=?',
|
|
(state, reason, time.time() + 600, group))
|
|
|
|
def rollout_rows(self):
|
|
with self.connect() as con:
|
|
return [dict(row) for row in con.execute('SELECT * FROM rollouts ORDER BY run_id,job_id,kk')]
|
|
|
|
def test_two_collectors_share_hard_block_without_claim_or_sdk(self):
|
|
self.job('s100p', k=2)
|
|
self.job('s600', k=2)
|
|
with self.connect() as con:
|
|
for table in ('jobs', 'rollouts', 'rollout_context'):
|
|
con.execute(f"UPDATE {table} SET run_id='other-run' WHERE job_id='s600'")
|
|
before = self.rollout_rows()
|
|
for state in ('RECONCILING', 'PAUSED', 'UNRECOGNIZED', ''):
|
|
with self.subTest(state=state):
|
|
self.group(state, 'REQUEST_OUTCOME_UNKNOWN' if state in ('RECONCILING', 'PAUSED') else None)
|
|
with patch.object(rollout, 'run_rollout') as launch, ThreadPoolExecutor(max_workers=2) as collectors:
|
|
futures = [collectors.submit(pipeline.collect_rollouts, run, parallel=1, log=lambda _: None)
|
|
for run in ('run', 'other-run')]
|
|
self.assertEqual([future.result(timeout=5) for future in futures], [{}, {}])
|
|
launch.assert_not_called()
|
|
self.assertEqual(self.rollout_rows(), before)
|
|
self.sdk.assert_not_called()
|
|
self.http.assert_not_called()
|
|
self.assertEqual(self.broker.snapshot()['executions'], [])
|
|
self.assertFalse((self.root / 'runs').exists())
|
|
|
|
def test_missing_unreadable_or_missing_group_keeps_queued(self):
|
|
self.job('queued', k=2)
|
|
before = self.rollout_rows()
|
|
invalid = self.root / 'invalid.sqlite'
|
|
invalid.write_bytes(b'not a SQLite database')
|
|
for ledger, group in ((self.root / 'missing.sqlite', 'fixture'), (invalid, 'fixture'),
|
|
(self.ledger, 'absent')):
|
|
with self.subTest(ledger=ledger.name, group=group):
|
|
self.runtime_config.return_value = ({'quota_ledger': str(ledger),
|
|
'credential_groups': {'teacher': group}}, {})
|
|
with patch.object(rollout, 'run_rollout') as launch:
|
|
self.assertEqual(self.collect(), {})
|
|
launch.assert_not_called()
|
|
self.assertEqual(self.rollout_rows(), before)
|
|
self.sdk.assert_not_called()
|
|
self.http.assert_not_called()
|
|
|
|
def test_normal_cooling_and_spent_budget_still_use_existing_request_queue(self):
|
|
now = time.time()
|
|
with self.broker._db() as con:
|
|
con.execute("INSERT INTO attempts(attempt_id,generation_id,group_id,owner,state,state_at_send,"
|
|
"epoch,started_at,ended_at,deadline,reserved_tokens,charged_tokens,estimator,sent) "
|
|
"VALUES ('spent','spent','fixture','fixture','FINISHED','OPEN',0,?,?,?,?,?,'fixture',1)",
|
|
(now, now, now + 600, 1000, 1000))
|
|
for state in ('OPEN', 'COOLING', 'HALF_OPEN', 'RECOVERING'):
|
|
with self.subTest(state=state):
|
|
self.group(state)
|
|
self.job(state, k=1)
|
|
with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch:
|
|
self.assertEqual(self.collect(job_ids=[state], parallel=1, limit=1), {'PASS': 1})
|
|
launch.assert_called_once()
|
|
self.assertEqual(self.sdk.call_count, 4)
|
|
self.http.assert_not_called()
|
|
|
|
def test_blocked_other_group_does_not_hold_selected_policy(self):
|
|
self.group('PAUSED', 'UPSTREAM_AUTH_OR_HARD_QUOTA', group='other')
|
|
self.job('selected', k=1)
|
|
with patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch:
|
|
self.assertEqual(self.collect(parallel=1, limit=1), {'PASS': 1})
|
|
launch.assert_called_once()
|
|
|
|
def test_watch_rechecks_without_new_recovery_path(self):
|
|
self.group('RECONCILING', 'REQUEST_OUTCOME_UNKNOWN')
|
|
self.job('queued', k=1)
|
|
checked = threading.Event()
|
|
original_snapshot = runtime.quota_snapshot
|
|
def snapshot(*args, **kwargs):
|
|
result = original_snapshot(*args, **kwargs)
|
|
checked.set()
|
|
return result
|
|
with patch.object(runtime, 'quota_snapshot', side_effect=snapshot), \
|
|
patch.object(rollout, 'run_rollout', side_effect=self.complete) as launch, \
|
|
ThreadPoolExecutor(max_workers=1) as collector:
|
|
future = collector.submit(self.collect, parallel=1, limit=1, watch=True)
|
|
try:
|
|
self.assertTrue(checked.wait(3))
|
|
self.assertEqual(self.rollout_rows()[0]['state'], 'QUEUED')
|
|
self.assertIsNone(self.rollout_rows()[0]['started_at'])
|
|
launch.assert_not_called()
|
|
self.group('OPEN')
|
|
self.assertEqual(future.result(timeout=8), {'PASS': 1})
|
|
finally:
|
|
pipeline.collect_drain_file('run').touch()
|
|
launch.assert_called_once()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|