312 lines
18 KiB
Python
312 lines
18 KiB
Python
"""Real Pi processes share one broker ledger and a controlled HTTP provider."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness.quota import Broker, QuotaPolicy, Upstream
|
|
from scripts import test_pi_session as sdk_fixture
|
|
from scripts.test_quota import Provider, ok
|
|
|
|
|
|
def write_response():
|
|
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'write-once',
|
|
'type': 'function', 'function': {'name': 'write',
|
|
'arguments': json.dumps({'path': 'side-effect.txt', 'content': 'once'})}}]}
|
|
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
|
|
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
|
|
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
|
|
body = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode()
|
|
return 200, {'Content-Type': 'text/event-stream'}, body + b'data: [DONE]\n\n'
|
|
|
|
|
|
@unittest.skipUnless(sdk_fixture.NODE.is_file(), 'installed Pi 0.73.1 SDK required')
|
|
class JointSDKQuota(unittest.TestCase):
|
|
def setup_shared(self, responder, **limits):
|
|
self.provider = Provider(responder)
|
|
self.addCleanup(self.provider.close)
|
|
policy = QuotaPolicy(**dict({'max_concurrency': 2, 'rpm': 1000, 'tpm': 10000000,
|
|
'max_attempts': 3, 'window_s': .2, 'wait_budget_s': 1,
|
|
'cooldown_initial_s': .05, 'cooldown_local_max_s': .1, 'jitter_fraction': 0,
|
|
'connect_timeout_s': .5, 'first_byte_timeout_s': 2,
|
|
'stream_idle_timeout_s': .5, 'request_total_s': 3, 'poll_s': .005}, **limits))
|
|
self.template = sdk_fixture.PiSDKTests()
|
|
self.addCleanup(self.template.doCleanups)
|
|
# Reuse the SDK fixture's settings/models/private-home setup. Its unused
|
|
# provider is closed; all actual SDK requests use the controlled server.
|
|
with patch('scripts.test_pi_session.Broker', side_effect=lambda path, *_:
|
|
Broker(path, {'account': policy}, {'teacher': Upstream(
|
|
'account', self.provider.url, 'upstream-fixture', ('fixture',))})):
|
|
self.template.setUp()
|
|
self.template.server.shutdown()
|
|
self.template.server.server_close()
|
|
self.broker = self.template.broker
|
|
self.runners = []
|
|
|
|
def runner(self):
|
|
runner = sdk_fixture.PiSDKTests()
|
|
runner.root = self.template.root / ('project-' + str(len(self.runners)))
|
|
runner.home = runner.root / 'home'
|
|
runner.agent_dir = runner.home / '.pi/agent'
|
|
shutil.copytree(self.template.home, runner.home)
|
|
shutil.copytree(self.template.root / '.pi', runner.root / '.pi')
|
|
execution_id = 'joint-peer-' + str(len(self.runners))
|
|
runner.scope = self.broker.issue_scope(execution_id, 'teacher', 'fixture')
|
|
runner.config = copy.deepcopy(self.template.config)
|
|
runner.config.update(execution_id=execution_id, session_dir=str(runner.root / '.pi-sessions'),
|
|
append_system_path=str(runner.root / 'system.md'), prompt_path=str(runner.root / 'prompt.md'))
|
|
(runner.root / 'system.md').write_text('Use write once, then report completion.')
|
|
(runner.root / 'prompt.md').write_text(execution_id + ': write once.')
|
|
self.runners.append(runner)
|
|
return runner
|
|
|
|
def together(self, runners):
|
|
with ThreadPoolExecutor(max_workers=len(runners)) as pool:
|
|
return list(pool.map(lambda r: r.run_sdk(), runners))
|
|
|
|
def wait_for(self, predicate, timeout=5):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
time.sleep(.01)
|
|
self.fail('bounded condition did not arrive')
|
|
|
|
def resume(self, runner):
|
|
checkpoint = json.loads((runner.root / '.bench-session.json').read_bytes())
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['resumable'])
|
|
self.assertEqual(checkpoint['phase'], 'PAUSED_INFRA')
|
|
header = json.loads((runner.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.wait_for(lambda: self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
|
|
self.broker.resume(runner.scope.execution_id, checkpoint['generation_id'],
|
|
checkpoint_complete=True, side_effects_known=True)
|
|
runner.config.update(resume=True, session_path=checkpoint['session_path'],
|
|
session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'],
|
|
system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'],
|
|
nonempty_delta=checkpoint['nonempty_delta'])
|
|
|
|
def assert_metered(self):
|
|
snapshot = self.broker.snapshot()
|
|
sent = [a for a in snapshot['attempts'] if a['sent'] == 1]
|
|
self.assertEqual(len(sent), len(self.provider.requests))
|
|
self.assertTrue(all(r['authorization'] == 'Bearer upstream-fixture' for r in self.provider.requests))
|
|
self.assertTrue(all(a['group_id'] == 'account' for a in sent))
|
|
probes = sorted((a for a in sent if a['state_at_send'] == 'HALF_OPEN'), key=lambda a: a['started_at'])
|
|
for before, after in zip(probes, probes[1:]):
|
|
self.assertGreaterEqual(after['started_at'], before['ended_at'])
|
|
self.assertNotEqual(after['epoch'], before['epoch'])
|
|
for attempt in sent:
|
|
cooling = [e for e in snapshot['quota_events'] if e['event'] == 'COOLING' and e['at'] <= attempt['started_at']]
|
|
if cooling:
|
|
self.assertGreaterEqual(attempt['started_at'], json.loads(cooling[-1]['detail'])['cooldown_until'])
|
|
return snapshot
|
|
|
|
def receipt(self, case, **facts):
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True))
|
|
|
|
def test_persistent_429_multiple_sdk_processes_share_cooling_and_budget(self):
|
|
self.setup_shared(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
|
|
runners = [self.runner(), self.runner()]
|
|
results = self.together(runners)
|
|
self.assertEqual([r.returncode for r in results], [76, 76])
|
|
snapshot = self.assert_metered()
|
|
generations = snapshot['generations']
|
|
self.assertEqual(len(generations), 2)
|
|
self.assertTrue(all(g['attempts'] == 3 and g['state'] == 'PAUSED' for g in generations))
|
|
self.assertTrue(any(a['state_at_send'] == 'HALF_OPEN' for a in snapshot['attempts']))
|
|
before = len(self.provider.requests)
|
|
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
|
|
self.assertEqual(len(self.provider.requests), before)
|
|
self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners))
|
|
self.receipt('JOINT-001/003', sdk_processes=4, upstream_requests=before,
|
|
attempts_per_generation=[g['attempts'] for g in generations], restart_extra_requests=0)
|
|
|
|
def test_two_tool_chains_pause_resume_into_429_without_replaying(self):
|
|
initial = threading.Barrier(2)
|
|
recovering = threading.Event()
|
|
def respond(_n, body):
|
|
if not any(m['role'] == 'tool' for m in body['messages']):
|
|
initial.wait(timeout=5)
|
|
return write_response()
|
|
if recovering.is_set():
|
|
return ok(_n, body)
|
|
return 429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}'
|
|
self.setup_shared(respond, wait_budget_s=.1)
|
|
runners = [self.runner(), self.runner()]
|
|
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
|
|
mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners]
|
|
before = [self.broker.execution_status(r.scope.execution_id) for r in runners]
|
|
requests_before_resume = len(self.provider.requests)
|
|
for runner in runners:
|
|
self.resume(runner)
|
|
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
|
|
self.assertGreater(len(self.provider.requests), requests_before_resume)
|
|
after = [self.broker.execution_status(r.scope.execution_id) for r in runners]
|
|
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners])
|
|
for runner, old, new in zip(runners, before, after):
|
|
self.assertEqual([g['generation_id'] for g in old['generations']], [g['generation_id'] for g in new['generations']])
|
|
self.assertGreaterEqual(new['wait_spent'], old['wait_spent'])
|
|
self.assertEqual(new['resumes'], 1)
|
|
recovering.set()
|
|
for runner in runners:
|
|
self.resume(runner)
|
|
self.assertEqual(runner.run_sdk().returncode, 0)
|
|
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners])
|
|
for runner in runners:
|
|
bodies = [r['body'] for r in self.provider.requests
|
|
if runner.scope.execution_id in json.dumps(r['body']) and any(m['role'] == 'tool' for m in r['body']['messages'])]
|
|
self.assertTrue(bodies)
|
|
self.assertTrue(all(body == bodies[0] for body in bodies))
|
|
records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()]
|
|
self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1)
|
|
snapshot = self.assert_metered()
|
|
self.assertTrue(all(g['state'] == 'COMPLETE' and g['attempts'] <= 3 for g in snapshot['generations']))
|
|
self.receipt('JOINT-002/003', sdk_processes=6, upstream_requests=len(self.provider.requests),
|
|
tool_executions=2, tool_replays=0, generation_ids=[g['generation_id'] for g in snapshot['generations']])
|
|
|
|
def test_broker_unavailable_never_falls_back_to_provider(self):
|
|
self.setup_shared(ok)
|
|
runners = [self.runner(), self.runner()]
|
|
self.broker.close()
|
|
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
|
|
self.assertEqual(self.provider.requests, [])
|
|
self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners))
|
|
self.receipt('JOINT-001-entry-loss', sdk_processes=2, upstream_requests=0, tool_executions=0)
|
|
|
|
def test_real_sdk_recovery_grows_one_per_window_then_shared_429_reduces(self):
|
|
initial, initial_tools = threading.Barrier(2), threading.Barrier(2)
|
|
healthy, repeated = threading.Event(), threading.Event()
|
|
def respond(n, body):
|
|
if not any(m['role'] == 'tool' for m in body['messages']):
|
|
if not healthy.is_set() and not repeated.is_set():
|
|
initial.wait(timeout=5)
|
|
return write_response()
|
|
if not healthy.is_set() and not repeated.is_set():
|
|
# Both peers must actually send the post-tool request before the
|
|
# first 429 fences the group; otherwise a peer may only queue.
|
|
initial_tools.wait(timeout=2)
|
|
return ok(n, body) if healthy.is_set() else (
|
|
429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}')
|
|
window, maximum = 4, 3
|
|
self.setup_shared(respond, max_concurrency=maximum, wait_budget_s=.1,
|
|
recovery_window_s=window, recovery_min_successes=2)
|
|
peers = [self.runner(), self.runner()]
|
|
self.assertEqual([r.returncode for r in self.together(peers)], [76, 76])
|
|
mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers]
|
|
self.assertEqual(self.broker.snapshot()['groups'][0]['target'], 1)
|
|
healthy.set()
|
|
for runner in peers:
|
|
# Exhausted queue budgets require immediate admission. At target=1,
|
|
# resume the next peer only after the previous probe has finished.
|
|
self.resume(runner)
|
|
self.assertEqual(runner.run_sdk().returncode, 0)
|
|
group = self.broker.snapshot()['groups'][0]
|
|
self.assertEqual((group['state'], group['target']), ('RECOVERING', 1))
|
|
self.assertLess(time.time() - group['recovery_at'], window)
|
|
steps = [{'target': group['target'], 'at': group['recovery_at']}]
|
|
for expected in range(2, maximum + 1):
|
|
previous = group
|
|
# Use the actual durable clock and real elapsed windows; never patch time.
|
|
time.sleep(max(0, previous['recovery_at'] + window + .05 - time.time()))
|
|
self.assertEqual(self.runner().run_sdk().returncode, 0)
|
|
group = self.broker.snapshot()['groups'][0]
|
|
self.assertEqual(group['target'], expected)
|
|
self.assertEqual(group['target'] - previous['target'], 1)
|
|
self.assertGreaterEqual(group['recovery_at'] - previous['recovery_at'], window)
|
|
steps.append({'target': group['target'], 'at': group['recovery_at']})
|
|
self.assertEqual(group['state'], 'OPEN')
|
|
self.assertEqual(self.runner().run_sdk().returncode, 0)
|
|
self.assertEqual(self.broker.snapshot()['groups'][0]['target'], maximum)
|
|
repeated.set(); healthy.clear()
|
|
self.assertEqual([r.returncode for r in self.together([self.runner(), self.runner()])], [76, 76])
|
|
snapshot = self.assert_metered()
|
|
self.assertEqual((snapshot['groups'][0]['state'], snapshot['groups'][0]['target']), ('COOLING', 1))
|
|
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers])
|
|
for runner in peers:
|
|
records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()]
|
|
self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1)
|
|
bodies = [request['body'] for request in self.provider.requests
|
|
if runner.scope.execution_id in json.dumps(request['body'])
|
|
and any(m['role'] == 'tool' for m in request['body']['messages'])]
|
|
self.assertGreaterEqual(len(bodies), 2)
|
|
self.assertTrue(all(body == bodies[0] for body in bodies))
|
|
self.assertTrue(all(g['attempts'] <= 3 for g in snapshot['generations']))
|
|
self.receipt('JOINT-002-real-windows', sdk_processes=9, recovery_window_s=window,
|
|
recovery_steps=steps, max_concurrency=maximum, final_shared_target=1,
|
|
upstream_requests=len(self.provider.requests), original_tool_replays=0)
|
|
|
|
def test_cancel_queued_sdk_never_sends_or_runs_tools(self):
|
|
entered, release = threading.Event(), threading.Event()
|
|
def hold(n, body):
|
|
entered.set()
|
|
if not release.wait(5):
|
|
raise TimeoutError('test release missing')
|
|
return ok(n, body)
|
|
self.setup_shared(hold, max_concurrency=1, wait_budget_s=5, first_byte_timeout_s=5, request_total_s=6)
|
|
first, queued = self.runner(), self.runner()
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
active = pool.submit(first.run_sdk)
|
|
try:
|
|
self.assertTrue(entered.wait(5))
|
|
pending = pool.submit(queued.run_sdk)
|
|
self.wait_for(lambda: any(g['state'] == 'QUEUED' for g in self.broker.execution_status(queued.scope.execution_id)['generations']))
|
|
self.broker.cancel(queued.scope.execution_id)
|
|
self.assertEqual(pending.result(timeout=5).returncode, 76)
|
|
state = self.broker.execution_status(queued.scope.execution_id)
|
|
self.assertEqual([(g['state'], g['attempts']) for g in state['generations']], [('CANCELLED', 0)])
|
|
self.assertFalse((queued.root / 'side-effect.txt').exists())
|
|
finally:
|
|
release.set()
|
|
self.assertEqual(active.result(timeout=5).returncode, 0)
|
|
self.assert_metered()
|
|
self.assertEqual(len(self.provider.requests), 1)
|
|
self.receipt('JOINT-005', sdk_processes=2, upstream_requests=1, cancelled_generation_attempts=0, cancelled_tool_executions=0)
|
|
|
|
def test_partial_stream_timeout_never_delivers_a_tool(self):
|
|
def stalled(_n, _body):
|
|
def stream(output):
|
|
output.write(write_response()[2].split(b'\n\n')[0] + b'\n\n')
|
|
output.flush()
|
|
time.sleep(.35)
|
|
return 200, {'Content-Type': 'text/event-stream'}, stream
|
|
self.setup_shared(stalled, stream_idle_timeout_s=.08, request_total_s=.5)
|
|
runner = self.runner()
|
|
self.assertEqual(runner.run_sdk().returncode, 76)
|
|
snapshot = self.assert_metered()
|
|
self.assertEqual(len(self.provider.requests), 1)
|
|
self.assertFalse((runner.root / 'side-effect.txt').exists())
|
|
self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED')
|
|
self.assertEqual(snapshot['attempts'][0]['reason'], 'INCOMPLETE_GENERATION')
|
|
self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'STREAM_IDLE_TIMEOUT')
|
|
self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
|
|
self.receipt('JOINT-004', sdk_processes=1, upstream_requests=1, tool_executions=0,
|
|
reason=snapshot['attempts'][0]['reason'], transport_reason='STREAM_IDLE_TIMEOUT',
|
|
group_state=snapshot['groups'][0]['state'])
|
|
|
|
def test_first_byte_timeout_never_delivers_late_tool_result(self):
|
|
def late(_n, _body):
|
|
time.sleep(.35)
|
|
return write_response()
|
|
self.setup_shared(late, first_byte_timeout_s=.08, request_total_s=.5)
|
|
runner = self.runner()
|
|
self.assertEqual(runner.run_sdk().returncode, 76)
|
|
snapshot = self.assert_metered()
|
|
self.assertEqual(len(self.provider.requests), 1)
|
|
self.assertFalse((runner.root / 'side-effect.txt').exists())
|
|
self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'FIRST_BYTE_TIMEOUT')
|
|
self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED')
|
|
self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
|
|
self.receipt('JOINT-004-first-byte', sdk_processes=1, upstream_requests=1,
|
|
tool_executions=0, transport_reason='FIRST_BYTE_TIMEOUT')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|