data-gen-bpu/scripts/test_collection_joint.py

312 lines
18 KiB
Python

"""Real Pi processes share one broker ledger and a controlled HTTP provider."""
from concurrent.futures import ThreadPoolExecutor
import copy
import json
from pathlib import Path
import shutil
import threading
import time
import unittest
from unittest.mock import patch
from harness.quota import Broker, QuotaPolicy, Upstream
from scripts import test_pi_session as sdk_fixture
from scripts.test_quota import Provider, ok
def write_response():
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'write-once',
'type': 'function', 'function': {'name': 'write',
'arguments': json.dumps({'path': 'side-effect.txt', 'content': 'once'})}}]}
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
body = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode()
return 200, {'Content-Type': 'text/event-stream'}, body + b'data: [DONE]\n\n'
@unittest.skipUnless(sdk_fixture.NODE.is_file(), 'installed Pi 0.73.1 SDK required')
class JointSDKQuota(unittest.TestCase):
def setup_shared(self, responder, **limits):
self.provider = Provider(responder)
self.addCleanup(self.provider.close)
policy = QuotaPolicy(**dict({'max_concurrency': 2, 'rpm': 1000, 'tpm': 10000000,
'max_attempts': 3, 'window_s': .2, 'wait_budget_s': 1,
'cooldown_initial_s': .05, 'cooldown_local_max_s': .1, 'jitter_fraction': 0,
'connect_timeout_s': .5, 'first_byte_timeout_s': 2,
'stream_idle_timeout_s': .5, 'request_total_s': 3, 'poll_s': .005}, **limits))
self.template = sdk_fixture.PiSDKTests()
self.addCleanup(self.template.doCleanups)
# Reuse the SDK fixture's settings/models/private-home setup. Its unused
# provider is closed; all actual SDK requests use the controlled server.
with patch('scripts.test_pi_session.Broker', side_effect=lambda path, *_:
Broker(path, {'account': policy}, {'teacher': Upstream(
'account', self.provider.url, 'upstream-fixture', ('fixture',))})):
self.template.setUp()
self.template.server.shutdown()
self.template.server.server_close()
self.broker = self.template.broker
self.runners = []
def runner(self):
runner = sdk_fixture.PiSDKTests()
runner.root = self.template.root / ('project-' + str(len(self.runners)))
runner.home = runner.root / 'home'
runner.agent_dir = runner.home / '.pi/agent'
shutil.copytree(self.template.home, runner.home)
shutil.copytree(self.template.root / '.pi', runner.root / '.pi')
execution_id = 'joint-peer-' + str(len(self.runners))
runner.scope = self.broker.issue_scope(execution_id, 'teacher', 'fixture')
runner.config = copy.deepcopy(self.template.config)
runner.config.update(execution_id=execution_id, session_dir=str(runner.root / '.pi-sessions'),
append_system_path=str(runner.root / 'system.md'), prompt_path=str(runner.root / 'prompt.md'))
(runner.root / 'system.md').write_text('Use write once, then report completion.')
(runner.root / 'prompt.md').write_text(execution_id + ': write once.')
self.runners.append(runner)
return runner
def together(self, runners):
with ThreadPoolExecutor(max_workers=len(runners)) as pool:
return list(pool.map(lambda r: r.run_sdk(), runners))
def wait_for(self, predicate, timeout=5):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(.01)
self.fail('bounded condition did not arrive')
def resume(self, runner):
checkpoint = json.loads((runner.root / '.bench-session.json').read_bytes())
self.assertFalse(checkpoint['complete'])
self.assertTrue(checkpoint['resumable'])
self.assertEqual(checkpoint['phase'], 'PAUSED_INFRA')
header = json.loads((runner.root / 'events.jsonl').read_text().splitlines()[0])
self.wait_for(lambda: self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
self.broker.resume(runner.scope.execution_id, checkpoint['generation_id'],
checkpoint_complete=True, side_effects_known=True)
runner.config.update(resume=True, session_path=checkpoint['session_path'],
session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'],
system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'],
nonempty_delta=checkpoint['nonempty_delta'])
def assert_metered(self):
snapshot = self.broker.snapshot()
sent = [a for a in snapshot['attempts'] if a['sent'] == 1]
self.assertEqual(len(sent), len(self.provider.requests))
self.assertTrue(all(r['authorization'] == 'Bearer upstream-fixture' for r in self.provider.requests))
self.assertTrue(all(a['group_id'] == 'account' for a in sent))
probes = sorted((a for a in sent if a['state_at_send'] == 'HALF_OPEN'), key=lambda a: a['started_at'])
for before, after in zip(probes, probes[1:]):
self.assertGreaterEqual(after['started_at'], before['ended_at'])
self.assertNotEqual(after['epoch'], before['epoch'])
for attempt in sent:
cooling = [e for e in snapshot['quota_events'] if e['event'] == 'COOLING' and e['at'] <= attempt['started_at']]
if cooling:
self.assertGreaterEqual(attempt['started_at'], json.loads(cooling[-1]['detail'])['cooldown_until'])
return snapshot
def receipt(self, case, **facts):
print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True))
def test_persistent_429_multiple_sdk_processes_share_cooling_and_budget(self):
self.setup_shared(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
runners = [self.runner(), self.runner()]
results = self.together(runners)
self.assertEqual([r.returncode for r in results], [76, 76])
snapshot = self.assert_metered()
generations = snapshot['generations']
self.assertEqual(len(generations), 2)
self.assertTrue(all(g['attempts'] == 3 and g['state'] == 'PAUSED' for g in generations))
self.assertTrue(any(a['state_at_send'] == 'HALF_OPEN' for a in snapshot['attempts']))
before = len(self.provider.requests)
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
self.assertEqual(len(self.provider.requests), before)
self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners))
self.receipt('JOINT-001/003', sdk_processes=4, upstream_requests=before,
attempts_per_generation=[g['attempts'] for g in generations], restart_extra_requests=0)
def test_two_tool_chains_pause_resume_into_429_without_replaying(self):
initial = threading.Barrier(2)
recovering = threading.Event()
def respond(_n, body):
if not any(m['role'] == 'tool' for m in body['messages']):
initial.wait(timeout=5)
return write_response()
if recovering.is_set():
return ok(_n, body)
return 429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}'
self.setup_shared(respond, wait_budget_s=.1)
runners = [self.runner(), self.runner()]
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners]
before = [self.broker.execution_status(r.scope.execution_id) for r in runners]
requests_before_resume = len(self.provider.requests)
for runner in runners:
self.resume(runner)
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
self.assertGreater(len(self.provider.requests), requests_before_resume)
after = [self.broker.execution_status(r.scope.execution_id) for r in runners]
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners])
for runner, old, new in zip(runners, before, after):
self.assertEqual([g['generation_id'] for g in old['generations']], [g['generation_id'] for g in new['generations']])
self.assertGreaterEqual(new['wait_spent'], old['wait_spent'])
self.assertEqual(new['resumes'], 1)
recovering.set()
for runner in runners:
self.resume(runner)
self.assertEqual(runner.run_sdk().returncode, 0)
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners])
for runner in runners:
bodies = [r['body'] for r in self.provider.requests
if runner.scope.execution_id in json.dumps(r['body']) and any(m['role'] == 'tool' for m in r['body']['messages'])]
self.assertTrue(bodies)
self.assertTrue(all(body == bodies[0] for body in bodies))
records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()]
self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1)
snapshot = self.assert_metered()
self.assertTrue(all(g['state'] == 'COMPLETE' and g['attempts'] <= 3 for g in snapshot['generations']))
self.receipt('JOINT-002/003', sdk_processes=6, upstream_requests=len(self.provider.requests),
tool_executions=2, tool_replays=0, generation_ids=[g['generation_id'] for g in snapshot['generations']])
def test_broker_unavailable_never_falls_back_to_provider(self):
self.setup_shared(ok)
runners = [self.runner(), self.runner()]
self.broker.close()
self.assertEqual([r.returncode for r in self.together(runners)], [76, 76])
self.assertEqual(self.provider.requests, [])
self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners))
self.receipt('JOINT-001-entry-loss', sdk_processes=2, upstream_requests=0, tool_executions=0)
def test_real_sdk_recovery_grows_one_per_window_then_shared_429_reduces(self):
initial, initial_tools = threading.Barrier(2), threading.Barrier(2)
healthy, repeated = threading.Event(), threading.Event()
def respond(n, body):
if not any(m['role'] == 'tool' for m in body['messages']):
if not healthy.is_set() and not repeated.is_set():
initial.wait(timeout=5)
return write_response()
if not healthy.is_set() and not repeated.is_set():
# Both peers must actually send the post-tool request before the
# first 429 fences the group; otherwise a peer may only queue.
initial_tools.wait(timeout=2)
return ok(n, body) if healthy.is_set() else (
429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}')
window, maximum = 4, 3
self.setup_shared(respond, max_concurrency=maximum, wait_budget_s=.1,
recovery_window_s=window, recovery_min_successes=2)
peers = [self.runner(), self.runner()]
self.assertEqual([r.returncode for r in self.together(peers)], [76, 76])
mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers]
self.assertEqual(self.broker.snapshot()['groups'][0]['target'], 1)
healthy.set()
for runner in peers:
# Exhausted queue budgets require immediate admission. At target=1,
# resume the next peer only after the previous probe has finished.
self.resume(runner)
self.assertEqual(runner.run_sdk().returncode, 0)
group = self.broker.snapshot()['groups'][0]
self.assertEqual((group['state'], group['target']), ('RECOVERING', 1))
self.assertLess(time.time() - group['recovery_at'], window)
steps = [{'target': group['target'], 'at': group['recovery_at']}]
for expected in range(2, maximum + 1):
previous = group
# Use the actual durable clock and real elapsed windows; never patch time.
time.sleep(max(0, previous['recovery_at'] + window + .05 - time.time()))
self.assertEqual(self.runner().run_sdk().returncode, 0)
group = self.broker.snapshot()['groups'][0]
self.assertEqual(group['target'], expected)
self.assertEqual(group['target'] - previous['target'], 1)
self.assertGreaterEqual(group['recovery_at'] - previous['recovery_at'], window)
steps.append({'target': group['target'], 'at': group['recovery_at']})
self.assertEqual(group['state'], 'OPEN')
self.assertEqual(self.runner().run_sdk().returncode, 0)
self.assertEqual(self.broker.snapshot()['groups'][0]['target'], maximum)
repeated.set(); healthy.clear()
self.assertEqual([r.returncode for r in self.together([self.runner(), self.runner()])], [76, 76])
snapshot = self.assert_metered()
self.assertEqual((snapshot['groups'][0]['state'], snapshot['groups'][0]['target']), ('COOLING', 1))
self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers])
for runner in peers:
records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()]
self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1)
bodies = [request['body'] for request in self.provider.requests
if runner.scope.execution_id in json.dumps(request['body'])
and any(m['role'] == 'tool' for m in request['body']['messages'])]
self.assertGreaterEqual(len(bodies), 2)
self.assertTrue(all(body == bodies[0] for body in bodies))
self.assertTrue(all(g['attempts'] <= 3 for g in snapshot['generations']))
self.receipt('JOINT-002-real-windows', sdk_processes=9, recovery_window_s=window,
recovery_steps=steps, max_concurrency=maximum, final_shared_target=1,
upstream_requests=len(self.provider.requests), original_tool_replays=0)
def test_cancel_queued_sdk_never_sends_or_runs_tools(self):
entered, release = threading.Event(), threading.Event()
def hold(n, body):
entered.set()
if not release.wait(5):
raise TimeoutError('test release missing')
return ok(n, body)
self.setup_shared(hold, max_concurrency=1, wait_budget_s=5, first_byte_timeout_s=5, request_total_s=6)
first, queued = self.runner(), self.runner()
with ThreadPoolExecutor(max_workers=2) as pool:
active = pool.submit(first.run_sdk)
try:
self.assertTrue(entered.wait(5))
pending = pool.submit(queued.run_sdk)
self.wait_for(lambda: any(g['state'] == 'QUEUED' for g in self.broker.execution_status(queued.scope.execution_id)['generations']))
self.broker.cancel(queued.scope.execution_id)
self.assertEqual(pending.result(timeout=5).returncode, 76)
state = self.broker.execution_status(queued.scope.execution_id)
self.assertEqual([(g['state'], g['attempts']) for g in state['generations']], [('CANCELLED', 0)])
self.assertFalse((queued.root / 'side-effect.txt').exists())
finally:
release.set()
self.assertEqual(active.result(timeout=5).returncode, 0)
self.assert_metered()
self.assertEqual(len(self.provider.requests), 1)
self.receipt('JOINT-005', sdk_processes=2, upstream_requests=1, cancelled_generation_attempts=0, cancelled_tool_executions=0)
def test_partial_stream_timeout_never_delivers_a_tool(self):
def stalled(_n, _body):
def stream(output):
output.write(write_response()[2].split(b'\n\n')[0] + b'\n\n')
output.flush()
time.sleep(.35)
return 200, {'Content-Type': 'text/event-stream'}, stream
self.setup_shared(stalled, stream_idle_timeout_s=.08, request_total_s=.5)
runner = self.runner()
self.assertEqual(runner.run_sdk().returncode, 76)
snapshot = self.assert_metered()
self.assertEqual(len(self.provider.requests), 1)
self.assertFalse((runner.root / 'side-effect.txt').exists())
self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED')
self.assertEqual(snapshot['attempts'][0]['reason'], 'INCOMPLETE_GENERATION')
self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'STREAM_IDLE_TIMEOUT')
self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
self.receipt('JOINT-004', sdk_processes=1, upstream_requests=1, tool_executions=0,
reason=snapshot['attempts'][0]['reason'], transport_reason='STREAM_IDLE_TIMEOUT',
group_state=snapshot['groups'][0]['state'])
def test_first_byte_timeout_never_delivers_late_tool_result(self):
def late(_n, _body):
time.sleep(.35)
return write_response()
self.setup_shared(late, first_byte_timeout_s=.08, request_total_s=.5)
runner = self.runner()
self.assertEqual(runner.run_sdk().returncode, 76)
snapshot = self.assert_metered()
self.assertEqual(len(self.provider.requests), 1)
self.assertFalse((runner.root / 'side-effect.txt').exists())
self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'FIRST_BYTE_TIMEOUT')
self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED')
self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible'])
self.receipt('JOINT-004-first-byte', sdk_processes=1, upstream_requests=1,
tool_executions=0, transport_reason='FIRST_BYTE_TIMEOUT')
if __name__ == '__main__':
unittest.main()