"""Real Pi processes share one broker ledger and a controlled HTTP provider.""" from concurrent.futures import ThreadPoolExecutor import copy import json from pathlib import Path import shutil import threading import time import unittest from unittest.mock import patch from harness.quota import Broker, QuotaPolicy, Upstream from scripts import test_pi_session as sdk_fixture from scripts.test_quota import Provider, ok def write_response(): delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'write-once', 'type': 'function', 'function': {'name': 'write', 'arguments': json.dumps({'path': 'side-effect.txt', 'content': 'once'})}}]} chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]}, {'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}], 'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}] body = ''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks).encode() return 200, {'Content-Type': 'text/event-stream'}, body + b'data: [DONE]\n\n' @unittest.skipUnless(sdk_fixture.NODE.is_file(), 'installed Pi 0.73.1 SDK required') class JointSDKQuota(unittest.TestCase): def setup_shared(self, responder, **limits): self.provider = Provider(responder) self.addCleanup(self.provider.close) policy = QuotaPolicy(**dict({'max_concurrency': 2, 'rpm': 1000, 'tpm': 10000000, 'max_attempts': 3, 'window_s': .2, 'wait_budget_s': 1, 'cooldown_initial_s': .05, 'cooldown_local_max_s': .1, 'jitter_fraction': 0, 'connect_timeout_s': .5, 'first_byte_timeout_s': 2, 'stream_idle_timeout_s': .5, 'request_total_s': 3, 'poll_s': .005}, **limits)) self.template = sdk_fixture.PiSDKTests() self.addCleanup(self.template.doCleanups) # Reuse the SDK fixture's settings/models/private-home setup. Its unused # provider is closed; all actual SDK requests use the controlled server. with patch('scripts.test_pi_session.Broker', side_effect=lambda path, *_: Broker(path, {'account': policy}, {'teacher': Upstream( 'account', self.provider.url, 'upstream-fixture', ('fixture',))})): self.template.setUp() self.template.server.shutdown() self.template.server.server_close() self.broker = self.template.broker self.runners = [] def runner(self): runner = sdk_fixture.PiSDKTests() runner.root = self.template.root / ('project-' + str(len(self.runners))) runner.home = runner.root / 'home' runner.agent_dir = runner.home / '.pi/agent' shutil.copytree(self.template.home, runner.home) shutil.copytree(self.template.root / '.pi', runner.root / '.pi') execution_id = 'joint-peer-' + str(len(self.runners)) runner.scope = self.broker.issue_scope(execution_id, 'teacher', 'fixture') runner.config = copy.deepcopy(self.template.config) runner.config.update(execution_id=execution_id, session_dir=str(runner.root / '.pi-sessions'), append_system_path=str(runner.root / 'system.md'), prompt_path=str(runner.root / 'prompt.md')) (runner.root / 'system.md').write_text('Use write once, then report completion.') (runner.root / 'prompt.md').write_text(execution_id + ': write once.') self.runners.append(runner) return runner def together(self, runners): with ThreadPoolExecutor(max_workers=len(runners)) as pool: return list(pool.map(lambda r: r.run_sdk(), runners)) def wait_for(self, predicate, timeout=5): deadline = time.monotonic() + timeout while time.monotonic() < deadline: value = predicate() if value: return value time.sleep(.01) self.fail('bounded condition did not arrive') def resume(self, runner): checkpoint = json.loads((runner.root / '.bench-session.json').read_bytes()) self.assertFalse(checkpoint['complete']) self.assertTrue(checkpoint['resumable']) self.assertEqual(checkpoint['phase'], 'PAUSED_INFRA') header = json.loads((runner.root / 'events.jsonl').read_text().splitlines()[0]) self.wait_for(lambda: self.broker.resume_readiness(runner.scope.execution_id)['eligible']) self.broker.resume(runner.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True) runner.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta']) def assert_metered(self): snapshot = self.broker.snapshot() sent = [a for a in snapshot['attempts'] if a['sent'] == 1] self.assertEqual(len(sent), len(self.provider.requests)) self.assertTrue(all(r['authorization'] == 'Bearer upstream-fixture' for r in self.provider.requests)) self.assertTrue(all(a['group_id'] == 'account' for a in sent)) probes = sorted((a for a in sent if a['state_at_send'] == 'HALF_OPEN'), key=lambda a: a['started_at']) for before, after in zip(probes, probes[1:]): self.assertGreaterEqual(after['started_at'], before['ended_at']) self.assertNotEqual(after['epoch'], before['epoch']) for attempt in sent: cooling = [e for e in snapshot['quota_events'] if e['event'] == 'COOLING' and e['at'] <= attempt['started_at']] if cooling: self.assertGreaterEqual(attempt['started_at'], json.loads(cooling[-1]['detail'])['cooldown_until']) return snapshot def receipt(self, case, **facts): print('JOINT_RECEIPT ' + json.dumps({'case': case, **facts}, sort_keys=True)) def test_persistent_429_multiple_sdk_processes_share_cooling_and_budget(self): self.setup_shared(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}')) runners = [self.runner(), self.runner()] results = self.together(runners) self.assertEqual([r.returncode for r in results], [76, 76]) snapshot = self.assert_metered() generations = snapshot['generations'] self.assertEqual(len(generations), 2) self.assertTrue(all(g['attempts'] == 3 and g['state'] == 'PAUSED' for g in generations)) self.assertTrue(any(a['state_at_send'] == 'HALF_OPEN' for a in snapshot['attempts'])) before = len(self.provider.requests) self.assertEqual([r.returncode for r in self.together(runners)], [76, 76]) self.assertEqual(len(self.provider.requests), before) self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners)) self.receipt('JOINT-001/003', sdk_processes=4, upstream_requests=before, attempts_per_generation=[g['attempts'] for g in generations], restart_extra_requests=0) def test_two_tool_chains_pause_resume_into_429_without_replaying(self): initial = threading.Barrier(2) recovering = threading.Event() def respond(_n, body): if not any(m['role'] == 'tool' for m in body['messages']): initial.wait(timeout=5) return write_response() if recovering.is_set(): return ok(_n, body) return 429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}' self.setup_shared(respond, wait_budget_s=.1) runners = [self.runner(), self.runner()] self.assertEqual([r.returncode for r in self.together(runners)], [76, 76]) mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners] before = [self.broker.execution_status(r.scope.execution_id) for r in runners] requests_before_resume = len(self.provider.requests) for runner in runners: self.resume(runner) self.assertEqual([r.returncode for r in self.together(runners)], [76, 76]) self.assertGreater(len(self.provider.requests), requests_before_resume) after = [self.broker.execution_status(r.scope.execution_id) for r in runners] self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners]) for runner, old, new in zip(runners, before, after): self.assertEqual([g['generation_id'] for g in old['generations']], [g['generation_id'] for g in new['generations']]) self.assertGreaterEqual(new['wait_spent'], old['wait_spent']) self.assertEqual(new['resumes'], 1) recovering.set() for runner in runners: self.resume(runner) self.assertEqual(runner.run_sdk().returncode, 0) self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in runners]) for runner in runners: bodies = [r['body'] for r in self.provider.requests if runner.scope.execution_id in json.dumps(r['body']) and any(m['role'] == 'tool' for m in r['body']['messages'])] self.assertTrue(bodies) self.assertTrue(all(body == bodies[0] for body in bodies)) records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()] self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1) snapshot = self.assert_metered() self.assertTrue(all(g['state'] == 'COMPLETE' and g['attempts'] <= 3 for g in snapshot['generations'])) self.receipt('JOINT-002/003', sdk_processes=6, upstream_requests=len(self.provider.requests), tool_executions=2, tool_replays=0, generation_ids=[g['generation_id'] for g in snapshot['generations']]) def test_broker_unavailable_never_falls_back_to_provider(self): self.setup_shared(ok) runners = [self.runner(), self.runner()] self.broker.close() self.assertEqual([r.returncode for r in self.together(runners)], [76, 76]) self.assertEqual(self.provider.requests, []) self.assertTrue(all(not (r.root / 'side-effect.txt').exists() for r in runners)) self.receipt('JOINT-001-entry-loss', sdk_processes=2, upstream_requests=0, tool_executions=0) def test_real_sdk_recovery_grows_one_per_window_then_shared_429_reduces(self): initial, initial_tools = threading.Barrier(2), threading.Barrier(2) healthy, repeated = threading.Event(), threading.Event() def respond(n, body): if not any(m['role'] == 'tool' for m in body['messages']): if not healthy.is_set() and not repeated.is_set(): initial.wait(timeout=5) return write_response() if not healthy.is_set() and not repeated.is_set(): # Both peers must actually send the post-tool request before the # first 429 fences the group; otherwise a peer may only queue. initial_tools.wait(timeout=2) return ok(n, body) if healthy.is_set() else ( 429, {'Retry-After': '.35'}, b'{"error":{"type":"rate_limit"}}') window, maximum = 4, 3 self.setup_shared(respond, max_concurrency=maximum, wait_budget_s=.1, recovery_window_s=window, recovery_min_successes=2) peers = [self.runner(), self.runner()] self.assertEqual([r.returncode for r in self.together(peers)], [76, 76]) mtimes = [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers] self.assertEqual(self.broker.snapshot()['groups'][0]['target'], 1) healthy.set() for runner in peers: # Exhausted queue budgets require immediate admission. At target=1, # resume the next peer only after the previous probe has finished. self.resume(runner) self.assertEqual(runner.run_sdk().returncode, 0) group = self.broker.snapshot()['groups'][0] self.assertEqual((group['state'], group['target']), ('RECOVERING', 1)) self.assertLess(time.time() - group['recovery_at'], window) steps = [{'target': group['target'], 'at': group['recovery_at']}] for expected in range(2, maximum + 1): previous = group # Use the actual durable clock and real elapsed windows; never patch time. time.sleep(max(0, previous['recovery_at'] + window + .05 - time.time())) self.assertEqual(self.runner().run_sdk().returncode, 0) group = self.broker.snapshot()['groups'][0] self.assertEqual(group['target'], expected) self.assertEqual(group['target'] - previous['target'], 1) self.assertGreaterEqual(group['recovery_at'] - previous['recovery_at'], window) steps.append({'target': group['target'], 'at': group['recovery_at']}) self.assertEqual(group['state'], 'OPEN') self.assertEqual(self.runner().run_sdk().returncode, 0) self.assertEqual(self.broker.snapshot()['groups'][0]['target'], maximum) repeated.set(); healthy.clear() self.assertEqual([r.returncode for r in self.together([self.runner(), self.runner()])], [76, 76]) snapshot = self.assert_metered() self.assertEqual((snapshot['groups'][0]['state'], snapshot['groups'][0]['target']), ('COOLING', 1)) self.assertEqual(mtimes, [(r.root / 'side-effect.txt').stat().st_mtime_ns for r in peers]) for runner in peers: records = [json.loads(line) for line in (runner.root / 'events.jsonl').read_text().splitlines()] self.assertEqual(sum(e.get('type') == 'tool_execution_end' for e in records), 1) bodies = [request['body'] for request in self.provider.requests if runner.scope.execution_id in json.dumps(request['body']) and any(m['role'] == 'tool' for m in request['body']['messages'])] self.assertGreaterEqual(len(bodies), 2) self.assertTrue(all(body == bodies[0] for body in bodies)) self.assertTrue(all(g['attempts'] <= 3 for g in snapshot['generations'])) self.receipt('JOINT-002-real-windows', sdk_processes=9, recovery_window_s=window, recovery_steps=steps, max_concurrency=maximum, final_shared_target=1, upstream_requests=len(self.provider.requests), original_tool_replays=0) def test_cancel_queued_sdk_never_sends_or_runs_tools(self): entered, release = threading.Event(), threading.Event() def hold(n, body): entered.set() if not release.wait(5): raise TimeoutError('test release missing') return ok(n, body) self.setup_shared(hold, max_concurrency=1, wait_budget_s=5, first_byte_timeout_s=5, request_total_s=6) first, queued = self.runner(), self.runner() with ThreadPoolExecutor(max_workers=2) as pool: active = pool.submit(first.run_sdk) try: self.assertTrue(entered.wait(5)) pending = pool.submit(queued.run_sdk) self.wait_for(lambda: any(g['state'] == 'QUEUED' for g in self.broker.execution_status(queued.scope.execution_id)['generations'])) self.broker.cancel(queued.scope.execution_id) self.assertEqual(pending.result(timeout=5).returncode, 76) state = self.broker.execution_status(queued.scope.execution_id) self.assertEqual([(g['state'], g['attempts']) for g in state['generations']], [('CANCELLED', 0)]) self.assertFalse((queued.root / 'side-effect.txt').exists()) finally: release.set() self.assertEqual(active.result(timeout=5).returncode, 0) self.assert_metered() self.assertEqual(len(self.provider.requests), 1) self.receipt('JOINT-005', sdk_processes=2, upstream_requests=1, cancelled_generation_attempts=0, cancelled_tool_executions=0) def test_partial_stream_timeout_never_delivers_a_tool(self): def stalled(_n, _body): def stream(output): output.write(write_response()[2].split(b'\n\n')[0] + b'\n\n') output.flush() time.sleep(.35) return 200, {'Content-Type': 'text/event-stream'}, stream self.setup_shared(stalled, stream_idle_timeout_s=.08, request_total_s=.5) runner = self.runner() self.assertEqual(runner.run_sdk().returncode, 76) snapshot = self.assert_metered() self.assertEqual(len(self.provider.requests), 1) self.assertFalse((runner.root / 'side-effect.txt').exists()) self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED') self.assertEqual(snapshot['attempts'][0]['reason'], 'INCOMPLETE_GENERATION') self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'STREAM_IDLE_TIMEOUT') self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible']) self.receipt('JOINT-004', sdk_processes=1, upstream_requests=1, tool_executions=0, reason=snapshot['attempts'][0]['reason'], transport_reason='STREAM_IDLE_TIMEOUT', group_state=snapshot['groups'][0]['state']) def test_first_byte_timeout_never_delivers_late_tool_result(self): def late(_n, _body): time.sleep(.35) return write_response() self.setup_shared(late, first_byte_timeout_s=.08, request_total_s=.5) runner = self.runner() self.assertEqual(runner.run_sdk().returncode, 76) snapshot = self.assert_metered() self.assertEqual(len(self.provider.requests), 1) self.assertFalse((runner.root / 'side-effect.txt').exists()) self.assertEqual(snapshot['attempts'][0]['transport_reason'], 'FIRST_BYTE_TIMEOUT') self.assertEqual(snapshot['generations'][0]['state'], 'PAUSED') self.assertFalse(self.broker.resume_readiness(runner.scope.execution_id)['eligible']) self.receipt('JOINT-004-first-byte', sdk_processes=1, upstream_requests=1, tool_executions=0, transport_reason='FIRST_BYTE_TIMEOUT') if __name__ == '__main__': unittest.main()