"""Real installed Pi SDK against a local fake provider; no account or board calls.""" from contextlib import contextmanager import dataclasses import hashlib from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import subprocess import tempfile import threading import time import unittest from harness.quota import Broker, QuotaPolicy, Upstream ROOT = Path(__file__).resolve().parents[1] NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node')) @unittest.skipUnless(NODE.is_file(), 'installed Pi 0.73.1 SDK required') class PiSDKTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.home = self.root / 'home' self.agent_dir = self.home / '.pi/agent' self.agent_dir.mkdir(parents=True) (self.root / '.pi').mkdir() self.bodies = [] self.tool_name, self.tool_id = 'write', 'write-once' self.tool_args = {'path': 'side-effect.txt', 'content': 'once'} self.fail_second = False self.delay_first_s = 0 self.always_tools = False self.tool_turns = 1 self.on_body = lambda n, raw: None self.fail_on = {2} self.overflow_on = set() outer = self class Handler(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_POST(self): raw = self.rfile.read(int(self.headers['Content-Length'])); body = json.loads(raw) outer.bodies.append(body); outer.on_body(len(outer.bodies), raw) if len(outer.bodies) == 1 and outer.delay_first_s: time.sleep(outer.delay_first_s) if len(outer.bodies) in outer.overflow_on: payload = b'{"object":"error","message":"This model\'s maximum context length is 262144 tokens."}' self.send_response(400); self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload) return if outer.fail_second and len(outer.bodies) in outer.fail_on: payload = b'{"error":{"message":"fixture rate limit","type":"rate_limit"}}' self.send_response(429); self.send_header('Retry-After', '0.2') self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload) return delta = ({'role': 'assistant', 'tool_calls': [{'index': 0, 'id': outer.tool_id if len(outer.bodies) == 1 else outer.tool_id + '-' + str(len(outer.bodies)), 'type': 'function', 'function': {'name': outer.tool_name, 'arguments': json.dumps(outer.tool_args)}}]} if len(outer.bodies) <= outer.tool_turns or outer.always_tools else {'role': 'assistant', 'content': 'done'}) finish = 'tool_calls' if len(outer.bodies) <= outer.tool_turns or outer.always_tools else 'stop' chunks = [dict(id='fixture', object='chat.completion.chunk', model='fixture', choices=[dict(index=0, delta=delta, finish_reason=None)]), dict(id='fixture', object='chat.completion.chunk', model='fixture', choices=[dict(index=0, delta={}, finish_reason=finish)], usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})] payload = ''.join('data: ' + json.dumps(chunk) + '\n\n' for chunk in chunks).encode() + b'data: [DONE]\n\n' self.send_response(200); self.send_header('Content-Type', 'text/event-stream') self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload) self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start() self.addCleanup(self.server.server_close); self.addCleanup(self.server.shutdown) self.broker = Broker(self.root / 'quota.sqlite', {'account': QuotaPolicy(1, 100, 10000000, wait_budget_s=.05, cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0, connect_timeout_s=1, first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.005)}, {'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port, 'upstream-fixture', ('fixture',))}) self.broker.start(); self.addCleanup(self.broker.close) self.scope = self.broker.issue_scope('execution-fixture', 'teacher', 'fixture') settings = {'defaultProvider': 'fixture-provider', 'defaultModel': 'fixture', 'compaction': {'enabled': False}, 'retry': {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': self.scope.client_timeout_s * 1000}}, 'extensions': [], 'skills': []} (self.root / '.pi/settings.json').write_text(json.dumps(settings)) models = {'providers': {'fixture-provider': {'baseUrl': self.scope.base_url, 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY', 'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False, 'input': ['text'], 'contextWindow': 128000, 'maxTokens': 512, 'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}} (self.agent_dir / 'models.json').write_text(json.dumps(models)) (self.root / 'system.md').write_text('Use write once, then report completion.') (self.root / 'prompt.md').write_text('Write once.') self.config = {'execution_id': self.scope.execution_id, 'model': 'fixture', 'provider': 'fixture-provider', 'base_url': self.scope.base_url, 'client_timeout_ms': self.scope.client_timeout_s * 1000, 'session_dir': str(self.root / '.pi-sessions'), 'tools': ['write'], 'append_system_path': str(self.root / 'system.md'), 'prompt_path': str(self.root / 'prompt.md'), 'resume': False} def run_sdk(self, timeout=25): config = self.root / 'runner.json'; config.write_text(json.dumps(self.config)) with (self.root / 'events.jsonl').open('ab') as output: result = subprocess.run([str(NODE), str(ROOT / 'agent/pi_session.js'), str(config)], cwd=self.root, env={**os.environ, 'HOME': str(self.home), 'PI_CODING_AGENT_DIR': str(self.agent_dir), 'BENCH_RUNNER_API_KEY': self.scope.api_key}, stdout=output, stderr=subprocess.PIPE, timeout=timeout) return result def test_broker_connection_failure_has_sanitized_transport_identity(self): self.broker.close() result = self.run_sdk() records = [json.loads(line) for line in result.stderr.decode().splitlines() if line.startswith('{')] errors = [r for r in records if r.get('type') == 'bench_transport_error'] self.assertTrue(errors, result.stderr) for error in errors: self.assertEqual(error['execution_id'], self.scope.execution_id) self.assertEqual(error['generation_id'], self.scope.execution_id + ':0') self.assertIn('ECONNREFUSED', (error['code'], error['cause_code'])) self.assertGreaterEqual(error['elapsed_ms'], 0) self.assertEqual(set(error), {'type', 'execution_id', 'generation_id', 'name', 'code', 'cause_code', 'elapsed_ms'}) self.assertNotIn(self.scope.api_key.encode(), result.stderr) self.assertEqual(self.bodies, []) def test_actual_prepare_binds_source_before_sdk_or_http(self): from contextlib import ExitStack from unittest.mock import patch import yaml from harness import collection_runtime as runtime, db, paths, rollout from harness.execution import Execution project = self.root / 'prepared' job_dir = project / 'runs/r/j' job_dir.mkdir(parents=True) job = {'model': 'resnet18', 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq', 'qcfg': 'fp16int8', 'model_spec': {'input_spec': '1x3x224x224'}, 'gates': {'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}}, 'data': {'calib': {'path': str(project / 'calib'), 'n': 32}, 't1': {'path': str(project / 'test.tar'), 'n': 100}}} (job_dir / 'job.json').write_text(json.dumps(job)) cfg = yaml.safe_load((ROOT / 'config.yaml').read_text()) cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE)) cfg['rollouts'].update(private_home=True, tools=['write']) cfg['endpoints']['teacher']['model'] = 'fixture' cfg['collection_v1']['credential_groups'] = {'teacher': 'account'} original_connect = db.connect database = project / 'state/jobs.sqlite' with ExitStack() as stack: for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent', 'RUNS': project / 'runs'}.items(): stack.enter_context(patch.object(paths, name, value)) stack.enter_context(patch.object(paths, 'config', return_value=cfg)) stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(database))) stack.enter_context(patch.object(rollout, 'onnx_path', return_value=None)) stack.enter_context(patch.object(rollout, 'baseline_dir', return_value=project / 'baseline')) stack.enter_context(patch.object(runtime, 'config', return_value=(cfg['collection_v1'], self.broker.groups))) stack.enter_context(patch.object(runtime, 'broker', return_value=self.broker)) with db.connect() as con: con.execute("INSERT INTO runs VALUES ('r',0,'{}','source-binding fixture')") rd, _, _ = rollout.prepare('r', 'j', 0, 'e2e', 'teacher') self.assertEqual(json.loads((rd / 'rollout.json').read_bytes())['runner_version'], 'pi-sdk-v1') settings = json.loads((rd / 'workspace/.pi/settings.json').read_text()) self.assertIn('terminal_exec', settings['tools']) self.assertTrue(any(value.endswith('/terminal.ts') for value in settings['extensions'])) definitions = json.loads((rd / 'agent/tools.json').read_text()) terminal = next(row['function'] for row in definitions if row['function']['name'] == 'terminal_exec') self.assertEqual(set(terminal['parameters']['properties']), {'wait_handle'}) self.assertNotIn('natural_agent_version', json.loads((rd / 'rollout.json').read_text())) execution = Execution(rd) execution.start(cfg['collection_v1']['lifecycle']) runtime.configure_pi(rd, execution, 'teacher') bound = json.loads((rd / 'rollout.json').read_bytes()) self.assertEqual(bound['execution_id'], execution.snapshot()['execution_id']) self.assertFalse((rd / 'agent/events.jsonl').exists()) self.assertEqual(self.bodies, []) for fields, reason in [({'execution_id': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'), ({'runner_version': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'), ({'execution_id': None}, 'RESUME_SOURCE_IDENTITY_MISSING')]: raw = json.dumps({**bound, **fields}).encode() (rd / 'rollout.json').write_bytes(raw) with self.assertRaisesRegex(ValueError, reason): runtime.configure_pi(rd, execution, 'teacher', resume=True) self.assertEqual((rd / 'rollout.json').read_bytes(), raw) (rd / 'rollout.json').write_text(json.dumps(bound)) def test_actual_sdk_uses_broker_and_persists_complete_tool_chain(self): result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) checkpoint = json.loads((self.root / '.bench-session.json').read_bytes()) self.assertEqual(checkpoint['phase'], 'COMPLETED') self.assertTrue(checkpoint['closure']) self.assertEqual(len(self.bodies), 2) self.assertEqual([row['generation_id'] for row in self.broker.execution_status(self.scope.execution_id)['generations']], ['execution-fixture:0', 'execution-fixture:1']) self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once') updates = [e for e in self._events() if e['type'] == 'message_update'] self.assertTrue(updates) # streamed, but without the per-token copies of the whole message self.assertFalse(any('message' in e or 'partial' in e['assistantMessageEvent'] for e in updates)) from harness.collection_runtime import budget_used (self.root / 'agent').mkdir(exist_ok=True) (self.root / 'agent/events.jsonl').write_bytes((self.root / 'events.jsonl').read_bytes()) self.assertEqual(budget_used(self.root)['turns'], sum(e['type'] == 'turn_end' for e in self._events())) @unittest.skipUnless(os.environ.get('BENCH_TEST_LONG_QUEUE') == '1', 'explicit 310 second regression') def test_actual_sdk_waits_past_node_fetch_five_minute_header_limit(self): self.broker.close() policy = QuotaPolicy(1, 100, 10000000, first_byte_timeout_s=340, stream_idle_timeout_s=340, request_total_s=350, wait_budget_s=340) self.broker = Broker(self.root / 'long-quota.sqlite', {'account': policy}, {'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port, 'upstream-fixture', ('fixture',))}).start() self.addCleanup(self.broker.close) self.scope = self.broker.issue_scope('long-execution', 'teacher', 'fixture') self.config.update(execution_id=self.scope.execution_id, base_url=self.scope.base_url, client_timeout_ms=self.scope.client_timeout_s * 1000) settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['retry']['provider']['timeoutMs'] = self.scope.client_timeout_s * 1000 settings_path.write_text(json.dumps(settings)) models_path = self.agent_dir / 'models.json'; models = json.loads(models_path.read_text()) models['providers']['fixture-provider']['baseUrl'] = self.scope.base_url models_path.write_text(json.dumps(models)) self.delay_first_s = 310 started = time.monotonic(); result = self.run_sdk(timeout=370) self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertGreaterEqual(time.monotonic() - started, 310) self.assertEqual(len(self.bodies), 2) checkpoint = json.loads((self.root / '.bench-session.json').read_text()) self.assertTrue(checkpoint['complete']) self.assertTrue(checkpoint['closure']) self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once') def test_bash_without_timeout_fails_alone_before_system_cap(self): self.tool_name, self.tool_args = 'bash', {'command': 'sleep 5; echo late'} self.config.update(tools=['bash'], ordinary_tool_deadline_s=62) result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(json.loads((self.root / '.bench-session.json').read_bytes())['phase'], 'COMPLETED') messages = self.bodies[1]['messages'] self.assertEqual(messages[-1]['content'], 'Command timed out after 1 seconds') call = next(m for m in messages if m.get('tool_calls'))['tool_calls'][0] self.assertEqual(json.loads(call['function']['arguments']), self.tool_args) def _events(self): return [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()] def test_exact_final_turn_uses_native_agent_end_and_persists(self): self.config.update(max_turns=2, max_tokens=100000) result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) checkpoint = json.loads((self.root / '.bench-session.json').read_text()) self.assertEqual(checkpoint['phase'], 'COMPLETED') self.assertTrue(checkpoint['complete']) self.assertFalse(checkpoint['resumable']) self.assertEqual(checkpoint['budget']['turns'], 2) self.assertEqual(len(self.bodies), 2) self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1) self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once') self.assertTrue(any(m.get('role') == 'custom' and m.get('customType') == 'bench-budget' for m in [e.get('message', {}) for e in self._events()])) def test_last_turn_tool_request_is_rejected_without_extra_generation(self): self.config.update(max_turns=2, max_tokens=100000) self.always_tools = True result = self.run_sdk() self.assertEqual(result.returncode, 75, result.stderr.decode()) checkpoint = json.loads((self.root / '.bench-session.json').read_text()) self.assertEqual(checkpoint['phase'], 'BUDGET_EXHAUSTED') self.assertFalse(checkpoint['complete']) self.assertTrue(checkpoint['closure']) self.assertEqual(len(self.bodies), 2) ends = [e for e in self._events() if e['type'] == 'tool_execution_end'] self.assertEqual(len(ends), 2) self.assertTrue(ends[-1]['isError']) self.assertIn('AGENT_BUDGET_EXHAUSTED', ends[-1]['result']['content'][0]['text']) self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1) def test_one_turn_is_reserved_and_cannot_execute_work(self): self.config.update(max_turns=1, max_tokens=100000) result = self.run_sdk() self.assertEqual(result.returncode, 75, result.stderr.decode()) self.assertEqual(len(self.bodies), 1) self.assertFalse((self.root / 'side-effect.txt').exists()) checkpoint = json.loads((self.root / '.bench-session.json').read_text()) self.assertTrue(checkpoint['closure']) self.assertFalse(checkpoint['complete']) def test_bash_tool_runs_through_the_agent_shell(self): settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['shellPath'] = str(ROOT / 'agent/shell') settings_path.write_text(json.dumps(settings)) self.config['tools'] = ['bash'] self.tool_name, self.tool_args = 'bash', {'command': '(echo out; exit 3) | cat'} result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) [end] = [e for e in self._events() if e['type'] == 'tool_execution_end'] text = end['result']['content'][0]['text'] self.assertFalse(end['isError']) self.assertIn('out', text); self.assertIn('管道第 1 步退出码 3', text) def test_request_over_the_account_limit_is_compacted_and_the_rollout_continues(self): # 10-19 staging: the broker's own per-request limit (tpm), not the provider's window, refused the request. # Pi must take it as a full context: compact (tool results shrink to 2000 chars) and go on. settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['compaction'] = {'enabled': True, 'keepRecentTokens': 1} settings_path.write_text(json.dumps(settings)) self.config['tools'] = ['bash'] self.tool_name, self.tool_id, self.tool_turns = 'bash', 'output', 2 self.tool_args = {'command': "if [ -f once ]; then head -c 6000 /dev/zero | tr '\\0' y; " "else touch once; head -c 12000 /dev/zero | tr '\\0' x; fi"} def limit_after_second(n, raw): # the next request, 6 KB bigger, no longer fits the account if n == 2: self.broker.groups['account'] = dataclasses.replace(self.broker.groups['account'], tpm=len(raw) + 3000) self.on_body = limit_after_second result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(json.loads((self.root / '.bench-session.json').read_bytes())['phase'], 'COMPLETED') rows = self.broker.execution_status(self.scope.execution_id)['generations'] self.assertEqual([(r['state'], r['reason']) for r in rows if ':g' not in r['generation_id']], [('COMPLETE', None), ('COMPLETE', None), ('FAILED', 'CONTEXT_TOO_LONG'), ('COMPLETE', None)]) self.assertIn('compacted into the following summary', json.dumps(self.bodies[-1]['messages'])) self.assertEqual(sum(e['type'] == 'compaction_end' for e in self._events()), 1) def test_context_overflow_is_compacted_by_pi_and_the_rollout_continues(self): # A full window: the broker hands the 400 back, Pi summarizes and resends under a new ID, the runner waits. settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['compaction'] = {'enabled': True, 'keepRecentTokens': 1} settings_path.write_text(json.dumps(settings)) self.overflow_on = {2} result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(json.loads((self.root / '.bench-session.json').read_bytes())['phase'], 'COMPLETED') self.assertEqual(len(self.bodies), 4) # tool call, overflow, summary, resent turn resent = json.dumps(self.bodies[3]['messages']) self.assertIn('compacted into the following summary', resent) self.assertNotIn('Write once.', resent) rows = self.broker.execution_status(self.scope.execution_id)['generations'] self.assertEqual([(r['generation_id'], r['state'], r['reason']) for r in rows if ':g' not in r['generation_id']], [('execution-fixture:0', 'COMPLETE', None), ('execution-fixture:1', 'FAILED', 'CONTEXT_TOO_LONG'), ('execution-fixture:2', 'COMPLETE', None)]) [end] = [e for e in self._events() if e['type'] == 'compaction_end'] kept = len(self.bodies[3]['messages']) - 2 # what Pi resent besides the system prompt and the summary self.assertEqual((end['willRetry'], end['bench_compaction']), (True, {'kept_messages': kept})) from export import trajectory rd = self.root / 'trajectory-fixture' (rd / 'agent').mkdir(parents=True) header = self._events()[0]['bench'] (rd / 'agent/events.jsonl').write_bytes((self.root / 'events.jsonl').read_bytes()) (rd / 'agent/system.md').write_text(header['system_prompt']) (rd / 'agent/tools.json').write_text(json.dumps(header['tools'])) (rd / 'rollout.json').write_text(json.dumps({'runner_version': 'pi-sdk-v1', 'state': 'DONE', 'execution_id': header['execution_id']})) frozen = trajectory.freeze(rd) output = trajectory.reconstruct(frozen) self.assertEqual(output['errors'], []) self.assertTrue(trajectory.validate(frozen, output)['pass']) roles = [m['role'] for m in output['messages']] self.assertEqual(roles, ['system', 'user', 'assistant', 'toolResult', 'assistant']) [view] = output['compactions'] self.assertEqual((view['at'], view['kept']), (4, kept)) seen = [output['messages'][0], view['summary_message']] + output['messages'][view['at'] - view['kept']:] self.assertEqual(len(seen), len(self.bodies[3]['messages']) + 1) # plus the final answer self.assertEqual(seen[1]['content'][0]['text'], self.bodies[3]['messages'][1]['content'][0]['text'] if isinstance(self.bodies[3]['messages'][1]['content'], list) else self.bodies[3]['messages'][1]['content']) def test_token_exhaustion_closes_native_call_without_starting_work(self): self.config.update(max_turns=10, max_tokens=1) result = self.run_sdk() self.assertEqual(result.returncode, 75, result.stderr.decode()) self.assertEqual(len(self.bodies), 1) self.assertFalse((self.root / 'side-effect.txt').exists()) checkpoint = json.loads((self.root / '.bench-session.json').read_text()) self.assertTrue(checkpoint['closure']) self.assertEqual(checkpoint['budget']['reason'], 'MAX_TOKENS') def test_answer_before_reserved_turn_does_not_trigger_extra_turn(self): self.config.update(max_turns=3, max_tokens=100000) result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(len(self.bodies), 2) self.assertFalse(any(e.get('message', {}).get('customType') == 'bench-budget' for e in self._events())) def _check_oe_doc(self, script, is_error, expected_text): from unittest.mock import patch from export import trajectory self.tool_name, self.tool_id = 'oe_doc', 'oe-doc-once' self.tool_args = {'query': 'fixture-query'} self.config['tools'] = ['oe_doc'] settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['extensions'] = [str(ROOT / 'agent/extensions/oe-doc.ts')] settings_path.write_text(json.dumps(settings)) if script is not None: script_path = self.root / 'agent/oe-doc.sh' script_path.parent.mkdir() script_path.write_text('#!/bin/sh\n' + script) script_path.chmod(0o700) with patch.dict(os.environ, {'BENCH_ROOT': str(self.root)}): result = self.run_sdk() self.assertEqual(result.returncode, 0, result.stderr.decode()) self.assertEqual(len(self.bodies), 2) events = [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()] ends = [e for e in events if e['type'] == 'tool_execution_end'] messages = [e['message'] for e in events if e['type'] == 'message_end'] results = [m for m in messages if m['role'] == 'toolResult'] self.assertEqual(len(ends), 1) self.assertEqual(len(results), 1) for event in (ends[0], results[0]): self.assertEqual((event['toolCallId'], event['toolName']), (self.tool_id, self.tool_name)) self.assertIs(event['isError'], is_error) self.assertIn(expected_text, results[0]['content'][0]['text']) checkpoint = json.loads((self.root / '.bench-session.json').read_bytes()) self.assertEqual(checkpoint['phase'], 'COMPLETED') self.assertTrue(checkpoint['closure']) persisted = [json.loads(line).get('message', {}) for line in Path(checkpoint['session_path']).read_text().splitlines()] self.assertEqual([m for m in persisted if m.get('role') == 'toolResult'], results) # Real SDK output through the native converter; metadata makes no board/SFT-eligibility claim. rd = self.root / 'trajectory-fixture' (rd / 'agent').mkdir(parents=True) header = events[0]['bench'] for name, content in { 'events.jsonl': (self.root / 'events.jsonl').read_bytes(), 'system.md': header['system_prompt'].encode(), 'tools.json': json.dumps(header['tools']).encode(), 'prompt.md': (self.root / 'prompt.md').read_bytes(), }.items(): (rd / 'agent' / name).write_bytes(content) (rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1', 'execution_id': self.scope.execution_id, 'state': 'COMPLETED', 'evidence_scope': 'fake-provider-no-board'})) frozen = trajectory.freeze(rd) output = trajectory.reconstruct(frozen) # Export preserves native fields while applying its source-seeded privacy contract. from export.redact import redact_preserving expected, _ = redact_preserving({'messages': [e['message'] for e in events if e.get('type') == 'message_end'], 'tools': header['tools']}) self.assertEqual([m for m in output['messages'] if m['role'] == 'toolResult'], [m for m in expected['messages'] if m['role'] == 'toolResult']) self.assertEqual(output['excluded'], []) quality = trajectory.validate(frozen, output) self.assertTrue(quality['pass'], quality) def test_oe_doc_success_preserves_success_through_sdk_and_export(self): self._check_oe_doc('printf "fixture docs: %s\\n" "$1"\n', False, 'fixture docs: fixture-query') def test_oe_doc_exit_failure_preserves_error_through_sdk_and_export(self): self._check_oe_doc('printf "fixture exit 7\\n" >&2\nexit 7\n', True, 'fixture exit 7') def test_oe_doc_spawn_failure_preserves_error_through_sdk_and_export(self): self._check_oe_doc(None, True, 'ENOENT') def test_pause_continue_same_generation_without_user_or_tool_replay(self): import time self.fail_second = True first = self.run_sdk() self.assertEqual(first.returncode, 76, first.stderr.decode()) checkpoint = json.loads((self.root / '.bench-session.json').read_bytes()) self.assertFalse(checkpoint['complete']) self.assertTrue(checkpoint['resumable']) original_stat = (self.root / 'side-effect.txt').stat() header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0]) self.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta']) time.sleep(.25) self.broker.resume(self.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True) second = self.run_sdk() self.assertEqual(second.returncode, 0, second.stderr.decode()) self.assertEqual(len(self.bodies), 3) self.assertEqual(self.bodies[1], self.bodies[2]) self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, original_stat.st_mtime_ns) messages = [json.loads(line)['message'] for line in (self.root / 'events.jsonl').read_text().splitlines() if json.loads(line).get('type') == 'message_end'] self.assertEqual(sum(m['role'] == 'user' for m in messages), 1) self.assertEqual(sum(m['role'] == 'toolResult' for m in messages), 1) def test_multiple_pauses_preserve_raw_errors_without_replaying_tools(self): import time self.fail_second, self.fail_on = True, {2, 3} mtime = None for expected in (76, 76, 0): result = self.run_sdk() self.assertEqual(result.returncode, expected, result.stderr.decode()) actual = (self.root / 'side-effect.txt').stat().st_mtime_ns if mtime is not None: self.assertEqual(actual, mtime) mtime = actual if expected == 0: break c = json.loads((self.root / '.bench-session.json').read_bytes()) header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0]) self.config.update(resume=True, session_path=c['session_path'], session_sha256=c['sha256'], system_prompt=header['bench']['system_prompt'], system_sha256=c['system_sha256'], generation=c['generation'], nonempty_delta=c['nonempty_delta']) time.sleep(.25) self.broker.resume(self.scope.execution_id, c['generation_id'], checkpoint_complete=True, side_effects_known=True) self.assertEqual(len(self.bodies), 4) self.assertEqual(self.bodies[1], self.bodies[2]) self.assertEqual(self.bodies[2], self.bodies[3]) records = [json.loads(line) for line in Path(c['session_path']).read_text().splitlines()] self.assertEqual(sum(r.get('message', {}).get('stopReason') == 'error' for r in records), 2) def test_budget_custom_pause_resume_preserves_context_and_export(self): """A real production extension's custom message survives safe continuation.""" import time from unittest.mock import patch from export import trajectory self.fail_second = True settings_path = self.root / '.pi/settings.json' settings = json.loads(settings_path.read_text()) settings['extensions'] = [str(ROOT / 'agent/extensions/budget-kill.ts')] settings_path.write_text(json.dumps(settings)) budget = {'BENCH_MAX_TURNS': '2', 'BENCH_MAX_TOKENS': '1000000', 'BENCH_WARN_AHEAD_TURNS': '5', 'BENCH_ROLLOUT_DIR': '', 'BENCH_INITIAL_TURNS': '0', 'BENCH_INITIAL_TOKENS': '0'} with patch.dict(os.environ, budget): first = self.run_sdk() self.assertEqual(first.returncode, 76, first.stderr.decode()) checkpoint = json.loads((self.root / '.bench-session.json').read_bytes()) self.assertFalse(checkpoint['complete']) self.assertTrue(checkpoint['resumable']) mtime = (self.root / 'side-effect.txt').stat().st_mtime_ns header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0]) self.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta']) time.sleep(.25) self.broker.resume(self.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True) with patch.dict(os.environ, {**budget, 'BENCH_INITIAL_TURNS': '1', 'BENCH_INITIAL_TOKENS': '15'}): second = self.run_sdk() self.assertEqual(second.returncode, 0, second.stderr.decode()) self.assertEqual(self.bodies[1], self.bodies[2]) self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, mtime) # Actual SDK events; only source metadata is a fixture, with no board claim. rd = self.root / 'trajectory-fixture' (rd / 'agent').mkdir(parents=True) for name, content in { 'events.jsonl': (self.root / 'events.jsonl').read_bytes(), 'system.md': header['bench']['system_prompt'].encode(), 'tools.json': json.dumps(header['bench']['tools']).encode(), 'prompt.md': (self.root / 'prompt.md').read_bytes(), }.items(): (rd / 'agent' / name).write_bytes(content) (rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1', 'execution_id': self.scope.execution_id, 'state': 'COMPLETED', 'evidence_scope': 'fake-provider-no-board'})) frozen = trajectory.freeze(rd) output = trajectory.reconstruct(frozen) quality = trajectory.validate(frozen, output) self.assertTrue(quality['pass'], quality) self.assertTrue(any(message['role'] == 'custom' and message.get('customType') == 'bench-budget' for message in output['messages'])) def test_harness_pause_resume_keeps_execution_budget_and_attempt(self): """Exercise actual Python -> broker -> SDK -> tool -> durable pause wiring.""" from contextlib import ExitStack from dataclasses import asdict import shutil import time from unittest.mock import patch import yaml from harness import collection_runtime as runtime, db, paths, rollout self.fail_second = True project = self.root / 'project' rd = project / 'runs/r/j/rollouts/00' workspace = rd / 'workspace' workspace.mkdir(parents=True) (rd / 'agent').mkdir() shutil.move(str(self.root / '.pi'), workspace / '.pi') shutil.move(str(self.home), workspace / '.agent-home') models_path = workspace / '.agent-home/.pi/agent/models.json' models = json.loads(models_path.read_bytes()) models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider') models_path.write_text(json.dumps(models)) settings_path = workspace / '.pi/settings.json' settings = json.loads(settings_path.read_bytes()); settings['defaultProvider'] = 'dr-gateway' settings_path.write_text(json.dumps(settings)) for name in ('system.md', 'append-system.md', 'prompt.md'): (rd / 'agent' / name).write_text('Write once, then report done.') from harness.runners import pi metadata = {'state': 'RUNNING', 'runner': 'pi', 'runner_version': pi.VERSION, 'policy': 'teacher', 'mode': 'e2e', 'jail_policy': 'private-home', 'max_turns': 60, 'max_tokens': 200000} (rd / 'rollout.json').write_text(json.dumps(metadata)) (rd.parent.parent / 'job.json').write_text('{}') cfg = yaml.safe_load((ROOT / 'config.yaml').read_text()) cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE)) cfg['rollouts'].update(jail=False, private_home=True, tools=['write']) cfg['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % self.server.server_port, model='fixture', api_key_env='FIXTURE_KEY') cfg['collection_v1'].update(quota_ledger=str(project / 'runtime-quota.sqlite'), quota_groups={'account': asdict(self.broker.groups['account'])}, credential_groups={'teacher': 'account'}) state = project / 'state'; state.mkdir() config_path = project / 'config.yaml'; config_path.write_text(yaml.safe_dump(cfg)) original_connect, original_readonly = db.connect, db.readonly with ExitStack() as stack: for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent', 'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': config_path}.items(): stack.enter_context(patch.object(paths, name, value)) stack.enter_context(patch.object(paths, 'config', return_value=cfg)) stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture')) stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(state / 'jobs.sqlite'))) stack.enter_context(patch.object(db, 'readonly', side_effect=lambda: original_readonly(state / 'jobs.sqlite'))) stack.enter_context(patch.object(runtime, '_broker', None)) stack.enter_context(patch.object(runtime, '_configuration', None)) stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, {}, 'Write once.'))) # Fake provider proves process/protocol wiring only, never board PASS. stack.enter_context(patch.object(rollout, 'verify_rollout', return_value={'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False})) with db.connect() as con: con.execute("INSERT INTO runs VALUES ('r',0,'{}','fixture')") db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING') try: first = rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False) self.assertEqual(first['terminal'], 'PAUSED_INFRA') before = json.loads((rd / 'execution.json').read_bytes()) bound = json.loads((rd / 'rollout.json').read_bytes()) self.assertEqual(bound['runner_version'], 'pi-sdk-v1') self.assertEqual(bound['execution_id'], before['execution_id']) self.assertFalse(runtime.broker().execution_status(before['execution_id'])['cancelled']) from export import trajectory frozen = trajectory.freeze(rd) self.assertEqual(trajectory.reconstruct(frozen)['context']['status'], 'COMPLETE') stat = (workspace / 'side-effect.txt').stat() time.sleep(.25) second = rollout.resume_rollout('r', 'j', 0) after = json.loads((rd / 'execution.json').read_bytes()) self.assertEqual(second['terminal'], 'UNVERIFIED') self.assertEqual(after['execution_outcome'], 'COMPLETED') self.assertTrue(runtime.broker().execution_status(after['execution_id'])['cancelled']) self.assertEqual((after['execution_id'], after['deadline_monotonic']), (before['execution_id'], before['deadline_monotonic'])) self.assertEqual(after['epoch'], before['epoch'] + 1) self.assertEqual(stat.st_mtime_ns, (workspace / 'side-effect.txt').stat().st_mtime_ns) self.assertEqual(runtime.budget_used(rd)['turns'], 2) self.assertEqual(len(list(rd.parent.iterdir())), 1) finally: if runtime._broker: runtime._broker.close() if __name__ == '__main__': unittest.main()