data-gen-bpu/scripts/test_pi_session.py
ming01.wang f611ef29ab fix: keep managed API requests within the broker lifecycle
Pi requests could hit Node fetch's independent five-minute header deadline,
and queued requests could outlive their local client or terminal rollout.
Set header/body deadlines only for the loopback broker origin, cancel scopes
when clients disconnect or executions terminate, and retain uncertain upstream
reservations instead of retrying after cancellation. Resumable pauses remain open.

Validation: real Pi SDK accepted a response delayed 310 seconds (314.139 s test);
quota, Pi session, acceptance, cancellation, and lifecycle regression coverage.
2026-09-22 20:28:42 +08:00

512 lines
32 KiB
Python

"""Real installed Pi SDK against a local fake provider; no account or board calls."""
from contextlib import contextmanager
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import subprocess
import tempfile
import threading
import time
import unittest
from harness.quota import Broker, QuotaPolicy, Upstream
ROOT = Path(__file__).resolve().parents[1]
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
@unittest.skipUnless(NODE.is_file(), 'installed Pi 0.73.1 SDK required')
class PiSDKTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.home = self.root / 'home'
self.agent_dir = self.home / '.pi/agent'
self.agent_dir.mkdir(parents=True)
(self.root / '.pi').mkdir()
self.bodies = []
self.tool_name, self.tool_id = 'write', 'write-once'
self.tool_args = {'path': 'side-effect.txt', 'content': 'once'}
self.fail_second = False
self.delay_first_s = 0
self.always_tools = False
self.fail_on = {2}
outer = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
outer.bodies.append(body)
if len(outer.bodies) == 1 and outer.delay_first_s:
time.sleep(outer.delay_first_s)
if outer.fail_second and len(outer.bodies) in outer.fail_on:
payload = b'{"error":{"message":"fixture rate limit","type":"rate_limit"}}'
self.send_response(429); self.send_header('Retry-After', '0.2')
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
return
delta = ({'role': 'assistant', 'tool_calls': [{'index': 0, 'id': outer.tool_id if len(outer.bodies) == 1 else outer.tool_id + '-' + str(len(outer.bodies)), 'type': 'function',
'function': {'name': outer.tool_name, 'arguments': json.dumps(outer.tool_args)}}]}
if len(outer.bodies) == 1 or outer.always_tools else {'role': 'assistant', 'content': 'done'})
finish = 'tool_calls' if len(outer.bodies) == 1 or outer.always_tools else 'stop'
chunks = [dict(id='fixture', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta=delta, finish_reason=None)]),
dict(id='fixture', object='chat.completion.chunk', model='fixture',
choices=[dict(index=0, delta={}, finish_reason=finish)], usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
payload = ''.join('data: ' + json.dumps(chunk) + '\n\n' for chunk in chunks).encode() + b'data: [DONE]\n\n'
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
self.addCleanup(self.server.server_close); self.addCleanup(self.server.shutdown)
self.broker = Broker(self.root / 'quota.sqlite', {'account': QuotaPolicy(1, 100, 10000000,
wait_budget_s=.05, cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0,
connect_timeout_s=1, first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.005)},
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port, 'upstream-fixture', ('fixture',))})
self.broker.start(); self.addCleanup(self.broker.close)
self.scope = self.broker.issue_scope('execution-fixture', 'teacher', 'fixture')
settings = {'defaultProvider': 'fixture-provider', 'defaultModel': 'fixture', 'compaction': {'enabled': False},
'retry': {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': self.scope.client_timeout_s * 1000}},
'extensions': [], 'skills': []}
(self.root / '.pi/settings.json').write_text(json.dumps(settings))
models = {'providers': {'fixture-provider': {'baseUrl': self.scope.base_url, 'api': 'openai-completions',
'apiKey': 'BENCH_RUNNER_API_KEY', 'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False,
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 512,
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}
(self.agent_dir / 'models.json').write_text(json.dumps(models))
(self.root / 'system.md').write_text('Use write once, then report completion.')
(self.root / 'prompt.md').write_text('Write once.')
self.config = {'execution_id': self.scope.execution_id, 'model': 'fixture', 'provider': 'fixture-provider',
'base_url': self.scope.base_url, 'client_timeout_ms': self.scope.client_timeout_s * 1000,
'session_dir': str(self.root / '.pi-sessions'), 'tools': ['write'],
'append_system_path': str(self.root / 'system.md'), 'prompt_path': str(self.root / 'prompt.md'), 'resume': False}
def run_sdk(self, timeout=25):
config = self.root / 'runner.json'; config.write_text(json.dumps(self.config))
with (self.root / 'events.jsonl').open('ab') as output:
result = subprocess.run([str(NODE), str(ROOT / 'agent/pi_session.js'), str(config)], cwd=self.root,
env={**os.environ, 'HOME': str(self.home), 'PI_CODING_AGENT_DIR': str(self.agent_dir),
'BENCH_RUNNER_API_KEY': self.scope.api_key}, stdout=output, stderr=subprocess.PIPE, timeout=timeout)
return result
def test_actual_prepare_binds_source_before_sdk_or_http(self):
from contextlib import ExitStack
from unittest.mock import patch
import yaml
from harness import collection_runtime as runtime, db, paths, rollout
from harness.execution import Execution
project = self.root / 'prepared'
job_dir = project / 'runs/r/j'
job_dir.mkdir(parents=True)
job = {'model': 'resnet18', 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq',
'qcfg': 'fp16int8', 'model_spec': {'input_spec': '1x3x224x224'},
'gates': {'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}},
'data': {'calib': {'path': str(project / 'calib'), 'n': 32},
't1': {'path': str(project / 'test.tar'), 'n': 100}}}
(job_dir / 'job.json').write_text(json.dumps(job))
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
cfg['rollouts'].update(private_home=True, tools=['write'])
cfg['endpoints']['teacher']['model'] = 'fixture'
cfg['collection_v1']['credential_groups'] = {'teacher': 'account'}
original_connect = db.connect
database = project / 'state/jobs.sqlite'
with ExitStack() as stack:
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent,
'AGENT': ROOT / 'agent', 'RUNS': project / 'runs'}.items():
stack.enter_context(patch.object(paths, name, value))
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(database)))
stack.enter_context(patch.object(rollout, 'onnx_path', return_value=None))
stack.enter_context(patch.object(rollout, 'baseline_dir', return_value=project / 'baseline'))
stack.enter_context(patch.object(runtime, 'config', return_value=(cfg['collection_v1'], self.broker.groups)))
stack.enter_context(patch.object(runtime, 'broker', return_value=self.broker))
with db.connect() as con:
con.execute("INSERT INTO runs VALUES ('r',0,'{}','source-binding fixture')")
rd, _, _ = rollout.prepare('r', 'j', 0, 'e2e', 'teacher')
self.assertEqual(json.loads((rd / 'rollout.json').read_bytes())['runner_version'], 'pi-sdk-v1')
execution = Execution(rd)
execution.start(cfg['collection_v1']['lifecycle'])
runtime.configure_pi(rd, execution, 'teacher')
bound = json.loads((rd / 'rollout.json').read_bytes())
self.assertEqual(bound['execution_id'], execution.snapshot()['execution_id'])
self.assertFalse((rd / 'agent/events.jsonl').exists())
self.assertEqual(self.bodies, [])
for fields, reason in [({'execution_id': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
({'runner_version': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
({'execution_id': None}, 'RESUME_SOURCE_IDENTITY_MISSING')]:
raw = json.dumps({**bound, **fields}).encode()
(rd / 'rollout.json').write_bytes(raw)
with self.assertRaisesRegex(ValueError, reason):
runtime.configure_pi(rd, execution, 'teacher', resume=True)
self.assertEqual((rd / 'rollout.json').read_bytes(), raw)
(rd / 'rollout.json').write_text(json.dumps(bound))
def test_actual_sdk_uses_broker_and_persists_complete_tool_chain(self):
result = self.run_sdk()
self.assertEqual(result.returncode, 0, result.stderr.decode())
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
self.assertEqual(checkpoint['phase'], 'COMPLETED')
self.assertTrue(checkpoint['closure'])
self.assertEqual(len(self.bodies), 2)
self.assertEqual([row['generation_id'] for row in self.broker.execution_status(self.scope.execution_id)['generations']],
['execution-fixture:0', 'execution-fixture:1'])
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
@unittest.skipUnless(os.environ.get('BENCH_TEST_LONG_QUEUE') == '1', 'explicit 310 second regression')
def test_actual_sdk_waits_past_node_fetch_five_minute_header_limit(self):
self.broker.close()
policy = QuotaPolicy(1, 100, 10000000, first_byte_timeout_s=340,
stream_idle_timeout_s=340, request_total_s=350, wait_budget_s=340)
self.broker = Broker(self.root / 'long-quota.sqlite', {'account': policy},
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port,
'upstream-fixture', ('fixture',))}).start()
self.addCleanup(self.broker.close)
self.scope = self.broker.issue_scope('long-execution', 'teacher', 'fixture')
self.config.update(execution_id=self.scope.execution_id, base_url=self.scope.base_url,
client_timeout_ms=self.scope.client_timeout_s * 1000)
settings_path = self.root / '.pi/settings.json'
settings = json.loads(settings_path.read_text())
settings['retry']['provider']['timeoutMs'] = self.scope.client_timeout_s * 1000
settings_path.write_text(json.dumps(settings))
models_path = self.agent_dir / 'models.json'; models = json.loads(models_path.read_text())
models['providers']['fixture-provider']['baseUrl'] = self.scope.base_url
models_path.write_text(json.dumps(models))
self.delay_first_s = 310
started = time.monotonic(); result = self.run_sdk(timeout=370)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertGreaterEqual(time.monotonic() - started, 310)
self.assertEqual(len(self.bodies), 2)
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
self.assertTrue(checkpoint['complete'])
self.assertTrue(checkpoint['closure'])
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
def _events(self):
return [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
def test_exact_final_turn_uses_native_agent_end_and_persists(self):
self.config.update(max_turns=2, max_tokens=100000)
result = self.run_sdk()
self.assertEqual(result.returncode, 0, result.stderr.decode())
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
self.assertEqual(checkpoint['phase'], 'COMPLETED')
self.assertTrue(checkpoint['complete'])
self.assertFalse(checkpoint['resumable'])
self.assertEqual(checkpoint['budget']['turns'], 2)
self.assertEqual(len(self.bodies), 2)
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
self.assertTrue(any(m.get('role') == 'custom' and m.get('customType') == 'bench-budget'
for m in [e.get('message', {}) for e in self._events()]))
def test_last_turn_tool_request_is_rejected_without_extra_generation(self):
self.config.update(max_turns=2, max_tokens=100000)
self.always_tools = True
result = self.run_sdk()
self.assertEqual(result.returncode, 75, result.stderr.decode())
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
self.assertEqual(checkpoint['phase'], 'BUDGET_EXHAUSTED')
self.assertFalse(checkpoint['complete'])
self.assertTrue(checkpoint['closure'])
self.assertEqual(len(self.bodies), 2)
ends = [e for e in self._events() if e['type'] == 'tool_execution_end']
self.assertEqual(len(ends), 2)
self.assertTrue(ends[-1]['isError'])
self.assertIn('AGENT_BUDGET_EXHAUSTED', ends[-1]['result']['content'][0]['text'])
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
def test_one_turn_is_reserved_and_cannot_execute_work(self):
self.config.update(max_turns=1, max_tokens=100000)
result = self.run_sdk()
self.assertEqual(result.returncode, 75, result.stderr.decode())
self.assertEqual(len(self.bodies), 1)
self.assertFalse((self.root / 'side-effect.txt').exists())
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
self.assertTrue(checkpoint['closure'])
self.assertFalse(checkpoint['complete'])
def test_token_exhaustion_closes_native_call_without_starting_work(self):
self.config.update(max_turns=10, max_tokens=1)
result = self.run_sdk()
self.assertEqual(result.returncode, 75, result.stderr.decode())
self.assertEqual(len(self.bodies), 1)
self.assertFalse((self.root / 'side-effect.txt').exists())
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
self.assertTrue(checkpoint['closure'])
self.assertEqual(checkpoint['budget']['reason'], 'MAX_TOKENS')
def test_answer_before_reserved_turn_does_not_trigger_extra_turn(self):
self.config.update(max_turns=3, max_tokens=100000)
result = self.run_sdk()
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(len(self.bodies), 2)
self.assertFalse(any(e.get('message', {}).get('customType') == 'bench-budget' for e in self._events()))
def _check_oe_doc(self, script, is_error, expected_text):
from unittest.mock import patch
from export import trajectory
self.tool_name, self.tool_id = 'oe_doc', 'oe-doc-once'
self.tool_args = {'query': 'fixture-query'}
self.config['tools'] = ['oe_doc']
settings_path = self.root / '.pi/settings.json'
settings = json.loads(settings_path.read_text())
settings['extensions'] = [str(ROOT / 'agent/extensions/oe-doc.ts')]
settings_path.write_text(json.dumps(settings))
if script is not None:
script_path = self.root / 'agent/oe-doc.sh'
script_path.parent.mkdir()
script_path.write_text('#!/bin/sh\n' + script)
script_path.chmod(0o700)
with patch.dict(os.environ, {'BENCH_ROOT': str(self.root)}):
result = self.run_sdk()
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(len(self.bodies), 2)
events = [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
ends = [e for e in events if e['type'] == 'tool_execution_end']
messages = [e['message'] for e in events if e['type'] == 'message_end']
results = [m for m in messages if m['role'] == 'toolResult']
self.assertEqual(len(ends), 1)
self.assertEqual(len(results), 1)
for event in (ends[0], results[0]):
self.assertEqual((event['toolCallId'], event['toolName']), (self.tool_id, self.tool_name))
self.assertIs(event['isError'], is_error)
self.assertIn(expected_text, results[0]['content'][0]['text'])
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
self.assertEqual(checkpoint['phase'], 'COMPLETED')
self.assertTrue(checkpoint['closure'])
persisted = [json.loads(line).get('message', {})
for line in Path(checkpoint['session_path']).read_text().splitlines()]
self.assertEqual([m for m in persisted if m.get('role') == 'toolResult'], results)
# Real SDK output through the native converter; metadata makes no board/SFT-eligibility claim.
rd = self.root / 'trajectory-fixture'
(rd / 'agent').mkdir(parents=True)
header = events[0]['bench']
for name, content in {
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
'system.md': header['system_prompt'].encode(),
'tools.json': json.dumps(header['tools']).encode(),
'prompt.md': (self.root / 'prompt.md').read_bytes(),
}.items():
(rd / 'agent' / name).write_bytes(content)
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
'evidence_scope': 'fake-provider-no-board'}))
frozen = trajectory.freeze(rd)
output = trajectory.reconstruct(frozen)
# Export preserves native fields while applying its source-seeded privacy contract.
from export.redact import redact_preserving
expected, _ = redact_preserving({'messages': [e['message'] for e in events
if e.get('type') == 'message_end'], 'tools': header['tools']})
self.assertEqual([m for m in output['messages'] if m['role'] == 'toolResult'],
[m for m in expected['messages'] if m['role'] == 'toolResult'])
self.assertEqual(output['excluded'], [])
quality = trajectory.validate(frozen, output)
self.assertTrue(quality['pass'], quality)
def test_oe_doc_success_preserves_success_through_sdk_and_export(self):
self._check_oe_doc('printf "fixture docs: %s\\n" "$1"\n', False, 'fixture docs: fixture-query')
def test_oe_doc_exit_failure_preserves_error_through_sdk_and_export(self):
self._check_oe_doc('printf "fixture exit 7\\n" >&2\nexit 7\n', True, 'fixture exit 7')
def test_oe_doc_spawn_failure_preserves_error_through_sdk_and_export(self):
self._check_oe_doc(None, True, 'ENOENT')
def test_pause_continue_same_generation_without_user_or_tool_replay(self):
import time
self.fail_second = True
first = self.run_sdk()
self.assertEqual(first.returncode, 76, first.stderr.decode())
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
self.assertFalse(checkpoint['complete'])
self.assertTrue(checkpoint['resumable'])
original_stat = (self.root / 'side-effect.txt').stat()
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
self.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'],
system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'],
generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta'])
time.sleep(.25)
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True)
second = self.run_sdk()
self.assertEqual(second.returncode, 0, second.stderr.decode())
self.assertEqual(len(self.bodies), 3)
self.assertEqual(self.bodies[1], self.bodies[2])
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, original_stat.st_mtime_ns)
messages = [json.loads(line)['message'] for line in (self.root / 'events.jsonl').read_text().splitlines()
if json.loads(line).get('type') == 'message_end']
self.assertEqual(sum(m['role'] == 'user' for m in messages), 1)
self.assertEqual(sum(m['role'] == 'toolResult' for m in messages), 1)
def test_multiple_pauses_preserve_raw_errors_without_replaying_tools(self):
import time
self.fail_second, self.fail_on = True, {2, 3}
mtime = None
for expected in (76, 76, 0):
result = self.run_sdk()
self.assertEqual(result.returncode, expected, result.stderr.decode())
actual = (self.root / 'side-effect.txt').stat().st_mtime_ns
if mtime is not None: self.assertEqual(actual, mtime)
mtime = actual
if expected == 0: break
c = json.loads((self.root / '.bench-session.json').read_bytes())
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
self.config.update(resume=True, session_path=c['session_path'], session_sha256=c['sha256'],
system_prompt=header['bench']['system_prompt'], system_sha256=c['system_sha256'],
generation=c['generation'], nonempty_delta=c['nonempty_delta'])
time.sleep(.25)
self.broker.resume(self.scope.execution_id, c['generation_id'], checkpoint_complete=True, side_effects_known=True)
self.assertEqual(len(self.bodies), 4)
self.assertEqual(self.bodies[1], self.bodies[2])
self.assertEqual(self.bodies[2], self.bodies[3])
records = [json.loads(line) for line in Path(c['session_path']).read_text().splitlines()]
self.assertEqual(sum(r.get('message', {}).get('stopReason') == 'error' for r in records), 2)
def test_budget_custom_pause_resume_preserves_context_and_export(self):
"""A real production extension's custom message survives safe continuation."""
import time
from unittest.mock import patch
from export import trajectory
self.fail_second = True
settings_path = self.root / '.pi/settings.json'
settings = json.loads(settings_path.read_text())
settings['extensions'] = [str(ROOT / 'agent/extensions/budget-kill.ts')]
settings_path.write_text(json.dumps(settings))
budget = {'BENCH_MAX_TURNS': '2', 'BENCH_MAX_TOKENS': '1000000',
'BENCH_WARN_AHEAD_TURNS': '5', 'BENCH_ROLLOUT_DIR': '',
'BENCH_INITIAL_TURNS': '0', 'BENCH_INITIAL_TOKENS': '0'}
with patch.dict(os.environ, budget):
first = self.run_sdk()
self.assertEqual(first.returncode, 76, first.stderr.decode())
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
self.assertFalse(checkpoint['complete'])
self.assertTrue(checkpoint['resumable'])
mtime = (self.root / 'side-effect.txt').stat().st_mtime_ns
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
self.config.update(resume=True, session_path=checkpoint['session_path'],
session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'],
system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'],
nonempty_delta=checkpoint['nonempty_delta'])
time.sleep(.25)
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'],
checkpoint_complete=True, side_effects_known=True)
with patch.dict(os.environ, {**budget, 'BENCH_INITIAL_TURNS': '1', 'BENCH_INITIAL_TOKENS': '15'}):
second = self.run_sdk()
self.assertEqual(second.returncode, 0, second.stderr.decode())
self.assertEqual(self.bodies[1], self.bodies[2])
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, mtime)
# Actual SDK events; only source metadata is a fixture, with no board claim.
rd = self.root / 'trajectory-fixture'
(rd / 'agent').mkdir(parents=True)
for name, content in {
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
'system.md': header['bench']['system_prompt'].encode(),
'tools.json': json.dumps(header['bench']['tools']).encode(),
'prompt.md': (self.root / 'prompt.md').read_bytes(),
}.items():
(rd / 'agent' / name).write_bytes(content)
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
'evidence_scope': 'fake-provider-no-board'}))
frozen = trajectory.freeze(rd)
output = trajectory.reconstruct(frozen)
quality = trajectory.validate(frozen, output)
self.assertTrue(quality['pass'], quality)
self.assertTrue(any(message['role'] == 'custom' and message.get('customType') == 'bench-budget'
for message in output['messages']))
def test_harness_pause_resume_keeps_execution_budget_and_attempt(self):
"""Exercise actual Python -> broker -> SDK -> tool -> durable pause wiring."""
from contextlib import ExitStack
from dataclasses import asdict
import shutil
import time
from unittest.mock import patch
import yaml
from harness import collection_runtime as runtime, db, paths, rollout
self.fail_second = True
project = self.root / 'project'
rd = project / 'runs/r/j/rollouts/00'
workspace = rd / 'workspace'
workspace.mkdir(parents=True)
(rd / 'agent').mkdir()
shutil.move(str(self.root / '.pi'), workspace / '.pi')
shutil.move(str(self.home), workspace / '.agent-home')
models_path = workspace / '.agent-home/.pi/agent/models.json'
models = json.loads(models_path.read_bytes())
models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider')
models_path.write_text(json.dumps(models))
settings_path = workspace / '.pi/settings.json'
settings = json.loads(settings_path.read_bytes()); settings['defaultProvider'] = 'dr-gateway'
settings_path.write_text(json.dumps(settings))
for name in ('system.md', 'append-system.md', 'prompt.md'):
(rd / 'agent' / name).write_text('Write once, then report done.')
from harness.runners import pi
metadata = {'state': 'RUNNING', 'runner': 'pi', 'runner_version': pi.VERSION, 'policy': 'teacher', 'mode': 'e2e', 'jail_policy': 'private-home',
'max_turns': 60, 'max_tokens': 200000}
(rd / 'rollout.json').write_text(json.dumps(metadata))
(rd.parent.parent / 'job.json').write_text('{}')
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
cfg['rollouts'].update(jail=False, private_home=True, tools=['write'])
cfg['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % self.server.server_port,
model='fixture', api_key_env='FIXTURE_KEY')
cfg['collection_v1'].update(quota_ledger=str(project / 'runtime-quota.sqlite'),
quota_groups={'account': asdict(self.broker.groups['account'])}, credential_groups={'teacher': 'account'})
state = project / 'state'; state.mkdir()
config_path = project / 'config.yaml'; config_path.write_text(yaml.safe_dump(cfg))
original_connect, original_readonly = db.connect, db.readonly
with ExitStack() as stack:
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': config_path}.items():
stack.enter_context(patch.object(paths, name, value))
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(state / 'jobs.sqlite')))
stack.enter_context(patch.object(db, 'readonly', side_effect=lambda: original_readonly(state / 'jobs.sqlite')))
stack.enter_context(patch.object(runtime, '_broker', None))
stack.enter_context(patch.object(runtime, '_configuration', None))
stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, {}, 'Write once.')))
# Fake provider proves process/protocol wiring only, never board PASS.
stack.enter_context(patch.object(rollout, 'verify_rollout', return_value={'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False}))
with db.connect() as con:
con.execute("INSERT INTO runs VALUES ('r',0,'{}','fixture')")
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
try:
first = rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False)
self.assertEqual(first['terminal'], 'PAUSED_INFRA')
before = json.loads((rd / 'execution.json').read_bytes())
bound = json.loads((rd / 'rollout.json').read_bytes())
self.assertEqual(bound['runner_version'], 'pi-sdk-v1')
self.assertEqual(bound['execution_id'], before['execution_id'])
self.assertFalse(runtime.broker().execution_status(before['execution_id'])['cancelled'])
from export import trajectory
frozen = trajectory.freeze(rd)
self.assertEqual(trajectory.reconstruct(frozen)['context']['status'], 'COMPLETE')
stat = (workspace / 'side-effect.txt').stat()
time.sleep(.25)
second = rollout.resume_rollout('r', 'j', 0)
after = json.loads((rd / 'execution.json').read_bytes())
self.assertEqual(second['terminal'], 'UNVERIFIED')
self.assertEqual(after['execution_outcome'], 'COMPLETED')
self.assertTrue(runtime.broker().execution_status(after['execution_id'])['cancelled'])
self.assertEqual((after['execution_id'], after['deadline_monotonic']),
(before['execution_id'], before['deadline_monotonic']))
self.assertEqual(after['epoch'], before['epoch'] + 1)
self.assertEqual(stat.st_mtime_ns, (workspace / 'side-effect.txt').stat().st_mtime_ns)
self.assertEqual(runtime.budget_used(rd)['turns'], 2)
self.assertEqual(len(list(rd.parent.iterdir())), 1)
finally:
if runtime._broker:
runtime._broker.close()
if __name__ == '__main__':
unittest.main()