Pi requests could hit Node fetch's independent five-minute header deadline, and queued requests could outlive their local client or terminal rollout. Set header/body deadlines only for the loopback broker origin, cancel scopes when clients disconnect or executions terminate, and retain uncertain upstream reservations instead of retrying after cancellation. Resumable pauses remain open. Validation: real Pi SDK accepted a response delayed 310 seconds (314.139 s test); quota, Pi session, acceptance, cancellation, and lifecycle regression coverage.
512 lines
32 KiB
Python
512 lines
32 KiB
Python
"""Real installed Pi SDK against a local fake provider; no account or board calls."""
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
|
|
from harness.quota import Broker, QuotaPolicy, Upstream
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
|
|
|
|
|
|
@unittest.skipUnless(NODE.is_file(), 'installed Pi 0.73.1 SDK required')
|
|
class PiSDKTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.home = self.root / 'home'
|
|
self.agent_dir = self.home / '.pi/agent'
|
|
self.agent_dir.mkdir(parents=True)
|
|
(self.root / '.pi').mkdir()
|
|
self.bodies = []
|
|
self.tool_name, self.tool_id = 'write', 'write-once'
|
|
self.tool_args = {'path': 'side-effect.txt', 'content': 'once'}
|
|
self.fail_second = False
|
|
self.delay_first_s = 0
|
|
self.always_tools = False
|
|
self.fail_on = {2}
|
|
outer = self
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
outer.bodies.append(body)
|
|
if len(outer.bodies) == 1 and outer.delay_first_s:
|
|
time.sleep(outer.delay_first_s)
|
|
if outer.fail_second and len(outer.bodies) in outer.fail_on:
|
|
payload = b'{"error":{"message":"fixture rate limit","type":"rate_limit"}}'
|
|
self.send_response(429); self.send_header('Retry-After', '0.2')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
return
|
|
delta = ({'role': 'assistant', 'tool_calls': [{'index': 0, 'id': outer.tool_id if len(outer.bodies) == 1 else outer.tool_id + '-' + str(len(outer.bodies)), 'type': 'function',
|
|
'function': {'name': outer.tool_name, 'arguments': json.dumps(outer.tool_args)}}]}
|
|
if len(outer.bodies) == 1 or outer.always_tools else {'role': 'assistant', 'content': 'done'})
|
|
finish = 'tool_calls' if len(outer.bodies) == 1 or outer.always_tools else 'stop'
|
|
chunks = [dict(id='fixture', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta=delta, finish_reason=None)]),
|
|
dict(id='fixture', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta={}, finish_reason=finish)], usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
|
|
payload = ''.join('data: ' + json.dumps(chunk) + '\n\n' for chunk in chunks).encode() + b'data: [DONE]\n\n'
|
|
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
|
|
self.addCleanup(self.server.server_close); self.addCleanup(self.server.shutdown)
|
|
self.broker = Broker(self.root / 'quota.sqlite', {'account': QuotaPolicy(1, 100, 10000000,
|
|
wait_budget_s=.05, cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0,
|
|
connect_timeout_s=1, first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.005)},
|
|
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port, 'upstream-fixture', ('fixture',))})
|
|
self.broker.start(); self.addCleanup(self.broker.close)
|
|
self.scope = self.broker.issue_scope('execution-fixture', 'teacher', 'fixture')
|
|
settings = {'defaultProvider': 'fixture-provider', 'defaultModel': 'fixture', 'compaction': {'enabled': False},
|
|
'retry': {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': self.scope.client_timeout_s * 1000}},
|
|
'extensions': [], 'skills': []}
|
|
(self.root / '.pi/settings.json').write_text(json.dumps(settings))
|
|
models = {'providers': {'fixture-provider': {'baseUrl': self.scope.base_url, 'api': 'openai-completions',
|
|
'apiKey': 'BENCH_RUNNER_API_KEY', 'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False,
|
|
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 512,
|
|
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}
|
|
(self.agent_dir / 'models.json').write_text(json.dumps(models))
|
|
(self.root / 'system.md').write_text('Use write once, then report completion.')
|
|
(self.root / 'prompt.md').write_text('Write once.')
|
|
self.config = {'execution_id': self.scope.execution_id, 'model': 'fixture', 'provider': 'fixture-provider',
|
|
'base_url': self.scope.base_url, 'client_timeout_ms': self.scope.client_timeout_s * 1000,
|
|
'session_dir': str(self.root / '.pi-sessions'), 'tools': ['write'],
|
|
'append_system_path': str(self.root / 'system.md'), 'prompt_path': str(self.root / 'prompt.md'), 'resume': False}
|
|
|
|
def run_sdk(self, timeout=25):
|
|
config = self.root / 'runner.json'; config.write_text(json.dumps(self.config))
|
|
with (self.root / 'events.jsonl').open('ab') as output:
|
|
result = subprocess.run([str(NODE), str(ROOT / 'agent/pi_session.js'), str(config)], cwd=self.root,
|
|
env={**os.environ, 'HOME': str(self.home), 'PI_CODING_AGENT_DIR': str(self.agent_dir),
|
|
'BENCH_RUNNER_API_KEY': self.scope.api_key}, stdout=output, stderr=subprocess.PIPE, timeout=timeout)
|
|
return result
|
|
|
|
def test_actual_prepare_binds_source_before_sdk_or_http(self):
|
|
from contextlib import ExitStack
|
|
from unittest.mock import patch
|
|
import yaml
|
|
from harness import collection_runtime as runtime, db, paths, rollout
|
|
from harness.execution import Execution
|
|
project = self.root / 'prepared'
|
|
job_dir = project / 'runs/r/j'
|
|
job_dir.mkdir(parents=True)
|
|
job = {'model': 'resnet18', 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq',
|
|
'qcfg': 'fp16int8', 'model_spec': {'input_spec': '1x3x224x224'},
|
|
'gates': {'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}},
|
|
'data': {'calib': {'path': str(project / 'calib'), 'n': 32},
|
|
't1': {'path': str(project / 'test.tar'), 'n': 100}}}
|
|
(job_dir / 'job.json').write_text(json.dumps(job))
|
|
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
|
|
cfg['rollouts'].update(private_home=True, tools=['write'])
|
|
cfg['endpoints']['teacher']['model'] = 'fixture'
|
|
cfg['collection_v1']['credential_groups'] = {'teacher': 'account'}
|
|
original_connect = db.connect
|
|
database = project / 'state/jobs.sqlite'
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent,
|
|
'AGENT': ROOT / 'agent', 'RUNS': project / 'runs'}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
|
|
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(database)))
|
|
stack.enter_context(patch.object(rollout, 'onnx_path', return_value=None))
|
|
stack.enter_context(patch.object(rollout, 'baseline_dir', return_value=project / 'baseline'))
|
|
stack.enter_context(patch.object(runtime, 'config', return_value=(cfg['collection_v1'], self.broker.groups)))
|
|
stack.enter_context(patch.object(runtime, 'broker', return_value=self.broker))
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO runs VALUES ('r',0,'{}','source-binding fixture')")
|
|
rd, _, _ = rollout.prepare('r', 'j', 0, 'e2e', 'teacher')
|
|
self.assertEqual(json.loads((rd / 'rollout.json').read_bytes())['runner_version'], 'pi-sdk-v1')
|
|
execution = Execution(rd)
|
|
execution.start(cfg['collection_v1']['lifecycle'])
|
|
runtime.configure_pi(rd, execution, 'teacher')
|
|
bound = json.loads((rd / 'rollout.json').read_bytes())
|
|
self.assertEqual(bound['execution_id'], execution.snapshot()['execution_id'])
|
|
self.assertFalse((rd / 'agent/events.jsonl').exists())
|
|
self.assertEqual(self.bodies, [])
|
|
for fields, reason in [({'execution_id': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
|
|
({'runner_version': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
|
|
({'execution_id': None}, 'RESUME_SOURCE_IDENTITY_MISSING')]:
|
|
raw = json.dumps({**bound, **fields}).encode()
|
|
(rd / 'rollout.json').write_bytes(raw)
|
|
with self.assertRaisesRegex(ValueError, reason):
|
|
runtime.configure_pi(rd, execution, 'teacher', resume=True)
|
|
self.assertEqual((rd / 'rollout.json').read_bytes(), raw)
|
|
(rd / 'rollout.json').write_text(json.dumps(bound))
|
|
|
|
def test_actual_sdk_uses_broker_and_persists_complete_tool_chain(self):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertEqual([row['generation_id'] for row in self.broker.execution_status(self.scope.execution_id)['generations']],
|
|
['execution-fixture:0', 'execution-fixture:1'])
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
|
|
@unittest.skipUnless(os.environ.get('BENCH_TEST_LONG_QUEUE') == '1', 'explicit 310 second regression')
|
|
def test_actual_sdk_waits_past_node_fetch_five_minute_header_limit(self):
|
|
self.broker.close()
|
|
policy = QuotaPolicy(1, 100, 10000000, first_byte_timeout_s=340,
|
|
stream_idle_timeout_s=340, request_total_s=350, wait_budget_s=340)
|
|
self.broker = Broker(self.root / 'long-quota.sqlite', {'account': policy},
|
|
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port,
|
|
'upstream-fixture', ('fixture',))}).start()
|
|
self.addCleanup(self.broker.close)
|
|
self.scope = self.broker.issue_scope('long-execution', 'teacher', 'fixture')
|
|
self.config.update(execution_id=self.scope.execution_id, base_url=self.scope.base_url,
|
|
client_timeout_ms=self.scope.client_timeout_s * 1000)
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['retry']['provider']['timeoutMs'] = self.scope.client_timeout_s * 1000
|
|
settings_path.write_text(json.dumps(settings))
|
|
models_path = self.agent_dir / 'models.json'; models = json.loads(models_path.read_text())
|
|
models['providers']['fixture-provider']['baseUrl'] = self.scope.base_url
|
|
models_path.write_text(json.dumps(models))
|
|
self.delay_first_s = 310
|
|
started = time.monotonic(); result = self.run_sdk(timeout=370)
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertGreaterEqual(time.monotonic() - started, 310)
|
|
self.assertEqual(len(self.bodies), 2)
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
|
|
def _events(self):
|
|
return [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
|
|
|
|
def test_exact_final_turn_uses_native_agent_end_and_persists(self):
|
|
self.config.update(max_turns=2, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['complete'])
|
|
self.assertFalse(checkpoint['resumable'])
|
|
self.assertEqual(checkpoint['budget']['turns'], 2)
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
self.assertTrue(any(m.get('role') == 'custom' and m.get('customType') == 'bench-budget'
|
|
for m in [e.get('message', {}) for e in self._events()]))
|
|
|
|
def test_last_turn_tool_request_is_rejected_without_extra_generation(self):
|
|
self.config.update(max_turns=2, max_tokens=100000)
|
|
self.always_tools = True
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertEqual(checkpoint['phase'], 'BUDGET_EXHAUSTED')
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(len(self.bodies), 2)
|
|
ends = [e for e in self._events() if e['type'] == 'tool_execution_end']
|
|
self.assertEqual(len(ends), 2)
|
|
self.assertTrue(ends[-1]['isError'])
|
|
self.assertIn('AGENT_BUDGET_EXHAUSTED', ends[-1]['result']['content'][0]['text'])
|
|
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
|
|
|
|
def test_one_turn_is_reserved_and_cannot_execute_work(self):
|
|
self.config.update(max_turns=1, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 1)
|
|
self.assertFalse((self.root / 'side-effect.txt').exists())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertFalse(checkpoint['complete'])
|
|
|
|
def test_token_exhaustion_closes_native_call_without_starting_work(self):
|
|
self.config.update(max_turns=10, max_tokens=1)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 1)
|
|
self.assertFalse((self.root / 'side-effect.txt').exists())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(checkpoint['budget']['reason'], 'MAX_TOKENS')
|
|
|
|
def test_answer_before_reserved_turn_does_not_trigger_extra_turn(self):
|
|
self.config.update(max_turns=3, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertFalse(any(e.get('message', {}).get('customType') == 'bench-budget' for e in self._events()))
|
|
|
|
def _check_oe_doc(self, script, is_error, expected_text):
|
|
from unittest.mock import patch
|
|
from export import trajectory
|
|
|
|
self.tool_name, self.tool_id = 'oe_doc', 'oe-doc-once'
|
|
self.tool_args = {'query': 'fixture-query'}
|
|
self.config['tools'] = ['oe_doc']
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['extensions'] = [str(ROOT / 'agent/extensions/oe-doc.ts')]
|
|
settings_path.write_text(json.dumps(settings))
|
|
if script is not None:
|
|
script_path = self.root / 'agent/oe-doc.sh'
|
|
script_path.parent.mkdir()
|
|
script_path.write_text('#!/bin/sh\n' + script)
|
|
script_path.chmod(0o700)
|
|
with patch.dict(os.environ, {'BENCH_ROOT': str(self.root)}):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 2)
|
|
events = [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
|
|
ends = [e for e in events if e['type'] == 'tool_execution_end']
|
|
messages = [e['message'] for e in events if e['type'] == 'message_end']
|
|
results = [m for m in messages if m['role'] == 'toolResult']
|
|
self.assertEqual(len(ends), 1)
|
|
self.assertEqual(len(results), 1)
|
|
for event in (ends[0], results[0]):
|
|
self.assertEqual((event['toolCallId'], event['toolName']), (self.tool_id, self.tool_name))
|
|
self.assertIs(event['isError'], is_error)
|
|
self.assertIn(expected_text, results[0]['content'][0]['text'])
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['closure'])
|
|
persisted = [json.loads(line).get('message', {})
|
|
for line in Path(checkpoint['session_path']).read_text().splitlines()]
|
|
self.assertEqual([m for m in persisted if m.get('role') == 'toolResult'], results)
|
|
|
|
# Real SDK output through the native converter; metadata makes no board/SFT-eligibility claim.
|
|
rd = self.root / 'trajectory-fixture'
|
|
(rd / 'agent').mkdir(parents=True)
|
|
header = events[0]['bench']
|
|
for name, content in {
|
|
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
|
|
'system.md': header['system_prompt'].encode(),
|
|
'tools.json': json.dumps(header['tools']).encode(),
|
|
'prompt.md': (self.root / 'prompt.md').read_bytes(),
|
|
}.items():
|
|
(rd / 'agent' / name).write_bytes(content)
|
|
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
|
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
|
|
'evidence_scope': 'fake-provider-no-board'}))
|
|
frozen = trajectory.freeze(rd)
|
|
output = trajectory.reconstruct(frozen)
|
|
# Export preserves native fields while applying its source-seeded privacy contract.
|
|
from export.redact import redact_preserving
|
|
expected, _ = redact_preserving({'messages': [e['message'] for e in events
|
|
if e.get('type') == 'message_end'], 'tools': header['tools']})
|
|
self.assertEqual([m for m in output['messages'] if m['role'] == 'toolResult'],
|
|
[m for m in expected['messages'] if m['role'] == 'toolResult'])
|
|
self.assertEqual(output['excluded'], [])
|
|
quality = trajectory.validate(frozen, output)
|
|
self.assertTrue(quality['pass'], quality)
|
|
|
|
def test_oe_doc_success_preserves_success_through_sdk_and_export(self):
|
|
self._check_oe_doc('printf "fixture docs: %s\\n" "$1"\n', False, 'fixture docs: fixture-query')
|
|
|
|
def test_oe_doc_exit_failure_preserves_error_through_sdk_and_export(self):
|
|
self._check_oe_doc('printf "fixture exit 7\\n" >&2\nexit 7\n', True, 'fixture exit 7')
|
|
|
|
def test_oe_doc_spawn_failure_preserves_error_through_sdk_and_export(self):
|
|
self._check_oe_doc(None, True, 'ENOENT')
|
|
|
|
def test_pause_continue_same_generation_without_user_or_tool_replay(self):
|
|
import time
|
|
self.fail_second = True
|
|
first = self.run_sdk()
|
|
self.assertEqual(first.returncode, 76, first.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['resumable'])
|
|
original_stat = (self.root / 'side-effect.txt').stat()
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'],
|
|
system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'],
|
|
generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True)
|
|
second = self.run_sdk()
|
|
self.assertEqual(second.returncode, 0, second.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 3)
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, original_stat.st_mtime_ns)
|
|
messages = [json.loads(line)['message'] for line in (self.root / 'events.jsonl').read_text().splitlines()
|
|
if json.loads(line).get('type') == 'message_end']
|
|
self.assertEqual(sum(m['role'] == 'user' for m in messages), 1)
|
|
self.assertEqual(sum(m['role'] == 'toolResult' for m in messages), 1)
|
|
|
|
def test_multiple_pauses_preserve_raw_errors_without_replaying_tools(self):
|
|
import time
|
|
self.fail_second, self.fail_on = True, {2, 3}
|
|
mtime = None
|
|
for expected in (76, 76, 0):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, expected, result.stderr.decode())
|
|
actual = (self.root / 'side-effect.txt').stat().st_mtime_ns
|
|
if mtime is not None: self.assertEqual(actual, mtime)
|
|
mtime = actual
|
|
if expected == 0: break
|
|
c = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=c['session_path'], session_sha256=c['sha256'],
|
|
system_prompt=header['bench']['system_prompt'], system_sha256=c['system_sha256'],
|
|
generation=c['generation'], nonempty_delta=c['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, c['generation_id'], checkpoint_complete=True, side_effects_known=True)
|
|
self.assertEqual(len(self.bodies), 4)
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual(self.bodies[2], self.bodies[3])
|
|
records = [json.loads(line) for line in Path(c['session_path']).read_text().splitlines()]
|
|
self.assertEqual(sum(r.get('message', {}).get('stopReason') == 'error' for r in records), 2)
|
|
|
|
def test_budget_custom_pause_resume_preserves_context_and_export(self):
|
|
"""A real production extension's custom message survives safe continuation."""
|
|
import time
|
|
from unittest.mock import patch
|
|
from export import trajectory
|
|
|
|
self.fail_second = True
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['extensions'] = [str(ROOT / 'agent/extensions/budget-kill.ts')]
|
|
settings_path.write_text(json.dumps(settings))
|
|
budget = {'BENCH_MAX_TURNS': '2', 'BENCH_MAX_TOKENS': '1000000',
|
|
'BENCH_WARN_AHEAD_TURNS': '5', 'BENCH_ROLLOUT_DIR': '',
|
|
'BENCH_INITIAL_TURNS': '0', 'BENCH_INITIAL_TOKENS': '0'}
|
|
with patch.dict(os.environ, budget):
|
|
first = self.run_sdk()
|
|
self.assertEqual(first.returncode, 76, first.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['resumable'])
|
|
mtime = (self.root / 'side-effect.txt').stat().st_mtime_ns
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=checkpoint['session_path'],
|
|
session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'],
|
|
system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'],
|
|
nonempty_delta=checkpoint['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'],
|
|
checkpoint_complete=True, side_effects_known=True)
|
|
with patch.dict(os.environ, {**budget, 'BENCH_INITIAL_TURNS': '1', 'BENCH_INITIAL_TOKENS': '15'}):
|
|
second = self.run_sdk()
|
|
self.assertEqual(second.returncode, 0, second.stderr.decode())
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, mtime)
|
|
|
|
# Actual SDK events; only source metadata is a fixture, with no board claim.
|
|
rd = self.root / 'trajectory-fixture'
|
|
(rd / 'agent').mkdir(parents=True)
|
|
for name, content in {
|
|
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
|
|
'system.md': header['bench']['system_prompt'].encode(),
|
|
'tools.json': json.dumps(header['bench']['tools']).encode(),
|
|
'prompt.md': (self.root / 'prompt.md').read_bytes(),
|
|
}.items():
|
|
(rd / 'agent' / name).write_bytes(content)
|
|
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
|
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
|
|
'evidence_scope': 'fake-provider-no-board'}))
|
|
frozen = trajectory.freeze(rd)
|
|
output = trajectory.reconstruct(frozen)
|
|
quality = trajectory.validate(frozen, output)
|
|
self.assertTrue(quality['pass'], quality)
|
|
self.assertTrue(any(message['role'] == 'custom' and message.get('customType') == 'bench-budget'
|
|
for message in output['messages']))
|
|
|
|
def test_harness_pause_resume_keeps_execution_budget_and_attempt(self):
|
|
"""Exercise actual Python -> broker -> SDK -> tool -> durable pause wiring."""
|
|
from contextlib import ExitStack
|
|
from dataclasses import asdict
|
|
import shutil
|
|
import time
|
|
from unittest.mock import patch
|
|
import yaml
|
|
from harness import collection_runtime as runtime, db, paths, rollout
|
|
self.fail_second = True
|
|
project = self.root / 'project'
|
|
rd = project / 'runs/r/j/rollouts/00'
|
|
workspace = rd / 'workspace'
|
|
workspace.mkdir(parents=True)
|
|
(rd / 'agent').mkdir()
|
|
shutil.move(str(self.root / '.pi'), workspace / '.pi')
|
|
shutil.move(str(self.home), workspace / '.agent-home')
|
|
models_path = workspace / '.agent-home/.pi/agent/models.json'
|
|
models = json.loads(models_path.read_bytes())
|
|
models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider')
|
|
models_path.write_text(json.dumps(models))
|
|
settings_path = workspace / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_bytes()); settings['defaultProvider'] = 'dr-gateway'
|
|
settings_path.write_text(json.dumps(settings))
|
|
for name in ('system.md', 'append-system.md', 'prompt.md'):
|
|
(rd / 'agent' / name).write_text('Write once, then report done.')
|
|
from harness.runners import pi
|
|
metadata = {'state': 'RUNNING', 'runner': 'pi', 'runner_version': pi.VERSION, 'policy': 'teacher', 'mode': 'e2e', 'jail_policy': 'private-home',
|
|
'max_turns': 60, 'max_tokens': 200000}
|
|
(rd / 'rollout.json').write_text(json.dumps(metadata))
|
|
(rd.parent.parent / 'job.json').write_text('{}')
|
|
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
|
|
cfg['rollouts'].update(jail=False, private_home=True, tools=['write'])
|
|
cfg['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % self.server.server_port,
|
|
model='fixture', api_key_env='FIXTURE_KEY')
|
|
cfg['collection_v1'].update(quota_ledger=str(project / 'runtime-quota.sqlite'),
|
|
quota_groups={'account': asdict(self.broker.groups['account'])}, credential_groups={'teacher': 'account'})
|
|
state = project / 'state'; state.mkdir()
|
|
config_path = project / 'config.yaml'; config_path.write_text(yaml.safe_dump(cfg))
|
|
original_connect, original_readonly = db.connect, db.readonly
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
|
|
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': config_path}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
|
|
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
|
|
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(state / 'jobs.sqlite')))
|
|
stack.enter_context(patch.object(db, 'readonly', side_effect=lambda: original_readonly(state / 'jobs.sqlite')))
|
|
stack.enter_context(patch.object(runtime, '_broker', None))
|
|
stack.enter_context(patch.object(runtime, '_configuration', None))
|
|
stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, {}, 'Write once.')))
|
|
# Fake provider proves process/protocol wiring only, never board PASS.
|
|
stack.enter_context(patch.object(rollout, 'verify_rollout', return_value={'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False}))
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO runs VALUES ('r',0,'{}','fixture')")
|
|
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
|
|
try:
|
|
first = rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False)
|
|
self.assertEqual(first['terminal'], 'PAUSED_INFRA')
|
|
before = json.loads((rd / 'execution.json').read_bytes())
|
|
bound = json.loads((rd / 'rollout.json').read_bytes())
|
|
self.assertEqual(bound['runner_version'], 'pi-sdk-v1')
|
|
self.assertEqual(bound['execution_id'], before['execution_id'])
|
|
self.assertFalse(runtime.broker().execution_status(before['execution_id'])['cancelled'])
|
|
from export import trajectory
|
|
frozen = trajectory.freeze(rd)
|
|
self.assertEqual(trajectory.reconstruct(frozen)['context']['status'], 'COMPLETE')
|
|
stat = (workspace / 'side-effect.txt').stat()
|
|
time.sleep(.25)
|
|
second = rollout.resume_rollout('r', 'j', 0)
|
|
after = json.loads((rd / 'execution.json').read_bytes())
|
|
self.assertEqual(second['terminal'], 'UNVERIFIED')
|
|
self.assertEqual(after['execution_outcome'], 'COMPLETED')
|
|
self.assertTrue(runtime.broker().execution_status(after['execution_id'])['cancelled'])
|
|
self.assertEqual((after['execution_id'], after['deadline_monotonic']),
|
|
(before['execution_id'], before['deadline_monotonic']))
|
|
self.assertEqual(after['epoch'], before['epoch'] + 1)
|
|
self.assertEqual(stat.st_mtime_ns, (workspace / 'side-effect.txt').stat().st_mtime_ns)
|
|
self.assertEqual(runtime.budget_used(rd)['turns'], 2)
|
|
self.assertEqual(len(list(rd.parent.iterdir())), 1)
|
|
finally:
|
|
if runtime._broker:
|
|
runtime._broker.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|