- broker: a context-window 400 fails its generation (fences nothing) and goes back to Pi as context_length_exceeded, so Pi compacts and resends - runner: new generation ID after compaction, wait for Pi's resumed run, record kept_messages; drop agent/context-mask.js and context_mask_chars - models.json: contextWindow large so Pi compacts only on a real overflow, never on an estimate right before a pause (resume would not match) - export: compactions[] gives what the model actually saw after each one - prompt: keep facts and acceptance, drop how-to rules and coaching (SYSTEM.md 89->64 lines, e2e.md 27->20); stage status == stage wait Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
602 lines
38 KiB
Python
602 lines
38 KiB
Python
"""Real installed Pi SDK against a local fake provider; no account or board calls."""
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
|
|
from harness.quota import Broker, QuotaPolicy, Upstream
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
NODE = Path(os.environ.get('BENCH_TEST_NODE', '/home/ming01.wang/toolchains/node/bin/node'))
|
|
|
|
|
|
@unittest.skipUnless(NODE.is_file(), 'installed Pi 0.73.1 SDK required')
|
|
class PiSDKTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.home = self.root / 'home'
|
|
self.agent_dir = self.home / '.pi/agent'
|
|
self.agent_dir.mkdir(parents=True)
|
|
(self.root / '.pi').mkdir()
|
|
self.bodies = []
|
|
self.tool_name, self.tool_id = 'write', 'write-once'
|
|
self.tool_args = {'path': 'side-effect.txt', 'content': 'once'}
|
|
self.fail_second = False
|
|
self.delay_first_s = 0
|
|
self.always_tools = False
|
|
self.fail_on = {2}
|
|
self.overflow_on = set()
|
|
outer = self
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
outer.bodies.append(body)
|
|
if len(outer.bodies) == 1 and outer.delay_first_s:
|
|
time.sleep(outer.delay_first_s)
|
|
if len(outer.bodies) in outer.overflow_on:
|
|
payload = b'{"object":"error","message":"This model\'s maximum context length is 262144 tokens."}'
|
|
self.send_response(400); self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
return
|
|
if outer.fail_second and len(outer.bodies) in outer.fail_on:
|
|
payload = b'{"error":{"message":"fixture rate limit","type":"rate_limit"}}'
|
|
self.send_response(429); self.send_header('Retry-After', '0.2')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
return
|
|
delta = ({'role': 'assistant', 'tool_calls': [{'index': 0, 'id': outer.tool_id if len(outer.bodies) == 1 else outer.tool_id + '-' + str(len(outer.bodies)), 'type': 'function',
|
|
'function': {'name': outer.tool_name, 'arguments': json.dumps(outer.tool_args)}}]}
|
|
if len(outer.bodies) == 1 or outer.always_tools else {'role': 'assistant', 'content': 'done'})
|
|
finish = 'tool_calls' if len(outer.bodies) == 1 or outer.always_tools else 'stop'
|
|
chunks = [dict(id='fixture', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta=delta, finish_reason=None)]),
|
|
dict(id='fixture', object='chat.completion.chunk', model='fixture',
|
|
choices=[dict(index=0, delta={}, finish_reason=finish)], usage={'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15})]
|
|
payload = ''.join('data: ' + json.dumps(chunk) + '\n\n' for chunk in chunks).encode() + b'data: [DONE]\n\n'
|
|
self.send_response(200); self.send_header('Content-Type', 'text/event-stream')
|
|
self.send_header('Content-Length', str(len(payload))); self.end_headers(); self.wfile.write(payload)
|
|
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
|
|
self.addCleanup(self.server.server_close); self.addCleanup(self.server.shutdown)
|
|
self.broker = Broker(self.root / 'quota.sqlite', {'account': QuotaPolicy(1, 100, 10000000,
|
|
wait_budget_s=.05, cooldown_initial_s=.05, cooldown_local_max_s=.1, jitter_fraction=0,
|
|
connect_timeout_s=1, first_byte_timeout_s=2, stream_idle_timeout_s=2, request_total_s=5, poll_s=.005)},
|
|
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port, 'upstream-fixture', ('fixture',))})
|
|
self.broker.start(); self.addCleanup(self.broker.close)
|
|
self.scope = self.broker.issue_scope('execution-fixture', 'teacher', 'fixture')
|
|
settings = {'defaultProvider': 'fixture-provider', 'defaultModel': 'fixture', 'compaction': {'enabled': False},
|
|
'retry': {'enabled': False, 'provider': {'maxRetries': 0, 'timeoutMs': self.scope.client_timeout_s * 1000}},
|
|
'extensions': [], 'skills': []}
|
|
(self.root / '.pi/settings.json').write_text(json.dumps(settings))
|
|
models = {'providers': {'fixture-provider': {'baseUrl': self.scope.base_url, 'api': 'openai-completions',
|
|
'apiKey': 'BENCH_RUNNER_API_KEY', 'models': [{'id': 'fixture', 'name': 'fixture', 'reasoning': False,
|
|
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 512,
|
|
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}}
|
|
(self.agent_dir / 'models.json').write_text(json.dumps(models))
|
|
(self.root / 'system.md').write_text('Use write once, then report completion.')
|
|
(self.root / 'prompt.md').write_text('Write once.')
|
|
self.config = {'execution_id': self.scope.execution_id, 'model': 'fixture', 'provider': 'fixture-provider',
|
|
'base_url': self.scope.base_url, 'client_timeout_ms': self.scope.client_timeout_s * 1000,
|
|
'session_dir': str(self.root / '.pi-sessions'), 'tools': ['write'],
|
|
'append_system_path': str(self.root / 'system.md'), 'prompt_path': str(self.root / 'prompt.md'), 'resume': False}
|
|
|
|
def run_sdk(self, timeout=25):
|
|
config = self.root / 'runner.json'; config.write_text(json.dumps(self.config))
|
|
with (self.root / 'events.jsonl').open('ab') as output:
|
|
result = subprocess.run([str(NODE), str(ROOT / 'agent/pi_session.js'), str(config)], cwd=self.root,
|
|
env={**os.environ, 'HOME': str(self.home), 'PI_CODING_AGENT_DIR': str(self.agent_dir),
|
|
'BENCH_RUNNER_API_KEY': self.scope.api_key}, stdout=output, stderr=subprocess.PIPE, timeout=timeout)
|
|
return result
|
|
|
|
def test_broker_connection_failure_has_sanitized_transport_identity(self):
|
|
self.broker.close()
|
|
result = self.run_sdk()
|
|
records = [json.loads(line) for line in result.stderr.decode().splitlines() if line.startswith('{')]
|
|
errors = [r for r in records if r.get('type') == 'bench_transport_error']
|
|
self.assertTrue(errors, result.stderr)
|
|
for error in errors:
|
|
self.assertEqual(error['execution_id'], self.scope.execution_id)
|
|
self.assertEqual(error['generation_id'], self.scope.execution_id + ':0')
|
|
self.assertIn('ECONNREFUSED', (error['code'], error['cause_code']))
|
|
self.assertGreaterEqual(error['elapsed_ms'], 0)
|
|
self.assertEqual(set(error), {'type', 'execution_id', 'generation_id', 'name', 'code', 'cause_code', 'elapsed_ms'})
|
|
self.assertNotIn(self.scope.api_key.encode(), result.stderr)
|
|
self.assertEqual(self.bodies, [])
|
|
|
|
def test_actual_prepare_binds_source_before_sdk_or_http(self):
|
|
from contextlib import ExitStack
|
|
from unittest.mock import patch
|
|
import yaml
|
|
from harness import collection_runtime as runtime, db, paths, rollout
|
|
from harness.execution import Execution
|
|
project = self.root / 'prepared'
|
|
job_dir = project / 'runs/r/j'
|
|
job_dir.mkdir(parents=True)
|
|
job = {'model': 'resnet18', 'task': 'cls', 'march': 'nash-p', 'backend': 'ptq',
|
|
'qcfg': 'fp16int8', 'model_spec': {'input_spec': '1x3x224x224'},
|
|
'gates': {'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}},
|
|
'data': {'calib': {'path': str(project / 'calib'), 'n': 32},
|
|
't1': {'path': str(project / 'test.tar'), 'n': 100}}}
|
|
(job_dir / 'job.json').write_text(json.dumps(job))
|
|
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
|
|
cfg['rollouts'].update(private_home=True, tools=['write'])
|
|
cfg['endpoints']['teacher']['model'] = 'fixture'
|
|
cfg['collection_v1']['credential_groups'] = {'teacher': 'account'}
|
|
original_connect = db.connect
|
|
database = project / 'state/jobs.sqlite'
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent,
|
|
'AGENT': ROOT / 'agent', 'RUNS': project / 'runs'}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
|
|
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(database)))
|
|
stack.enter_context(patch.object(rollout, 'onnx_path', return_value=None))
|
|
stack.enter_context(patch.object(rollout, 'baseline_dir', return_value=project / 'baseline'))
|
|
stack.enter_context(patch.object(runtime, 'config', return_value=(cfg['collection_v1'], self.broker.groups)))
|
|
stack.enter_context(patch.object(runtime, 'broker', return_value=self.broker))
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO runs VALUES ('r',0,'{}','source-binding fixture')")
|
|
rd, _, _ = rollout.prepare('r', 'j', 0, 'e2e', 'teacher')
|
|
self.assertEqual(json.loads((rd / 'rollout.json').read_bytes())['runner_version'], 'pi-sdk-v1')
|
|
settings = json.loads((rd / 'workspace/.pi/settings.json').read_text())
|
|
self.assertIn('terminal_exec', settings['tools'])
|
|
self.assertTrue(any(value.endswith('/terminal.ts') for value in settings['extensions']))
|
|
definitions = json.loads((rd / 'agent/tools.json').read_text())
|
|
terminal = next(row['function'] for row in definitions if row['function']['name'] == 'terminal_exec')
|
|
self.assertEqual(set(terminal['parameters']['properties']), {'wait_handle'})
|
|
self.assertNotIn('natural_agent_version', json.loads((rd / 'rollout.json').read_text()))
|
|
execution = Execution(rd)
|
|
execution.start(cfg['collection_v1']['lifecycle'])
|
|
runtime.configure_pi(rd, execution, 'teacher')
|
|
bound = json.loads((rd / 'rollout.json').read_bytes())
|
|
self.assertEqual(bound['execution_id'], execution.snapshot()['execution_id'])
|
|
self.assertFalse((rd / 'agent/events.jsonl').exists())
|
|
self.assertEqual(self.bodies, [])
|
|
for fields, reason in [({'execution_id': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
|
|
({'runner_version': 'different'}, 'ROLLOUT_EXECUTION_MISMATCH'),
|
|
({'execution_id': None}, 'RESUME_SOURCE_IDENTITY_MISSING')]:
|
|
raw = json.dumps({**bound, **fields}).encode()
|
|
(rd / 'rollout.json').write_bytes(raw)
|
|
with self.assertRaisesRegex(ValueError, reason):
|
|
runtime.configure_pi(rd, execution, 'teacher', resume=True)
|
|
self.assertEqual((rd / 'rollout.json').read_bytes(), raw)
|
|
(rd / 'rollout.json').write_text(json.dumps(bound))
|
|
|
|
def test_actual_sdk_uses_broker_and_persists_complete_tool_chain(self):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertEqual([row['generation_id'] for row in self.broker.execution_status(self.scope.execution_id)['generations']],
|
|
['execution-fixture:0', 'execution-fixture:1'])
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
updates = [e for e in self._events() if e['type'] == 'message_update']
|
|
self.assertTrue(updates) # streamed, but without the per-token copies of the whole message
|
|
self.assertFalse(any('message' in e or 'partial' in e['assistantMessageEvent'] for e in updates))
|
|
from harness.collection_runtime import budget_used
|
|
(self.root / 'agent').mkdir(exist_ok=True)
|
|
(self.root / 'agent/events.jsonl').write_bytes((self.root / 'events.jsonl').read_bytes())
|
|
self.assertEqual(budget_used(self.root)['turns'], sum(e['type'] == 'turn_end' for e in self._events()))
|
|
|
|
@unittest.skipUnless(os.environ.get('BENCH_TEST_LONG_QUEUE') == '1', 'explicit 310 second regression')
|
|
def test_actual_sdk_waits_past_node_fetch_five_minute_header_limit(self):
|
|
self.broker.close()
|
|
policy = QuotaPolicy(1, 100, 10000000, first_byte_timeout_s=340,
|
|
stream_idle_timeout_s=340, request_total_s=350, wait_budget_s=340)
|
|
self.broker = Broker(self.root / 'long-quota.sqlite', {'account': policy},
|
|
{'teacher': Upstream('account', 'http://127.0.0.1:%d/v1' % self.server.server_port,
|
|
'upstream-fixture', ('fixture',))}).start()
|
|
self.addCleanup(self.broker.close)
|
|
self.scope = self.broker.issue_scope('long-execution', 'teacher', 'fixture')
|
|
self.config.update(execution_id=self.scope.execution_id, base_url=self.scope.base_url,
|
|
client_timeout_ms=self.scope.client_timeout_s * 1000)
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['retry']['provider']['timeoutMs'] = self.scope.client_timeout_s * 1000
|
|
settings_path.write_text(json.dumps(settings))
|
|
models_path = self.agent_dir / 'models.json'; models = json.loads(models_path.read_text())
|
|
models['providers']['fixture-provider']['baseUrl'] = self.scope.base_url
|
|
models_path.write_text(json.dumps(models))
|
|
self.delay_first_s = 310
|
|
started = time.monotonic(); result = self.run_sdk(timeout=370)
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertGreaterEqual(time.monotonic() - started, 310)
|
|
self.assertEqual(len(self.bodies), 2)
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
|
|
def test_bash_without_timeout_fails_alone_before_system_cap(self):
|
|
self.tool_name, self.tool_args = 'bash', {'command': 'sleep 5; echo late'}
|
|
self.config.update(tools=['bash'], ordinary_tool_deadline_s=62)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(json.loads((self.root / '.bench-session.json').read_bytes())['phase'], 'COMPLETED')
|
|
messages = self.bodies[1]['messages']
|
|
self.assertEqual(messages[-1]['content'], 'Command timed out after 1 seconds')
|
|
call = next(m for m in messages if m.get('tool_calls'))['tool_calls'][0]
|
|
self.assertEqual(json.loads(call['function']['arguments']), self.tool_args)
|
|
|
|
def _events(self):
|
|
return [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
|
|
|
|
def test_exact_final_turn_uses_native_agent_end_and_persists(self):
|
|
self.config.update(max_turns=2, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['complete'])
|
|
self.assertFalse(checkpoint['resumable'])
|
|
self.assertEqual(checkpoint['budget']['turns'], 2)
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
|
|
self.assertEqual((self.root / 'side-effect.txt').read_text(), 'once')
|
|
self.assertTrue(any(m.get('role') == 'custom' and m.get('customType') == 'bench-budget'
|
|
for m in [e.get('message', {}) for e in self._events()]))
|
|
|
|
def test_last_turn_tool_request_is_rejected_without_extra_generation(self):
|
|
self.config.update(max_turns=2, max_tokens=100000)
|
|
self.always_tools = True
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertEqual(checkpoint['phase'], 'BUDGET_EXHAUSTED')
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(len(self.bodies), 2)
|
|
ends = [e for e in self._events() if e['type'] == 'tool_execution_end']
|
|
self.assertEqual(len(ends), 2)
|
|
self.assertTrue(ends[-1]['isError'])
|
|
self.assertIn('AGENT_BUDGET_EXHAUSTED', ends[-1]['result']['content'][0]['text'])
|
|
self.assertEqual(sum(e['type'] == 'agent_end' for e in self._events()), 1)
|
|
|
|
def test_one_turn_is_reserved_and_cannot_execute_work(self):
|
|
self.config.update(max_turns=1, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 1)
|
|
self.assertFalse((self.root / 'side-effect.txt').exists())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertFalse(checkpoint['complete'])
|
|
|
|
def test_context_overflow_is_compacted_by_pi_and_the_rollout_continues(self):
|
|
# A full window: the broker hands the 400 back, Pi summarizes and resends under a new ID, the runner waits.
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['compaction'] = {'enabled': True, 'keepRecentTokens': 1}
|
|
settings_path.write_text(json.dumps(settings))
|
|
self.overflow_on = {2}
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(json.loads((self.root / '.bench-session.json').read_bytes())['phase'], 'COMPLETED')
|
|
self.assertEqual(len(self.bodies), 4) # tool call, overflow, summary, resent turn
|
|
resent = json.dumps(self.bodies[3]['messages'])
|
|
self.assertIn('compacted into the following summary', resent)
|
|
self.assertNotIn('Write once.', resent)
|
|
rows = self.broker.execution_status(self.scope.execution_id)['generations']
|
|
self.assertEqual([(r['generation_id'], r['state'], r['reason']) for r in rows if ':g' not in r['generation_id']],
|
|
[('execution-fixture:0', 'COMPLETE', None), ('execution-fixture:1', 'FAILED', 'CONTEXT_TOO_LONG'),
|
|
('execution-fixture:2', 'COMPLETE', None)])
|
|
[end] = [e for e in self._events() if e['type'] == 'compaction_end']
|
|
kept = len(self.bodies[3]['messages']) - 2 # what Pi resent besides the system prompt and the summary
|
|
self.assertEqual((end['willRetry'], end['bench_compaction']), (True, {'kept_messages': kept}))
|
|
|
|
from export import trajectory
|
|
rd = self.root / 'trajectory-fixture'
|
|
(rd / 'agent').mkdir(parents=True)
|
|
header = self._events()[0]['bench']
|
|
(rd / 'agent/events.jsonl').write_bytes((self.root / 'events.jsonl').read_bytes())
|
|
(rd / 'agent/system.md').write_text(header['system_prompt'])
|
|
(rd / 'agent/tools.json').write_text(json.dumps(header['tools']))
|
|
(rd / 'rollout.json').write_text(json.dumps({'runner_version': 'pi-sdk-v1', 'state': 'DONE',
|
|
'execution_id': header['execution_id']}))
|
|
frozen = trajectory.freeze(rd)
|
|
output = trajectory.reconstruct(frozen)
|
|
self.assertEqual(output['errors'], [])
|
|
self.assertTrue(trajectory.validate(frozen, output)['pass'])
|
|
roles = [m['role'] for m in output['messages']]
|
|
self.assertEqual(roles, ['system', 'user', 'assistant', 'toolResult', 'assistant'])
|
|
[view] = output['compactions']
|
|
self.assertEqual((view['at'], view['kept']), (4, kept))
|
|
seen = [output['messages'][0], view['summary_message']] + output['messages'][view['at'] - view['kept']:]
|
|
self.assertEqual(len(seen), len(self.bodies[3]['messages']) + 1) # plus the final answer
|
|
self.assertEqual(seen[1]['content'][0]['text'], self.bodies[3]['messages'][1]['content'][0]['text']
|
|
if isinstance(self.bodies[3]['messages'][1]['content'], list) else self.bodies[3]['messages'][1]['content'])
|
|
|
|
def test_token_exhaustion_closes_native_call_without_starting_work(self):
|
|
self.config.update(max_turns=10, max_tokens=1)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 75, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 1)
|
|
self.assertFalse((self.root / 'side-effect.txt').exists())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_text())
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(checkpoint['budget']['reason'], 'MAX_TOKENS')
|
|
|
|
def test_answer_before_reserved_turn_does_not_trigger_extra_turn(self):
|
|
self.config.update(max_turns=3, max_tokens=100000)
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 2)
|
|
self.assertFalse(any(e.get('message', {}).get('customType') == 'bench-budget' for e in self._events()))
|
|
|
|
def _check_oe_doc(self, script, is_error, expected_text):
|
|
from unittest.mock import patch
|
|
from export import trajectory
|
|
|
|
self.tool_name, self.tool_id = 'oe_doc', 'oe-doc-once'
|
|
self.tool_args = {'query': 'fixture-query'}
|
|
self.config['tools'] = ['oe_doc']
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['extensions'] = [str(ROOT / 'agent/extensions/oe-doc.ts')]
|
|
settings_path.write_text(json.dumps(settings))
|
|
if script is not None:
|
|
script_path = self.root / 'agent/oe-doc.sh'
|
|
script_path.parent.mkdir()
|
|
script_path.write_text('#!/bin/sh\n' + script)
|
|
script_path.chmod(0o700)
|
|
with patch.dict(os.environ, {'BENCH_ROOT': str(self.root)}):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 2)
|
|
events = [json.loads(line) for line in (self.root / 'events.jsonl').read_text().splitlines()]
|
|
ends = [e for e in events if e['type'] == 'tool_execution_end']
|
|
messages = [e['message'] for e in events if e['type'] == 'message_end']
|
|
results = [m for m in messages if m['role'] == 'toolResult']
|
|
self.assertEqual(len(ends), 1)
|
|
self.assertEqual(len(results), 1)
|
|
for event in (ends[0], results[0]):
|
|
self.assertEqual((event['toolCallId'], event['toolName']), (self.tool_id, self.tool_name))
|
|
self.assertIs(event['isError'], is_error)
|
|
self.assertIn(expected_text, results[0]['content'][0]['text'])
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['closure'])
|
|
persisted = [json.loads(line).get('message', {})
|
|
for line in Path(checkpoint['session_path']).read_text().splitlines()]
|
|
self.assertEqual([m for m in persisted if m.get('role') == 'toolResult'], results)
|
|
|
|
# Real SDK output through the native converter; metadata makes no board/SFT-eligibility claim.
|
|
rd = self.root / 'trajectory-fixture'
|
|
(rd / 'agent').mkdir(parents=True)
|
|
header = events[0]['bench']
|
|
for name, content in {
|
|
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
|
|
'system.md': header['system_prompt'].encode(),
|
|
'tools.json': json.dumps(header['tools']).encode(),
|
|
'prompt.md': (self.root / 'prompt.md').read_bytes(),
|
|
}.items():
|
|
(rd / 'agent' / name).write_bytes(content)
|
|
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
|
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
|
|
'evidence_scope': 'fake-provider-no-board'}))
|
|
frozen = trajectory.freeze(rd)
|
|
output = trajectory.reconstruct(frozen)
|
|
# Export preserves native fields while applying its source-seeded privacy contract.
|
|
from export.redact import redact_preserving
|
|
expected, _ = redact_preserving({'messages': [e['message'] for e in events
|
|
if e.get('type') == 'message_end'], 'tools': header['tools']})
|
|
self.assertEqual([m for m in output['messages'] if m['role'] == 'toolResult'],
|
|
[m for m in expected['messages'] if m['role'] == 'toolResult'])
|
|
self.assertEqual(output['excluded'], [])
|
|
quality = trajectory.validate(frozen, output)
|
|
self.assertTrue(quality['pass'], quality)
|
|
|
|
def test_oe_doc_success_preserves_success_through_sdk_and_export(self):
|
|
self._check_oe_doc('printf "fixture docs: %s\\n" "$1"\n', False, 'fixture docs: fixture-query')
|
|
|
|
def test_oe_doc_exit_failure_preserves_error_through_sdk_and_export(self):
|
|
self._check_oe_doc('printf "fixture exit 7\\n" >&2\nexit 7\n', True, 'fixture exit 7')
|
|
|
|
def test_oe_doc_spawn_failure_preserves_error_through_sdk_and_export(self):
|
|
self._check_oe_doc(None, True, 'ENOENT')
|
|
|
|
def test_pause_continue_same_generation_without_user_or_tool_replay(self):
|
|
import time
|
|
self.fail_second = True
|
|
first = self.run_sdk()
|
|
self.assertEqual(first.returncode, 76, first.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['resumable'])
|
|
original_stat = (self.root / 'side-effect.txt').stat()
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=checkpoint['session_path'], session_sha256=checkpoint['sha256'],
|
|
system_prompt=header['bench']['system_prompt'], system_sha256=checkpoint['system_sha256'],
|
|
generation=checkpoint['generation'], nonempty_delta=checkpoint['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'], checkpoint_complete=True, side_effects_known=True)
|
|
second = self.run_sdk()
|
|
self.assertEqual(second.returncode, 0, second.stderr.decode())
|
|
self.assertEqual(len(self.bodies), 3)
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, original_stat.st_mtime_ns)
|
|
messages = [json.loads(line)['message'] for line in (self.root / 'events.jsonl').read_text().splitlines()
|
|
if json.loads(line).get('type') == 'message_end']
|
|
self.assertEqual(sum(m['role'] == 'user' for m in messages), 1)
|
|
self.assertEqual(sum(m['role'] == 'toolResult' for m in messages), 1)
|
|
|
|
def test_multiple_pauses_preserve_raw_errors_without_replaying_tools(self):
|
|
import time
|
|
self.fail_second, self.fail_on = True, {2, 3}
|
|
mtime = None
|
|
for expected in (76, 76, 0):
|
|
result = self.run_sdk()
|
|
self.assertEqual(result.returncode, expected, result.stderr.decode())
|
|
actual = (self.root / 'side-effect.txt').stat().st_mtime_ns
|
|
if mtime is not None: self.assertEqual(actual, mtime)
|
|
mtime = actual
|
|
if expected == 0: break
|
|
c = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=c['session_path'], session_sha256=c['sha256'],
|
|
system_prompt=header['bench']['system_prompt'], system_sha256=c['system_sha256'],
|
|
generation=c['generation'], nonempty_delta=c['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, c['generation_id'], checkpoint_complete=True, side_effects_known=True)
|
|
self.assertEqual(len(self.bodies), 4)
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual(self.bodies[2], self.bodies[3])
|
|
records = [json.loads(line) for line in Path(c['session_path']).read_text().splitlines()]
|
|
self.assertEqual(sum(r.get('message', {}).get('stopReason') == 'error' for r in records), 2)
|
|
|
|
def test_budget_custom_pause_resume_preserves_context_and_export(self):
|
|
"""A real production extension's custom message survives safe continuation."""
|
|
import time
|
|
from unittest.mock import patch
|
|
from export import trajectory
|
|
|
|
self.fail_second = True
|
|
settings_path = self.root / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_text())
|
|
settings['extensions'] = [str(ROOT / 'agent/extensions/budget-kill.ts')]
|
|
settings_path.write_text(json.dumps(settings))
|
|
budget = {'BENCH_MAX_TURNS': '2', 'BENCH_MAX_TOKENS': '1000000',
|
|
'BENCH_WARN_AHEAD_TURNS': '5', 'BENCH_ROLLOUT_DIR': '',
|
|
'BENCH_INITIAL_TURNS': '0', 'BENCH_INITIAL_TOKENS': '0'}
|
|
with patch.dict(os.environ, budget):
|
|
first = self.run_sdk()
|
|
self.assertEqual(first.returncode, 76, first.stderr.decode())
|
|
checkpoint = json.loads((self.root / '.bench-session.json').read_bytes())
|
|
self.assertFalse(checkpoint['complete'])
|
|
self.assertTrue(checkpoint['resumable'])
|
|
mtime = (self.root / 'side-effect.txt').stat().st_mtime_ns
|
|
header = json.loads((self.root / 'events.jsonl').read_text().splitlines()[0])
|
|
self.config.update(resume=True, session_path=checkpoint['session_path'],
|
|
session_sha256=checkpoint['sha256'], system_prompt=header['bench']['system_prompt'],
|
|
system_sha256=checkpoint['system_sha256'], generation=checkpoint['generation'],
|
|
nonempty_delta=checkpoint['nonempty_delta'])
|
|
time.sleep(.25)
|
|
self.broker.resume(self.scope.execution_id, checkpoint['generation_id'],
|
|
checkpoint_complete=True, side_effects_known=True)
|
|
with patch.dict(os.environ, {**budget, 'BENCH_INITIAL_TURNS': '1', 'BENCH_INITIAL_TOKENS': '15'}):
|
|
second = self.run_sdk()
|
|
self.assertEqual(second.returncode, 0, second.stderr.decode())
|
|
self.assertEqual(self.bodies[1], self.bodies[2])
|
|
self.assertEqual((self.root / 'side-effect.txt').stat().st_mtime_ns, mtime)
|
|
|
|
# Actual SDK events; only source metadata is a fixture, with no board claim.
|
|
rd = self.root / 'trajectory-fixture'
|
|
(rd / 'agent').mkdir(parents=True)
|
|
for name, content in {
|
|
'events.jsonl': (self.root / 'events.jsonl').read_bytes(),
|
|
'system.md': header['bench']['system_prompt'].encode(),
|
|
'tools.json': json.dumps(header['bench']['tools']).encode(),
|
|
'prompt.md': (self.root / 'prompt.md').read_bytes(),
|
|
}.items():
|
|
(rd / 'agent' / name).write_bytes(content)
|
|
(rd / 'rollout.json').write_text(json.dumps({'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
|
'execution_id': self.scope.execution_id, 'state': 'COMPLETED',
|
|
'evidence_scope': 'fake-provider-no-board'}))
|
|
frozen = trajectory.freeze(rd)
|
|
output = trajectory.reconstruct(frozen)
|
|
quality = trajectory.validate(frozen, output)
|
|
self.assertTrue(quality['pass'], quality)
|
|
self.assertTrue(any(message['role'] == 'custom' and message.get('customType') == 'bench-budget'
|
|
for message in output['messages']))
|
|
|
|
def test_harness_pause_resume_keeps_execution_budget_and_attempt(self):
|
|
"""Exercise actual Python -> broker -> SDK -> tool -> durable pause wiring."""
|
|
from contextlib import ExitStack
|
|
from dataclasses import asdict
|
|
import shutil
|
|
import time
|
|
from unittest.mock import patch
|
|
import yaml
|
|
from harness import collection_runtime as runtime, db, paths, rollout
|
|
self.fail_second = True
|
|
project = self.root / 'project'
|
|
rd = project / 'runs/r/j/rollouts/00'
|
|
workspace = rd / 'workspace'
|
|
workspace.mkdir(parents=True)
|
|
(rd / 'agent').mkdir()
|
|
shutil.move(str(self.root / '.pi'), workspace / '.pi')
|
|
shutil.move(str(self.home), workspace / '.agent-home')
|
|
models_path = workspace / '.agent-home/.pi/agent/models.json'
|
|
models = json.loads(models_path.read_bytes())
|
|
models['providers']['dr-gateway'] = models['providers'].pop('fixture-provider')
|
|
models_path.write_text(json.dumps(models))
|
|
settings_path = workspace / '.pi/settings.json'
|
|
settings = json.loads(settings_path.read_bytes()); settings['defaultProvider'] = 'dr-gateway'
|
|
settings_path.write_text(json.dumps(settings))
|
|
for name in ('system.md', 'append-system.md', 'prompt.md'):
|
|
(rd / 'agent' / name).write_text('Write once, then report done.')
|
|
from harness.runners import pi
|
|
metadata = {'state': 'RUNNING', 'runner': 'pi', 'runner_version': pi.VERSION, 'policy': 'teacher', 'mode': 'e2e', 'jail_policy': 'private-home',
|
|
'max_turns': 60, 'max_tokens': 200000}
|
|
(rd / 'rollout.json').write_text(json.dumps(metadata))
|
|
(rd.parent.parent / 'job.json').write_text('{}')
|
|
cfg = yaml.safe_load((ROOT / 'config.yaml').read_text())
|
|
cfg['paths'].update(work_root=str(project / 'work'), node_bin=str(NODE))
|
|
cfg['rollouts'].update(jail=False, private_home=True, tools=['write'])
|
|
cfg['endpoints']['teacher'].update(base_url='http://127.0.0.1:%d/v1' % self.server.server_port,
|
|
model='fixture', api_key_env='FIXTURE_KEY')
|
|
cfg['collection_v1'].update(quota_ledger=str(project / 'runtime-quota.sqlite'),
|
|
quota_groups={'account': asdict(self.broker.groups['account'])}, credential_groups={'teacher': 'account'})
|
|
state = project / 'state'; state.mkdir()
|
|
config_path = project / 'config.yaml'; config_path.write_text(yaml.safe_dump(cfg))
|
|
original_connect, original_readonly = db.connect, db.readonly
|
|
with ExitStack() as stack:
|
|
for name, value in {'BENCH_ROOT': project, 'PROJECT_ROOT': project.parent, 'AGENT': ROOT / 'agent',
|
|
'RUNS': project / 'runs', 'STATE': state, 'CONFIG_PATH': config_path}.items():
|
|
stack.enter_context(patch.object(paths, name, value))
|
|
stack.enter_context(patch.object(paths, 'config', return_value=cfg))
|
|
stack.enter_context(patch.object(paths, 'secret', return_value='upstream-fixture'))
|
|
stack.enter_context(patch.object(db, 'connect', side_effect=lambda: original_connect(state / 'jobs.sqlite')))
|
|
stack.enter_context(patch.object(db, 'readonly', side_effect=lambda: original_readonly(state / 'jobs.sqlite')))
|
|
stack.enter_context(patch.object(runtime, '_broker', None))
|
|
stack.enter_context(patch.object(runtime, '_configuration', None))
|
|
stack.enter_context(patch.object(rollout, 'prepare', return_value=(rd, {}, 'Write once.')))
|
|
# Fake provider proves process/protocol wiring only, never board PASS.
|
|
stack.enter_context(patch.object(rollout, 'verify_rollout', return_value={'terminal': 'UNVERIFIED', 'claim_consistent': False, 'review': False}))
|
|
with db.connect() as con:
|
|
con.execute("INSERT INTO runs VALUES ('r',0,'{}','fixture')")
|
|
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='RUNNING')
|
|
try:
|
|
first = rollout._execute_rollout('r', 'j', 0, 'e2e', 'teacher', False)
|
|
self.assertEqual(first['terminal'], 'PAUSED_INFRA')
|
|
before = json.loads((rd / 'execution.json').read_bytes())
|
|
bound = json.loads((rd / 'rollout.json').read_bytes())
|
|
self.assertEqual(bound['runner_version'], 'pi-sdk-v1')
|
|
self.assertEqual(bound['execution_id'], before['execution_id'])
|
|
self.assertFalse(runtime.broker().execution_status(before['execution_id'])['cancelled'])
|
|
from export import trajectory
|
|
frozen = trajectory.freeze(rd)
|
|
self.assertEqual(trajectory.reconstruct(frozen)['context']['status'], 'COMPLETE')
|
|
stat = (workspace / 'side-effect.txt').stat()
|
|
time.sleep(.25)
|
|
second = rollout.resume_rollout('r', 'j', 0)
|
|
after = json.loads((rd / 'execution.json').read_bytes())
|
|
self.assertEqual(second['terminal'], 'UNVERIFIED')
|
|
self.assertEqual(after['execution_outcome'], 'COMPLETED')
|
|
self.assertTrue(runtime.broker().execution_status(after['execution_id'])['cancelled'])
|
|
self.assertEqual((after['execution_id'], after['deadline_monotonic']),
|
|
(before['execution_id'], before['deadline_monotonic']))
|
|
self.assertEqual(after['epoch'], before['epoch'] + 1)
|
|
self.assertEqual(stat.st_mtime_ns, (workspace / 'side-effect.txt').stat().st_mtime_ns)
|
|
self.assertEqual(runtime.budget_used(rd)['turns'], 2)
|
|
self.assertEqual(len(list(rd.parent.iterdir())), 1)
|
|
finally:
|
|
if runtime._broker:
|
|
runtime._broker.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|