data-gen-bpu/scripts/test_natural_jail.py
bench 4b5b118909 terminal_exec: wait for a ticket queued behind the rollout's own worker
The stage worker runs one ticket at a time, but terminal_exec wait_handle gave up
after 1 s if the ticket had no operation yet ("Unknown protected stage handle").
10-20 staging: `bench stage run baseline` during a 40-minute bench dev compile,
and the agent followed the printed "# Pi wait" hint straight into that error.
In the last 3 days 85% of these errors (production 05/06 and staging) were real,
submitted tickets. A submitted ticket now waits under the execution's own stop
and deadline checks; a ticket the worker rejected before admission says why;
a handle that was never submitted is still unknown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 01:55:47 +08:00

477 lines
29 KiB
Python

"""Real Linux natural-agent mount boundary; no mocked mounts or real secrets.
BENCH_TEST_AGENT_NAMESPACE=1 enables execution. The optional
BENCH_NATURAL_TEST_REAL_BENCH supplies real data/cache paths for open-only denial
checks: the probe never reads bytes from them, even if an isolation failure occurs.
"""
import hashlib
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import tempfile
import textwrap
import threading
import time
import unittest
from unittest.mock import patch
import yaml
from scripts import test_pi_session as sdk_fixtures
@unittest.skipUnless(sys.platform == 'linux' and os.environ.get('BENCH_TEST_AGENT_NAMESPACE') == '1',
'requires explicit real Linux mount-namespace execution')
class NaturalAgentJail(unittest.TestCase):
def test_real_namespace_hides_labels_cache_host_keys_and_preserves_task_materials(self):
source = Path(__file__).resolve().parents[1]
with tempfile.TemporaryDirectory(prefix='natural-jail-independent-', dir=source.parent) as temp:
root = Path(temp).resolve()
bench = root / 'bench'
rd = bench / 'runs/run/job/rollouts/00'
workspace = rd / 'workspace'
for path in (workspace / '.agent-home', rd / 'handles', rd / 'materials',
bench / 'state', bench / 'agent', bench / 'bin', bench / 'scripts',
bench / 'harness/runners', bench / 'harness/adapters',
root / 'shared-data/t1', root / 'shared-cache', root / 'private-host'):
path.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source / 'scripts/jail_setup.py', bench / 'scripts/jail_setup.py')
for path in (source / 'harness/runners').glob('*.py'):
shutil.copyfile(path, bench / 'harness/runners' / path.name)
(bench / 'data').symlink_to(root / 'shared-data', target_is_directory=True)
(bench / 'cache').symlink_to(root / 'shared-cache', target_is_directory=True)
(rd.parent.parent / 'data').symlink_to(root / 'shared-data', target_is_directory=True)
hidden_tar = root / 'shared-data/t1/test.tar'
with tarfile.open(hidden_tar, 'w') as archive:
body = json.dumps({'items': [{'file': 'image.jpg', 'label': 713}]}).encode()
header = tarfile.TarInfo('manifest.json')
header.size = len(body)
archive.addfile(header, io.BytesIO(body))
markers = {
root / 'private-host/id_fixture_key': 'synthetic-host-key-not-a-real-secret',
root / 'shared-cache/labels.json': 'synthetic-cache-labels',
bench / '.env.local': 'synthetic-platform-key',
bench / 'state/jobs.sqlite': 'synthetic-state',
bench / 'harness/scorer.py': 'synthetic-scorer',
rd / 'materials/input.txt': 'current-task-material',
rd / 'rollout.json': json.dumps({'runner': 'pi', 'jail_policy': 'private-home',
'natural_agent_version': 'natural-agent-v1'}),
rd.parent.parent / 'job.json': '{}',
}
for path, value in markers.items():
path.write_text(value)
# Candidate attempts alternate paths as well as advertised paths.
(workspace / 'label-alias').symlink_to(hidden_tar)
denied = [hidden_tar, bench / 'data/t1/test.tar', rd.parent.parent / 'data/t1/test.tar',
root / 'shared-cache/labels.json', bench / 'cache/labels.json',
root / 'private-host/id_fixture_key', bench / '.env.local',
bench / 'state/jobs.sqlite', bench / 'harness/scorer.py', workspace / 'label-alias']
real = os.environ.get('BENCH_NATURAL_TEST_REAL_BENCH')
if real:
actual = Path(real)
denied += [actual / 'data/t1/imagenet1k/n100-s20260914/20260914.tar',
actual / 'cache/weights/index.json']
for path in denied[-2:]:
self.assertTrue(path.is_file(), 'real read-denial precondition missing: ' + str(path))
denied += [p.resolve() for p in denied[-2:]]
config = root / 'test-config.yaml'
config.write_text(yaml.safe_dump({'paths': {'venv': {'bench': sys.prefix}}}))
payload = {'denied': [str(p) for p in denied], 'material': str(rd / 'materials/input.txt'),
'workspace': str(workspace), 'metadata': str(rd / 'rollout.json')}
probe = textwrap.dedent('''
import json, os, subprocess, sys
from pathlib import Path
spec = json.loads(sys.argv[1])
assert os.environ['BENCH_JAIL_POLICY'] == 'private-home'
checks = []
for name in spec['denied']:
try:
fd = os.open(name, os.O_RDONLY | os.O_NONBLOCK)
except OSError as error:
checks.append({'path': name, 'read_denied': True, 'errno': error.errno})
else:
os.close(fd) # Never read actual protected bytes on failure.
raise AssertionError('PROTECTED_PATH_OPENED: ' + name)
assert Path(spec['material']).read_text() == 'current-task-material'
for name in [spec['material'], spec['metadata']]:
try:
fd = os.open(name, os.O_WRONLY | os.O_APPEND)
except OSError:
pass
else:
os.close(fd)
raise AssertionError('TRUSTED_PATH_WRITABLE: ' + name)
workspace = Path(spec['workspace'])
(workspace / 'written-by-candidate.txt').write_text('actual-namespace-write')
assert (workspace / 'written-by-candidate.txt').read_text() == 'actual-namespace-write'
command = 'mkdir -p src && cd src && printf candidate > child.txt'
subprocess.run(['/bin/bash', '-c', command], check=True)
assert (workspace / 'src/child.txt').read_text() == 'candidate'
proc_count = len(list(Path('/proc').glob('[0-9]*')))
assert proc_count < 15, proc_count
remount = subprocess.run(['mount', '-o', 'remount,bind,rw', str(Path(spec['metadata']).parent)], capture_output=True)
assert remount.returncode != 0
print(json.dumps({'schema':'bench.natural-jail-acceptance/1', 'checks':checks,
'materials_readable':True, 'materials_readonly':True,
'workspace_writable':True, 'pid_namespace_processes':proc_count,
'remount_denied':True}))
''')
before = {str(path): hashlib.sha256(path.read_bytes()).hexdigest() for path in markers}
env = dict(os.environ, BENCH_CONFIG=str(config), BENCH_HOST_HOME=str(Path.home()),
BENCH_OUTER_WORKER='1', HOME=str(workspace / '.agent-home'),
PYTHONDONTWRITEBYTECODE='1', TMPDIR=str(root / 'private-host'))
result = subprocess.run(['unshare', '-Urmpf', '--mount-proc', '--kill-child=KILL',
sys.executable, '-B', str(bench / 'scripts/jail_setup.py'), str(bench), str(rd), 'e2e', '--',
sys.executable, '-B', '-c', probe, json.dumps(payload)],
env=env, capture_output=True, text=True, timeout=60)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
observed = json.loads(result.stdout.strip().splitlines()[-1])
self.assertEqual(len(observed['checks']), len(denied))
self.assertTrue(all(row['read_denied'] for row in observed['checks']))
self.assertEqual((workspace / 'written-by-candidate.txt').read_text(), 'actual-namespace-write')
self.assertEqual(before, {str(path): hashlib.sha256(path.read_bytes()).hexdigest() for path in markers})
print(json.dumps(observed, ensure_ascii=False))
@unittest.skipUnless(sdk_fixtures.NODE.is_file(), 'requires the installed Pi 0.73.1 SDK')
class NaturalTerminalSDKWait(unittest.TestCase):
def test_disabled_terminal_exposes_only_wait_in_actual_sdk_and_snapshot(self):
from harness.rollout import rollout_tools
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
fixture.setUp()
self.addCleanup(fixture.doCleanups)
source = Path(__file__).resolve().parents[1]
settings = fixture.root / '.pi/settings.json'
value = json.loads(settings.read_text())
value['extensions'] = [str(source / 'agent/extensions/terminal.ts')]
settings.write_text(json.dumps(value))
cfg = {'terminal': {'enabled': False}, 'rollouts': {'tools': ['write', 'terminal_exec']}}
fixture.config['tools'] = rollout_tools(cfg)
self.assertEqual(fixture.config['tools'], ['write', 'terminal_exec'])
result = fixture.run_sdk()
self.assertEqual(result.returncode, 0, result.stderr.decode())
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
names = [row['function']['name'] for row in events[0]['bench']['tools']]
self.assertEqual(set(names), {'write', 'terminal_exec'})
actual = next(row['function'] for row in events[0]['bench']['tools'] if row['function']['name'] == 'terminal_exec')
self.assertEqual(set(actual['parameters']['properties']), {'wait_handle'})
rd = fixture.root / 'rollout'
rd.mkdir()
for enabled in (False, True):
cfg['terminal']['enabled'] = enabled
(rd / 'rollout.json').write_text(json.dumps(
{'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'} if enabled else {}))
selected = rollout_tools(cfg)
dumped = subprocess.run([str(sdk_fixtures.NODE), str(source / 'agent/dump_tools.js'), ','.join(selected)],
capture_output=True, text=True, timeout=15, check=True,
env={**os.environ, 'BENCH_ROLLOUT_DIR': str(rd)})
snapshot_names = [row['function']['name'] for row in json.loads(dumped.stdout)]
self.assertEqual(snapshot_names.count('terminal_exec'), 1)
definition = next(row['function'] for row in json.loads(dumped.stdout) if row['function']['name'] == 'terminal_exec')
self.assertEqual('command' in definition['parameters']['properties'], enabled)
# Invoke execute directly as well: the schema is not the command boundary.
(rd / 'rollout.json').write_text('{}')
probe = """
(async () => {
const {pathToFileURL} = require('node:url');
const root = require(process.argv[1] + '/agent/dump_tools.js').piRoot();
const {loadExtensions} = await import(pathToFileURL(root + '/dist/core/extensions/loader.js'));
const loaded = await loadExtensions([process.argv[1] + '/agent/extensions/terminal.ts'], process.cwd());
const tool = loaded.extensions[0].tools.get('terminal_exec').definition;
try { await tool.execute('blocked', {command:'never', target:'development', timeout:1}, new AbortController().signal); }
catch (error) { if (error.message.includes('Command execution is disabled')) return; throw error; }
throw new Error('command unexpectedly accepted');
})().catch(error => { console.error(error); process.exitCode = 1; });
"""
result = subprocess.run([str(sdk_fixtures.NODE), '-e', probe, str(source)], capture_output=True,
text=True, timeout=15, env={**os.environ, 'BENCH_ROLLOUT_DIR': str(rd)})
self.assertEqual(result.returncode, 0, result.stderr)
def _stage_fixture(self):
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
fixture.setUp()
self.addCleanup(fixture.doCleanups)
fixture.tool_name, fixture.tool_id = 'terminal_exec', 'stage-long-wait'
fixture.tool_args = {'wait_handle': '05-compile-existing'}
fixture.config.update(tools=['terminal_exec'], ordinary_tool_deadline_s=.2,
managed_tool_deadline_s=4, max_turns=2, max_tokens=100000)
settings = fixture.root / '.pi/settings.json'
value = json.loads(settings.read_text())
value['extensions'] = [str(Path(__file__).resolve().parents[1] / 'agent/extensions/terminal.ts')]
settings.write_text(json.dumps(value))
rd = fixture.root / 'rollout'
queue = rd / 'stages/_queue/05-compile-existing'
queue.mkdir(parents=True)
(rd / 'handles/05-compile-existing').mkdir(parents=True)
identity = {'operation_id': queue.name, 'execution_id': fixture.scope.execution_id,
'epoch': 1, 'payload_sha256': 'fixture-digest'}
(rd / 'rollout.json').write_text(json.dumps({'execution_id': fixture.scope.execution_id}))
operation = {**identity, 'state': 'DISPATCHED'}
state = {'execution_id': fixture.scope.execution_id, 'epoch': 1, 'boot_id': 'fixture-boot',
'execution_state': 'ACTIVE', 'deadline_monotonic': time.monotonic() + 30,
'operations': {queue.name: operation}, 'phases': {queue.name: {
'state': 'RUNNING', 'epoch': 1, 'boot_id': 'fixture-boot',
'execution_deadline_monotonic': time.monotonic() + 20}}}
(rd / 'execution.json').write_text(json.dumps(state))
(queue / 'operation.json').write_text(json.dumps(operation))
(queue / 'status.json').write_text(json.dumps({**identity, 'state': 'RUNNING'}))
# A writable client status is never an authority for wait_handle.
(rd / 'handles' / queue.name / 'status.json').write_text(json.dumps({'state': 'DONE', 'result': 'OK', 'forged': True}))
return fixture, rd, queue, identity, state
def _run_stage(self, fixture, rd):
with patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}):
result = fixture.run_sdk()
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
results = [row['message'] for row in events if row['type'] == 'message_end' and row['message'].get('role') == 'toolResult']
self.assertEqual(len(results), 1)
return result, events, results[0]
def test_stage_wait_uses_protected_receipt_once_and_accepts_skipped(self):
fixture, rd, queue, identity, state = self._stage_fixture()
operation = state['operations'].pop(queue.name)
(rd / 'execution.json').write_text(json.dumps(state))
(queue / 'operation.json').unlink()
observed, errors = {}, []
def finish():
try:
limit = time.monotonic() + 10
while not fixture.bodies and time.monotonic() < limit:
time.sleep(.01)
time.sleep(.35) # Submit may return before the existing Worker polls its ticket.
(queue / 'operation.json').write_text(json.dumps(operation))
state['operations'][queue.name] = operation
temp = rd / 'execution.tmp'
temp.write_text(json.dumps(state))
temp.replace(rd / 'execution.json')
time.sleep(1.1)
observed['requests_while_waiting'] = len(fixture.bodies)
target = queue / 'status.json'
temp = queue / 'status.tmp'
temp.write_text(json.dumps({**identity, 'state': 'DONE', 'result': 'SKIPPED', 'exit_code': 0}))
temp.replace(target)
time.sleep(.35) # Stage result precedes the Worker's confirmed closure.
observed['requests_before_worker_closed'] = len(fixture.bodies)
operation['state'] = 'DONE'
temp = queue / 'operation.tmp'
temp.write_text(json.dumps(operation))
temp.replace(queue / 'operation.json')
temp = rd / 'execution.tmp'
temp.write_text(json.dumps(state))
temp.replace(rd / 'execution.json')
except BaseException as error:
errors.append(error)
thread = threading.Thread(target=finish, daemon=True)
thread.start()
result, events, message = self._run_stage(fixture, rd)
thread.join(timeout=2)
self.assertFalse(thread.is_alive())
if errors: raise errors[0]
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(observed['requests_while_waiting'], 1)
self.assertEqual(observed['requests_before_worker_closed'], 1)
self.assertEqual(len(fixture.bodies), 2)
self.assertFalse(message['isError'])
self.assertIn('SKIPPED', message['content'][0]['text'])
self.assertNotIn('forged', message['content'][0]['text'])
self.assertEqual(list((rd / 'handles').glob('terminal-*')), [])
self.assertFalse(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
def test_stage_wait_outlasts_a_ticket_queued_behind_the_worker(self):
# 10-20: the worker runs one ticket at a time; a baseline submitted during a long bench dev compile
# answered "Unknown protected stage handle" after 1 s. A submitted ticket waits; a rejected one says why.
for case in ('admitted', 'rejected'):
with self.subTest(case=case):
fixture, rd, queue, identity, state = self._stage_fixture()
operation = state['operations'].pop(queue.name)
(rd / 'execution.json').write_text(json.dumps(state))
(queue / 'operation.json').unlink()
handle = rd / 'handles' / queue.name
(handle / 'request.json').write_text(json.dumps({'stage': '05-compile'}))
(handle / 'status.json').write_text(json.dumps({'state': 'QUEUED'}))
def later():
time.sleep(2)
if case == 'rejected':
(handle / 'status.json').write_text(json.dumps({'state': 'DONE', 'result': 'FAIL', 'error_class': 'invalid_request'}))
return
operation['state'] = 'DONE'
(queue / 'operation.json').write_text(json.dumps(operation))
(queue / 'status.json').write_text(json.dumps({**identity, 'state': 'DONE', 'result': 'OK'}))
state['operations'][queue.name] = operation
(rd / 'execution.tmp').write_text(json.dumps(state))
(rd / 'execution.tmp').replace(rd / 'execution.json')
threading.Thread(target=later, daemon=True).start()
result, _, message = self._run_stage(fixture, rd)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(message['isError'], case == 'rejected')
self.assertNotIn('Unknown protected stage handle', message['content'][0]['text'])
if case == 'rejected':
self.assertIn('invalid_request', message['content'][0]['text'])
def test_stage_wait_rejects_unknown_cross_execution_epoch_receipt_and_expired_phase(self):
for case in ('unknown', 'execution', 'epoch', 'receipt', 'expired', 'queued_expired', 'preparing_expired', 'command', 'mixed'):
with self.subTest(case=case):
fixture, rd, queue, identity, state = self._stage_fixture()
if case == 'unknown': fixture.tool_args = {'wait_handle': 'other-task'}
if case == 'execution': state['execution_id'] = 'other-execution'
if case == 'epoch': state['epoch'] = 2
if case == 'receipt':
(queue / 'status.json').write_text(json.dumps({**identity, 'payload_sha256': 'different', 'state': 'DONE', 'result': 'OK'}))
if case == 'expired': state['phases'][queue.name]['execution_deadline_monotonic'] = time.monotonic() - 1
if case == 'queued_expired': state['phases'][queue.name].update(state='QUEUED', queue_deadline_monotonic=time.monotonic() - 1)
if case == 'preparing_expired':
state['phases'] = {}
state['config'] = {'queue_deadline_s': 1000, 'phases': {'05-compile': {'queue_deadline_s': 1}}}
operation = state['operations'][queue.name]
operation.update(state='PREPARING', received_monotonic=time.monotonic() - 5)
(queue / 'operation.json').write_text(json.dumps(operation))
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile'}))
if case in ('command', 'mixed'):
fixture.tool_args = {'command': 'must never run', 'target': 'development', 'timeout': 1,
**({'wait_handle': queue.name} if case == 'mixed' else {})}
(rd / 'execution.json').write_text(json.dumps(state))
result, _, message = self._run_stage(fixture, rd)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertTrue(message['isError'])
self.assertEqual(list((rd / 'handles').glob('terminal-*')), [])
def test_failed_stage_receipt_is_preserved_even_after_cancellation(self):
fixture, rd, queue, identity, state = self._stage_fixture()
state['cancel_requested'] = {'reason': 'USER_STOP'}
state['operations'][queue.name]['state'] = 'DONE'
(queue / 'operation.json').write_text(json.dumps(state['operations'][queue.name]))
(rd / 'execution.json').write_text(json.dumps(state))
receipt = {**identity, 'state': 'DONE', 'result': 'CANCELLED', 'exit_code': -15, 'error_class': 'USER_STOP'}
(queue / 'status.json').write_text(json.dumps(receipt))
result, _, message = self._run_stage(fixture, rd)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertTrue(message['isError'])
self.assertIn('USER_STOP', message['content'][0]['text'])
self.assertIn('"exit_code":-15', message['content'][0]['text'].replace(' ', ''))
def test_stage_wait_allows_completed_same_execution_previous_epoch(self):
fixture, rd, queue, identity, state = self._stage_fixture()
state['epoch'] = 2
state['operations'][queue.name]['state'] = 'DONE'
(rd / 'execution.json').write_text(json.dumps(state))
(queue / 'operation.json').write_text(json.dumps(state['operations'][queue.name]))
(queue / 'status.json').write_text(json.dumps({**identity, 'state': 'DONE', 'result': 'OK'}))
result, _, message = self._run_stage(fixture, rd)
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertFalse(message['isError'])
def test_stage_wait_obeys_existing_managed_abort(self):
fixture, rd, _, _, _ = self._stage_fixture()
fixture.config['managed_tool_deadline_s'] = .4
result, events, message = self._run_stage(fixture, rd)
self.assertEqual(result.returncode, 78, result.stderr.decode())
self.assertTrue(message['isError'])
self.assertTrue(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
self.assertEqual(len(fixture.bodies), 1)
def test_real_sdk_terminal_wait_exceeds_ordinary_cap_without_model_polling(self):
self._run_sdk_terminal_receipt(fails=False)
def test_failed_terminal_receipt_is_native_error_then_natural_final_at_budget(self):
self._run_sdk_terminal_receipt(fails=True)
def _run_sdk_terminal_receipt(self, *, fails):
"""Real SDK/extension, fake provider and receipt; not Worker/board proof."""
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
fixture.setUp()
self.addCleanup(fixture.doCleanups)
fixture.tool_name = 'terminal_exec'
fixture.tool_id = 'terminal-long-wait'
fixture.tool_args = {'target': 'development', 'command': 'controlled fixture receipt', 'timeout': 3}
fixture.config.update(tools=['terminal_exec'], ordinary_tool_deadline_s=0.2,
managed_tool_deadline_s=4, max_turns=2, max_tokens=100000)
settings = fixture.root / '.pi/settings.json'
value = json.loads(settings.read_text())
value['extensions'] = [str(Path(__file__).resolve().parents[1] / 'agent/extensions/terminal.ts')]
settings.write_text(json.dumps(value))
rd = fixture.root / 'rollout'
(rd / 'handles').mkdir(parents=True)
(rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
receipt_errors = []
observed = {}
receipt = {'state': 'DONE', 'result': 'FAIL' if fails else 'OK',
'exit_code': 7 if fails else 0, 'fixture': True,
'error_class': 'COMMAND_EXIT' if fails else None,
'stderr_tail': 'controlled-command-error' if fails else ''}
def receipt_fixture():
try:
limit = time.monotonic() + 15
while time.monotonic() < limit:
found = list((rd / 'handles').glob('terminal-*/request.json'))
if found:
break
time.sleep(.01)
else:
raise AssertionError('terminal extension never submitted a request')
self.assertEqual(len(found), 1)
self.assertEqual(json.loads(found[0].read_text())['command'], 'controlled fixture receipt')
observed['requests_before_wait'] = len(fixture.bodies)
started = time.monotonic()
time.sleep(1.1) # >5x ordinary cap, within the managed cap.
observed['requests_after_wait'] = len(fixture.bodies)
observed['wait_seconds'] = time.monotonic() - started
status = found[0].parent / 'status.json'
temporary = status.with_suffix('.tmp')
temporary.write_text(json.dumps(receipt))
temporary.replace(status)
except BaseException as error:
receipt_errors.append(error)
thread = threading.Thread(target=receipt_fixture, daemon=True)
thread.start()
with patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}):
result = fixture.run_sdk()
thread.join(timeout=2)
self.assertFalse(thread.is_alive(), 'receipt fixture did not finish')
if receipt_errors:
raise receipt_errors[0]
self.assertEqual(result.returncode, 0, result.stderr.decode())
self.assertEqual(observed['requests_before_wait'], 1)
self.assertEqual(observed['requests_after_wait'], 1)
self.assertEqual(len(fixture.bodies), 2)
checkpoint = json.loads((fixture.root / '.bench-session.json').read_text())
self.assertEqual(checkpoint['phase'], 'COMPLETED')
self.assertTrue(checkpoint['closure'])
self.assertEqual(checkpoint['budget']['turns'], 2)
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
boundary = [row for row in events if row['type'] == 'tool_execution_start'][0]['bench_tool']
self.assertEqual(boundary['cap_s'], 4)
self.assertFalse(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
tool_ends = [row for row in events if row['type'] == 'tool_execution_end']
tool_results = [row['message'] for row in events if row['type'] == 'message_end'
and row['message'].get('role') == 'toolResult']
self.assertEqual(len(tool_ends), 1)
self.assertEqual(len(tool_results), 1)
self.assertIs(tool_ends[0]['isError'], fails)
self.assertIs(tool_results[0]['isError'], fails)
self.assertEqual(tool_results[0]['toolCallId'], fixture.tool_id)
if fails:
content = tool_results[0]['content'][0]['text']
self.assertIn('controlled-command-error', content)
self.assertIn('"exit_code":7', content.replace(' ', ''))
self.assertIn('"result":"FAIL"', content.replace(' ', ''))
finals = [row['message'] for row in events if row['type'] == 'message_end'
and row['message'].get('role') == 'assistant']
self.assertEqual(finals[-1]['stopReason'], 'stop')
self.assertTrue(any(row['type'] == 'agent_end' for row in events))
print(json.dumps({'schema':'bench.natural-sdk-wait-acceptance/1', **observed,
'generations_during_wait': 0, 'http_requests_total': len(fixture.bodies),
'ordinary_cap_s': .2, 'managed_cap_s': 4, 'provider': 'controlled-fixture',
'receipt': 'controlled-fixture', 'sdk': 'real-installed',
'receipt_failed': fails, 'native_tool_result_is_error': tool_results[0]['isError'],
'budget_turns': checkpoint['budget']['turns'], 'normal_final': True}))
if __name__ == '__main__':
unittest.main()