446 lines
27 KiB
Python
446 lines
27 KiB
Python
"""Real Linux natural-agent mount boundary; no mocked mounts or real secrets.
|
|
|
|
BENCH_TEST_AGENT_NAMESPACE=1 enables execution. The optional
|
|
BENCH_NATURAL_TEST_REAL_BENCH supplies real data/cache paths for open-only denial
|
|
checks: the probe never reads bytes from them, even if an isolation failure occurs.
|
|
"""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import textwrap
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import yaml
|
|
from scripts import test_pi_session as sdk_fixtures
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux' and os.environ.get('BENCH_TEST_AGENT_NAMESPACE') == '1',
|
|
'requires explicit real Linux mount-namespace execution')
|
|
class NaturalAgentJail(unittest.TestCase):
|
|
def test_real_namespace_hides_labels_cache_host_keys_and_preserves_task_materials(self):
|
|
source = Path(__file__).resolve().parents[1]
|
|
with tempfile.TemporaryDirectory(prefix='natural-jail-independent-', dir=source.parent) as temp:
|
|
root = Path(temp).resolve()
|
|
bench = root / 'bench'
|
|
rd = bench / 'runs/run/job/rollouts/00'
|
|
workspace = rd / 'workspace'
|
|
for path in (workspace / '.agent-home', rd / 'handles', rd / 'materials',
|
|
bench / 'state', bench / 'agent', bench / 'bin', bench / 'scripts',
|
|
bench / 'harness/runners', bench / 'harness/adapters',
|
|
root / 'shared-data/t1', root / 'shared-cache', root / 'private-host'):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(source / 'scripts/jail_setup.py', bench / 'scripts/jail_setup.py')
|
|
for path in (source / 'harness/runners').glob('*.py'):
|
|
shutil.copyfile(path, bench / 'harness/runners' / path.name)
|
|
(bench / 'data').symlink_to(root / 'shared-data', target_is_directory=True)
|
|
(bench / 'cache').symlink_to(root / 'shared-cache', target_is_directory=True)
|
|
(rd.parent.parent / 'data').symlink_to(root / 'shared-data', target_is_directory=True)
|
|
hidden_tar = root / 'shared-data/t1/test.tar'
|
|
with tarfile.open(hidden_tar, 'w') as archive:
|
|
body = json.dumps({'items': [{'file': 'image.jpg', 'label': 713}]}).encode()
|
|
header = tarfile.TarInfo('manifest.json')
|
|
header.size = len(body)
|
|
archive.addfile(header, io.BytesIO(body))
|
|
markers = {
|
|
root / 'private-host/id_fixture_key': 'synthetic-host-key-not-a-real-secret',
|
|
root / 'shared-cache/labels.json': 'synthetic-cache-labels',
|
|
bench / '.env.local': 'synthetic-platform-key',
|
|
bench / 'state/jobs.sqlite': 'synthetic-state',
|
|
bench / 'harness/scorer.py': 'synthetic-scorer',
|
|
rd / 'materials/input.txt': 'current-task-material',
|
|
rd / 'rollout.json': json.dumps({'runner': 'pi', 'jail_policy': 'private-home',
|
|
'natural_agent_version': 'natural-agent-v1'}),
|
|
rd.parent.parent / 'job.json': '{}',
|
|
}
|
|
for path, value in markers.items():
|
|
path.write_text(value)
|
|
# Candidate attempts alternate paths as well as advertised paths.
|
|
(workspace / 'label-alias').symlink_to(hidden_tar)
|
|
denied = [hidden_tar, bench / 'data/t1/test.tar', rd.parent.parent / 'data/t1/test.tar',
|
|
root / 'shared-cache/labels.json', bench / 'cache/labels.json',
|
|
root / 'private-host/id_fixture_key', bench / '.env.local',
|
|
bench / 'state/jobs.sqlite', bench / 'harness/scorer.py', workspace / 'label-alias']
|
|
real = os.environ.get('BENCH_NATURAL_TEST_REAL_BENCH')
|
|
if real:
|
|
actual = Path(real)
|
|
denied += [actual / 'data/t1/imagenet1k/n100-s20260914/20260914.tar',
|
|
actual / 'cache/weights/index.json']
|
|
for path in denied[-2:]:
|
|
self.assertTrue(path.is_file(), 'real read-denial precondition missing: ' + str(path))
|
|
denied += [p.resolve() for p in denied[-2:]]
|
|
config = root / 'test-config.yaml'
|
|
config.write_text(yaml.safe_dump({'paths': {'venv': {'bench': sys.prefix}}}))
|
|
payload = {'denied': [str(p) for p in denied], 'material': str(rd / 'materials/input.txt'),
|
|
'workspace': str(workspace), 'metadata': str(rd / 'rollout.json')}
|
|
probe = textwrap.dedent('''
|
|
import json, os, subprocess, sys
|
|
from pathlib import Path
|
|
spec = json.loads(sys.argv[1])
|
|
assert os.environ['BENCH_JAIL_POLICY'] == 'private-home'
|
|
checks = []
|
|
for name in spec['denied']:
|
|
try:
|
|
fd = os.open(name, os.O_RDONLY | os.O_NONBLOCK)
|
|
except OSError as error:
|
|
checks.append({'path': name, 'read_denied': True, 'errno': error.errno})
|
|
else:
|
|
os.close(fd) # Never read actual protected bytes on failure.
|
|
raise AssertionError('PROTECTED_PATH_OPENED: ' + name)
|
|
assert Path(spec['material']).read_text() == 'current-task-material'
|
|
for name in [spec['material'], spec['metadata']]:
|
|
try:
|
|
fd = os.open(name, os.O_WRONLY | os.O_APPEND)
|
|
except OSError:
|
|
pass
|
|
else:
|
|
os.close(fd)
|
|
raise AssertionError('TRUSTED_PATH_WRITABLE: ' + name)
|
|
workspace = Path(spec['workspace'])
|
|
(workspace / 'written-by-candidate.txt').write_text('actual-namespace-write')
|
|
assert (workspace / 'written-by-candidate.txt').read_text() == 'actual-namespace-write'
|
|
command = 'mkdir -p src && cd src && printf candidate > child.txt'
|
|
subprocess.run(['/bin/bash', '-c', command], check=True)
|
|
assert (workspace / 'src/child.txt').read_text() == 'candidate'
|
|
proc_count = len(list(Path('/proc').glob('[0-9]*')))
|
|
assert proc_count < 15, proc_count
|
|
remount = subprocess.run(['mount', '-o', 'remount,bind,rw', str(Path(spec['metadata']).parent)], capture_output=True)
|
|
assert remount.returncode != 0
|
|
print(json.dumps({'schema':'bench.natural-jail-acceptance/1', 'checks':checks,
|
|
'materials_readable':True, 'materials_readonly':True,
|
|
'workspace_writable':True, 'pid_namespace_processes':proc_count,
|
|
'remount_denied':True}))
|
|
''')
|
|
before = {str(path): hashlib.sha256(path.read_bytes()).hexdigest() for path in markers}
|
|
env = dict(os.environ, BENCH_CONFIG=str(config), BENCH_HOST_HOME=str(Path.home()),
|
|
BENCH_OUTER_WORKER='1', HOME=str(workspace / '.agent-home'),
|
|
PYTHONDONTWRITEBYTECODE='1', TMPDIR=str(root / 'private-host'))
|
|
result = subprocess.run(['unshare', '-Urmpf', '--mount-proc', '--kill-child=KILL',
|
|
sys.executable, '-B', str(bench / 'scripts/jail_setup.py'), str(bench), str(rd), 'e2e', '--',
|
|
sys.executable, '-B', '-c', probe, json.dumps(payload)],
|
|
env=env, capture_output=True, text=True, timeout=60)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
observed = json.loads(result.stdout.strip().splitlines()[-1])
|
|
self.assertEqual(len(observed['checks']), len(denied))
|
|
self.assertTrue(all(row['read_denied'] for row in observed['checks']))
|
|
self.assertEqual((workspace / 'written-by-candidate.txt').read_text(), 'actual-namespace-write')
|
|
self.assertEqual(before, {str(path): hashlib.sha256(path.read_bytes()).hexdigest() for path in markers})
|
|
print(json.dumps(observed, ensure_ascii=False))
|
|
|
|
|
|
@unittest.skipUnless(sdk_fixtures.NODE.is_file(), 'requires the installed Pi 0.73.1 SDK')
|
|
class NaturalTerminalSDKWait(unittest.TestCase):
|
|
def test_disabled_terminal_exposes_only_wait_in_actual_sdk_and_snapshot(self):
|
|
from harness.rollout import rollout_tools
|
|
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
|
|
fixture.setUp()
|
|
self.addCleanup(fixture.doCleanups)
|
|
source = Path(__file__).resolve().parents[1]
|
|
settings = fixture.root / '.pi/settings.json'
|
|
value = json.loads(settings.read_text())
|
|
value['extensions'] = [str(source / 'agent/extensions/terminal.ts')]
|
|
settings.write_text(json.dumps(value))
|
|
cfg = {'terminal': {'enabled': False}, 'rollouts': {'tools': ['write', 'terminal_exec']}}
|
|
fixture.config['tools'] = rollout_tools(cfg)
|
|
self.assertEqual(fixture.config['tools'], ['write', 'terminal_exec'])
|
|
result = fixture.run_sdk()
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
|
|
names = [row['function']['name'] for row in events[0]['bench']['tools']]
|
|
self.assertEqual(set(names), {'write', 'terminal_exec'})
|
|
actual = next(row['function'] for row in events[0]['bench']['tools'] if row['function']['name'] == 'terminal_exec')
|
|
self.assertEqual(set(actual['parameters']['properties']), {'wait_handle'})
|
|
rd = fixture.root / 'rollout'
|
|
rd.mkdir()
|
|
for enabled in (False, True):
|
|
cfg['terminal']['enabled'] = enabled
|
|
(rd / 'rollout.json').write_text(json.dumps(
|
|
{'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'} if enabled else {}))
|
|
selected = rollout_tools(cfg)
|
|
dumped = subprocess.run([str(sdk_fixtures.NODE), str(source / 'agent/dump_tools.js'), ','.join(selected)],
|
|
capture_output=True, text=True, timeout=15, check=True,
|
|
env={**os.environ, 'BENCH_ROLLOUT_DIR': str(rd)})
|
|
snapshot_names = [row['function']['name'] for row in json.loads(dumped.stdout)]
|
|
self.assertEqual(snapshot_names.count('terminal_exec'), 1)
|
|
definition = next(row['function'] for row in json.loads(dumped.stdout) if row['function']['name'] == 'terminal_exec')
|
|
self.assertEqual('command' in definition['parameters']['properties'], enabled)
|
|
# Invoke execute directly as well: the schema is not the command boundary.
|
|
(rd / 'rollout.json').write_text('{}')
|
|
probe = """
|
|
(async () => {
|
|
const {pathToFileURL} = require('node:url');
|
|
const root = require(process.argv[1] + '/agent/dump_tools.js').piRoot();
|
|
const {loadExtensions} = await import(pathToFileURL(root + '/dist/core/extensions/loader.js'));
|
|
const loaded = await loadExtensions([process.argv[1] + '/agent/extensions/terminal.ts'], process.cwd());
|
|
const tool = loaded.extensions[0].tools.get('terminal_exec').definition;
|
|
try { await tool.execute('blocked', {command:'never', target:'development', timeout:1}, new AbortController().signal); }
|
|
catch (error) { if (error.message.includes('Command execution is disabled')) return; throw error; }
|
|
throw new Error('command unexpectedly accepted');
|
|
})().catch(error => { console.error(error); process.exitCode = 1; });
|
|
"""
|
|
result = subprocess.run([str(sdk_fixtures.NODE), '-e', probe, str(source)], capture_output=True,
|
|
text=True, timeout=15, env={**os.environ, 'BENCH_ROLLOUT_DIR': str(rd)})
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
|
|
def _stage_fixture(self):
|
|
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
|
|
fixture.setUp()
|
|
self.addCleanup(fixture.doCleanups)
|
|
fixture.tool_name, fixture.tool_id = 'terminal_exec', 'stage-long-wait'
|
|
fixture.tool_args = {'wait_handle': '05-compile-existing'}
|
|
fixture.config.update(tools=['terminal_exec'], ordinary_tool_deadline_s=.2,
|
|
managed_tool_deadline_s=4, max_turns=2, max_tokens=100000)
|
|
settings = fixture.root / '.pi/settings.json'
|
|
value = json.loads(settings.read_text())
|
|
value['extensions'] = [str(Path(__file__).resolve().parents[1] / 'agent/extensions/terminal.ts')]
|
|
settings.write_text(json.dumps(value))
|
|
rd = fixture.root / 'rollout'
|
|
queue = rd / 'stages/_queue/05-compile-existing'
|
|
queue.mkdir(parents=True)
|
|
(rd / 'handles/05-compile-existing').mkdir(parents=True)
|
|
identity = {'operation_id': queue.name, 'execution_id': fixture.scope.execution_id,
|
|
'epoch': 1, 'payload_sha256': 'fixture-digest'}
|
|
(rd / 'rollout.json').write_text(json.dumps({'execution_id': fixture.scope.execution_id}))
|
|
operation = {**identity, 'state': 'DISPATCHED'}
|
|
state = {'execution_id': fixture.scope.execution_id, 'epoch': 1, 'boot_id': 'fixture-boot',
|
|
'execution_state': 'ACTIVE', 'deadline_monotonic': time.monotonic() + 30,
|
|
'operations': {queue.name: operation}, 'phases': {queue.name: {
|
|
'state': 'RUNNING', 'epoch': 1, 'boot_id': 'fixture-boot',
|
|
'execution_deadline_monotonic': time.monotonic() + 20}}}
|
|
(rd / 'execution.json').write_text(json.dumps(state))
|
|
(queue / 'operation.json').write_text(json.dumps(operation))
|
|
(queue / 'status.json').write_text(json.dumps({**identity, 'state': 'RUNNING'}))
|
|
# A writable client status is never an authority for wait_handle.
|
|
(rd / 'handles' / queue.name / 'status.json').write_text(json.dumps({'state': 'DONE', 'result': 'OK', 'forged': True}))
|
|
return fixture, rd, queue, identity, state
|
|
|
|
def _run_stage(self, fixture, rd):
|
|
with patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}):
|
|
result = fixture.run_sdk()
|
|
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
|
|
results = [row['message'] for row in events if row['type'] == 'message_end' and row['message'].get('role') == 'toolResult']
|
|
self.assertEqual(len(results), 1)
|
|
return result, events, results[0]
|
|
|
|
def test_stage_wait_uses_protected_receipt_once_and_accepts_skipped(self):
|
|
fixture, rd, queue, identity, state = self._stage_fixture()
|
|
operation = state['operations'].pop(queue.name)
|
|
(rd / 'execution.json').write_text(json.dumps(state))
|
|
(queue / 'operation.json').unlink()
|
|
observed, errors = {}, []
|
|
def finish():
|
|
try:
|
|
limit = time.monotonic() + 10
|
|
while not fixture.bodies and time.monotonic() < limit:
|
|
time.sleep(.01)
|
|
time.sleep(.35) # Submit may return before the existing Worker polls its ticket.
|
|
(queue / 'operation.json').write_text(json.dumps(operation))
|
|
state['operations'][queue.name] = operation
|
|
temp = rd / 'execution.tmp'
|
|
temp.write_text(json.dumps(state))
|
|
temp.replace(rd / 'execution.json')
|
|
time.sleep(1.1)
|
|
observed['requests_while_waiting'] = len(fixture.bodies)
|
|
target = queue / 'status.json'
|
|
temp = queue / 'status.tmp'
|
|
temp.write_text(json.dumps({**identity, 'state': 'DONE', 'result': 'SKIPPED', 'exit_code': 0}))
|
|
temp.replace(target)
|
|
time.sleep(.35) # Stage result precedes the Worker's confirmed closure.
|
|
observed['requests_before_worker_closed'] = len(fixture.bodies)
|
|
operation['state'] = 'DONE'
|
|
temp = queue / 'operation.tmp'
|
|
temp.write_text(json.dumps(operation))
|
|
temp.replace(queue / 'operation.json')
|
|
temp = rd / 'execution.tmp'
|
|
temp.write_text(json.dumps(state))
|
|
temp.replace(rd / 'execution.json')
|
|
except BaseException as error:
|
|
errors.append(error)
|
|
thread = threading.Thread(target=finish, daemon=True)
|
|
thread.start()
|
|
result, events, message = self._run_stage(fixture, rd)
|
|
thread.join(timeout=2)
|
|
self.assertFalse(thread.is_alive())
|
|
if errors: raise errors[0]
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(observed['requests_while_waiting'], 1)
|
|
self.assertEqual(observed['requests_before_worker_closed'], 1)
|
|
self.assertEqual(len(fixture.bodies), 2)
|
|
self.assertFalse(message['isError'])
|
|
self.assertIn('SKIPPED', message['content'][0]['text'])
|
|
self.assertNotIn('forged', message['content'][0]['text'])
|
|
self.assertEqual(list((rd / 'handles').glob('terminal-*')), [])
|
|
self.assertFalse(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
|
|
|
|
def test_stage_wait_rejects_unknown_cross_execution_epoch_receipt_and_expired_phase(self):
|
|
for case in ('unknown', 'execution', 'epoch', 'receipt', 'expired', 'queued_expired', 'preparing_expired', 'command', 'mixed'):
|
|
with self.subTest(case=case):
|
|
fixture, rd, queue, identity, state = self._stage_fixture()
|
|
if case == 'unknown': fixture.tool_args = {'wait_handle': 'other-task'}
|
|
if case == 'execution': state['execution_id'] = 'other-execution'
|
|
if case == 'epoch': state['epoch'] = 2
|
|
if case == 'receipt':
|
|
(queue / 'status.json').write_text(json.dumps({**identity, 'payload_sha256': 'different', 'state': 'DONE', 'result': 'OK'}))
|
|
if case == 'expired': state['phases'][queue.name]['execution_deadline_monotonic'] = time.monotonic() - 1
|
|
if case == 'queued_expired': state['phases'][queue.name].update(state='QUEUED', queue_deadline_monotonic=time.monotonic() - 1)
|
|
if case == 'preparing_expired':
|
|
state['phases'] = {}
|
|
state['config'] = {'queue_deadline_s': 1000, 'phases': {'05-compile': {'queue_deadline_s': 1}}}
|
|
operation = state['operations'][queue.name]
|
|
operation.update(state='PREPARING', received_monotonic=time.monotonic() - 5)
|
|
(queue / 'operation.json').write_text(json.dumps(operation))
|
|
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile'}))
|
|
if case in ('command', 'mixed'):
|
|
fixture.tool_args = {'command': 'must never run', 'target': 'development', 'timeout': 1,
|
|
**({'wait_handle': queue.name} if case == 'mixed' else {})}
|
|
(rd / 'execution.json').write_text(json.dumps(state))
|
|
result, _, message = self._run_stage(fixture, rd)
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertTrue(message['isError'])
|
|
self.assertEqual(list((rd / 'handles').glob('terminal-*')), [])
|
|
|
|
def test_failed_stage_receipt_is_preserved_even_after_cancellation(self):
|
|
fixture, rd, queue, identity, state = self._stage_fixture()
|
|
state['cancel_requested'] = {'reason': 'USER_STOP'}
|
|
state['operations'][queue.name]['state'] = 'DONE'
|
|
(queue / 'operation.json').write_text(json.dumps(state['operations'][queue.name]))
|
|
(rd / 'execution.json').write_text(json.dumps(state))
|
|
receipt = {**identity, 'state': 'DONE', 'result': 'CANCELLED', 'exit_code': -15, 'error_class': 'USER_STOP'}
|
|
(queue / 'status.json').write_text(json.dumps(receipt))
|
|
result, _, message = self._run_stage(fixture, rd)
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertTrue(message['isError'])
|
|
self.assertIn('USER_STOP', message['content'][0]['text'])
|
|
self.assertIn('"exit_code":-15', message['content'][0]['text'].replace(' ', ''))
|
|
|
|
def test_stage_wait_allows_completed_same_execution_previous_epoch(self):
|
|
fixture, rd, queue, identity, state = self._stage_fixture()
|
|
state['epoch'] = 2
|
|
state['operations'][queue.name]['state'] = 'DONE'
|
|
(rd / 'execution.json').write_text(json.dumps(state))
|
|
(queue / 'operation.json').write_text(json.dumps(state['operations'][queue.name]))
|
|
(queue / 'status.json').write_text(json.dumps({**identity, 'state': 'DONE', 'result': 'OK'}))
|
|
result, _, message = self._run_stage(fixture, rd)
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertFalse(message['isError'])
|
|
|
|
def test_stage_wait_obeys_existing_managed_abort(self):
|
|
fixture, rd, _, _, _ = self._stage_fixture()
|
|
fixture.config['managed_tool_deadline_s'] = .4
|
|
result, events, message = self._run_stage(fixture, rd)
|
|
self.assertEqual(result.returncode, 78, result.stderr.decode())
|
|
self.assertTrue(message['isError'])
|
|
self.assertTrue(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
|
|
self.assertEqual(len(fixture.bodies), 1)
|
|
|
|
def test_real_sdk_terminal_wait_exceeds_ordinary_cap_without_model_polling(self):
|
|
self._run_sdk_terminal_receipt(fails=False)
|
|
|
|
def test_failed_terminal_receipt_is_native_error_then_natural_final_at_budget(self):
|
|
self._run_sdk_terminal_receipt(fails=True)
|
|
|
|
def _run_sdk_terminal_receipt(self, *, fails):
|
|
"""Real SDK/extension, fake provider and receipt; not Worker/board proof."""
|
|
fixture = sdk_fixtures.PiSDKTests(methodName='runTest')
|
|
fixture.setUp()
|
|
self.addCleanup(fixture.doCleanups)
|
|
fixture.tool_name = 'terminal_exec'
|
|
fixture.tool_id = 'terminal-long-wait'
|
|
fixture.tool_args = {'target': 'development', 'command': 'controlled fixture receipt', 'timeout': 3}
|
|
fixture.config.update(tools=['terminal_exec'], ordinary_tool_deadline_s=0.2,
|
|
managed_tool_deadline_s=4, max_turns=2, max_tokens=100000)
|
|
settings = fixture.root / '.pi/settings.json'
|
|
value = json.loads(settings.read_text())
|
|
value['extensions'] = [str(Path(__file__).resolve().parents[1] / 'agent/extensions/terminal.ts')]
|
|
settings.write_text(json.dumps(value))
|
|
rd = fixture.root / 'rollout'
|
|
(rd / 'handles').mkdir(parents=True)
|
|
(rd / 'rollout.json').write_text(json.dumps({'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home'}))
|
|
receipt_errors = []
|
|
observed = {}
|
|
receipt = {'state': 'DONE', 'result': 'FAIL' if fails else 'OK',
|
|
'exit_code': 7 if fails else 0, 'fixture': True,
|
|
'error_class': 'COMMAND_EXIT' if fails else None,
|
|
'stderr_tail': 'controlled-command-error' if fails else ''}
|
|
|
|
def receipt_fixture():
|
|
try:
|
|
limit = time.monotonic() + 15
|
|
while time.monotonic() < limit:
|
|
found = list((rd / 'handles').glob('terminal-*/request.json'))
|
|
if found:
|
|
break
|
|
time.sleep(.01)
|
|
else:
|
|
raise AssertionError('terminal extension never submitted a request')
|
|
self.assertEqual(len(found), 1)
|
|
self.assertEqual(json.loads(found[0].read_text())['command'], 'controlled fixture receipt')
|
|
observed['requests_before_wait'] = len(fixture.bodies)
|
|
started = time.monotonic()
|
|
time.sleep(1.1) # >5x ordinary cap, within the managed cap.
|
|
observed['requests_after_wait'] = len(fixture.bodies)
|
|
observed['wait_seconds'] = time.monotonic() - started
|
|
status = found[0].parent / 'status.json'
|
|
temporary = status.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(receipt))
|
|
temporary.replace(status)
|
|
except BaseException as error:
|
|
receipt_errors.append(error)
|
|
|
|
thread = threading.Thread(target=receipt_fixture, daemon=True)
|
|
thread.start()
|
|
with patch.dict(os.environ, {'BENCH_ROLLOUT_DIR': str(rd)}):
|
|
result = fixture.run_sdk()
|
|
thread.join(timeout=2)
|
|
self.assertFalse(thread.is_alive(), 'receipt fixture did not finish')
|
|
if receipt_errors:
|
|
raise receipt_errors[0]
|
|
self.assertEqual(result.returncode, 0, result.stderr.decode())
|
|
self.assertEqual(observed['requests_before_wait'], 1)
|
|
self.assertEqual(observed['requests_after_wait'], 1)
|
|
self.assertEqual(len(fixture.bodies), 2)
|
|
checkpoint = json.loads((fixture.root / '.bench-session.json').read_text())
|
|
self.assertEqual(checkpoint['phase'], 'COMPLETED')
|
|
self.assertTrue(checkpoint['closure'])
|
|
self.assertEqual(checkpoint['budget']['turns'], 2)
|
|
events = [json.loads(line) for line in (fixture.root / 'events.jsonl').read_text().splitlines()]
|
|
boundary = [row for row in events if row['type'] == 'tool_execution_start'][0]['bench_tool']
|
|
self.assertEqual(boundary['cap_s'], 4)
|
|
self.assertFalse(any(row.get('bench_tool', {}).get('state') == 'TIMED_OUT' for row in events))
|
|
tool_ends = [row for row in events if row['type'] == 'tool_execution_end']
|
|
tool_results = [row['message'] for row in events if row['type'] == 'message_end'
|
|
and row['message'].get('role') == 'toolResult']
|
|
self.assertEqual(len(tool_ends), 1)
|
|
self.assertEqual(len(tool_results), 1)
|
|
self.assertIs(tool_ends[0]['isError'], fails)
|
|
self.assertIs(tool_results[0]['isError'], fails)
|
|
self.assertEqual(tool_results[0]['toolCallId'], fixture.tool_id)
|
|
if fails:
|
|
content = tool_results[0]['content'][0]['text']
|
|
self.assertIn('controlled-command-error', content)
|
|
self.assertIn('"exit_code":7', content.replace(' ', ''))
|
|
self.assertIn('"result":"FAIL"', content.replace(' ', ''))
|
|
finals = [row['message'] for row in events if row['type'] == 'message_end'
|
|
and row['message'].get('role') == 'assistant']
|
|
self.assertEqual(finals[-1]['stopReason'], 'stop')
|
|
self.assertTrue(any(row['type'] == 'agent_end' for row in events))
|
|
print(json.dumps({'schema':'bench.natural-sdk-wait-acceptance/1', **observed,
|
|
'generations_during_wait': 0, 'http_requests_total': len(fixture.bodies),
|
|
'ordinary_cap_s': .2, 'managed_cap_s': 4, 'provider': 'controlled-fixture',
|
|
'receipt': 'controlled-fixture', 'sdk': 'real-installed',
|
|
'receipt_failed': fails, 'native_tool_result_is_error': tool_results[0]['isError'],
|
|
'budget_turns': checkpoint['budget']['turns'], 'normal_final': True}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|