A queued rollout was never claimed once any sibling ended DEADLINE_EXCEEDED, CANCELLED, INFRA_ERROR, LOST or BUDGET_EXHAUSTED. Almost all of those were system stops (tool cap, context rejected by the local quota, board cleanup or request outcome unknown, pauses that expired, operator restarts), so 1107 queued rollouts across r20260929-04/05/06 sat behind them. Siblings still run one at a time, and two stops still hold the rest: an exhausted turn/token budget, and a DEADLINE_EXCEEDED whose verdict says the trajectory really worked until the 24 h execution deadline. Agent- finished FAIL/UNVERIFIED keep not holding siblings, as before. Tests that pinned the old hold now assert the repeat is claimable; the lease-recovery waves no longer depend on sibling holds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
562 lines
35 KiB
Python
562 lines
35 KiB
Python
"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
|
|
import io
|
|
import json
|
|
import shlex
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import threading
|
|
import unittest
|
|
from contextlib import contextmanager
|
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from export import trajectory
|
|
from harness import db, execution, monitor, paths, pipeline, rollout
|
|
from scripts import test_collection_crash, test_pi_session
|
|
from scripts.test_execution import policy
|
|
from scripts.test_quota import ok
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
|
|
class ToolBoundaryCommit(unittest.TestCase):
|
|
@contextmanager
|
|
def first_reader_bytes(self, path, raw):
|
|
"""Expose a stale first read without changing the real event evidence."""
|
|
original = Path.open
|
|
opened = []
|
|
def opening(target, mode='r', *args, **kwargs):
|
|
if target == path and mode == 'rb':
|
|
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
|
|
opened.append(stream)
|
|
return stream
|
|
return original(target, mode, *args, **kwargs)
|
|
with patch.object(Path, 'open', new=opening):
|
|
yield opened
|
|
|
|
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
|
|
from harness import collection_runtime
|
|
for bad in (b'{"type":"note","value":"\x00"}\n',
|
|
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
|
|
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
path = rd / 'agent/events.jsonl'
|
|
prefix = b'{"type":"already_consumed"}\n'
|
|
path.write_bytes(prefix)
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='completed', tool_name='bash', state='RUNNING',
|
|
started_monotonic=started, cap_s=10)
|
|
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
|
|
done = dict(start, state='DONE', observed_monotonic=started + 1)
|
|
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
|
|
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
|
|
original = prefix + start_line + good + done_line
|
|
path.write_bytes(original)
|
|
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
|
|
with self.first_reader_bytes(path, stale) as readers, \
|
|
patch.object(execution.time, 'monotonic', return_value=started + 11), \
|
|
patch.object(collection_runtime, 'broker') as broker, \
|
|
self.assertLogs(level='WARNING') as logs:
|
|
self.assertIsNone(observer.poll())
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
broker.assert_not_called()
|
|
self.assertEqual(len(logs.output), 1)
|
|
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
|
|
self.assertNotIn('完整原始证据', logs.output[0])
|
|
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
|
|
self.assertEqual(observer.offset, len(original))
|
|
self.assertEqual(path.read_bytes(), original)
|
|
|
|
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
|
|
from harness import collection_runtime
|
|
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
|
|
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
|
|
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot()
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
|
|
started_monotonic=time.monotonic(), cap_s=10)
|
|
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
|
|
observer.path.write_bytes(raw)
|
|
with self.first_reader_bytes(observer.path, raw) as readers, \
|
|
self.assertLogs(level='WARNING'), self.assertRaises(error):
|
|
observer.poll()
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
self.assertEqual(observer.offset, 0)
|
|
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
|
|
self.assertEqual(observer.path.read_bytes(), raw)
|
|
|
|
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
original = b'{"type":"note","value":"complete JSON but no newline"}'
|
|
observer.path.write_bytes(original)
|
|
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
|
|
self.assertLogs(level='WARNING'), \
|
|
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
|
|
observer.poll()
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
self.assertEqual(observer.offset, 0)
|
|
self.assertEqual(observer.path.read_bytes(), original)
|
|
|
|
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot()
|
|
clock = [time.monotonic()]
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='completed', tool_name='bash', state='RUNNING',
|
|
started_monotonic=clock[0], cap_s=10)
|
|
observer.path.write_text(json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n')
|
|
ready = threading.Event()
|
|
observe = owner.observe_tools
|
|
def waiting(events):
|
|
ready.set()
|
|
return observe(events)
|
|
with patch.object(owner, 'observe_tools', side_effect=waiting), \
|
|
patch.object(execution.time, 'monotonic', side_effect=lambda: clock[0]), \
|
|
patch.object(collection_runtime, 'broker') as broker, \
|
|
ThreadPoolExecutor(max_workers=1) as pool:
|
|
with owner.locked():
|
|
result = pool.submit(observer.poll)
|
|
self.assertTrue(ready.wait(2))
|
|
done = dict(start, state='DONE', observed_monotonic=clock[0] + 1)
|
|
with observer.path.open('a') as stream:
|
|
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
|
|
clock[0] += 11
|
|
self.assertIsNone(result.result(timeout=2))
|
|
broker.assert_not_called()
|
|
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
|
|
self.assertEqual(observer.offset, observer.path.stat().st_size)
|
|
|
|
def test_failed_apply_keeps_cursor_for_retry(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
observer.path.write_text('{"type":"tool_execution_start","bench_tool":{"state":"RUNNING"}}\n')
|
|
def fail_after_read(events):
|
|
list(events)
|
|
raise OSError('fixture commit failed')
|
|
with patch.object(owner, 'observe_tools', side_effect=fail_after_read):
|
|
with self.assertRaises(OSError):
|
|
observer.poll()
|
|
self.assertEqual(observer.offset, 0)
|
|
|
|
def test_partial_boundary_retried_and_late_done_still_times_out(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='late', tool_name='bash', state='RUNNING',
|
|
started_monotonic=started, cap_s=10)
|
|
raw = json.dumps({'type': 'tool_execution_start', 'bench_tool': event})
|
|
observer.path.write_text(raw)
|
|
self.assertIsNone(observer.poll())
|
|
self.assertEqual(observer.offset, 0)
|
|
with observer.path.open('a') as stream:
|
|
stream.write('\n')
|
|
self.assertIsNone(observer.poll())
|
|
done = dict(event, state='DONE', observed_monotonic=started + 11)
|
|
with observer.path.open('a') as stream:
|
|
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
|
|
with patch.object(collection_runtime, 'broker') as broker:
|
|
self.assertEqual(observer.poll(), 'TOOL_DEADLINE_EXCEEDED')
|
|
broker.return_value.cancel.assert_called_once_with(state['execution_id'])
|
|
|
|
|
|
def tool_response(arguments):
|
|
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'ordinary-call',
|
|
'type': 'function', 'function': {'name': 'bash', 'arguments': json.dumps(arguments)}}]}
|
|
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
|
|
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
|
|
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
|
|
return 200, {'Content-Type': 'text/event-stream'}, (
|
|
''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks) + 'data: [DONE]\n\n').encode()
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and Pi SDK required')
|
|
class OrdinaryToolDeadline(unittest.TestCase):
|
|
@contextmanager
|
|
def collection(self, arguments, cap=.12):
|
|
fixture = test_collection_crash.CollectionCrash()
|
|
try:
|
|
fixture.setup_collection(lambda n, body: tool_response(arguments) if n == 1 else ok(n, body))
|
|
cfg = paths.config()
|
|
cfg['rollouts']['tools'] = ['bash']
|
|
cfg['collection_v1']['lifecycle']['ordinary_tool_deadline_s'] = cap
|
|
# Fixture mutates policy before the first real Execution is created.
|
|
from harness import collection_runtime
|
|
collection_runtime._configuration = json.dumps(cfg['collection_v1'], sort_keys=True)
|
|
original_prepare = fixture.prepare
|
|
def prepare(*args, **kwargs):
|
|
rd, job, prompt = original_prepare(*args, **kwargs)
|
|
job['gates'] = {'required': ['G1', 'G3'], 'G1': {}, 'G2': {}, 'G4': {},
|
|
'G3': {'metric': 'top1', 'kind': 'rel_pct', 'tol': 5, 'require_source': 'board'}}
|
|
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
|
|
return rd, job, prompt
|
|
from harness.verify import verify_rollout
|
|
with patch.object(rollout, 'prepare', side_effect=prepare), patch.object(rollout, 'verify_rollout', verify_rollout):
|
|
yield fixture
|
|
finally:
|
|
fixture.doCleanups()
|
|
|
|
def test_requested_timeout_cannot_bypass_frozen_cap(self):
|
|
receipts = []
|
|
for timeout in (None, 0, -1, 9999):
|
|
arguments = {'command': 'sleep .6'}
|
|
if timeout is not None:
|
|
arguments['timeout'] = timeout
|
|
with self.subTest(timeout=timeout), self.collection(arguments) as fixture:
|
|
if timeout is None:
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
started = time.monotonic()
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertEqual(fixture.rows()[0]['state'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
|
|
# A tool-cap stop no longer holds the queued repeat.
|
|
self.assertEqual([r['kk'] for r in pipeline._collection_snapshot('r', None, None)[0]],
|
|
[1] if timeout is None else [])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
if timeout is None:
|
|
self.assertEqual(fixture.rows()[1]['state'], 'QUEUED')
|
|
self.assertFalse((rd.parent / '01').exists())
|
|
source = trajectory.freeze(rd)
|
|
view = trajectory.reconstruct(source)
|
|
quality = trajectory.validate(source, view)
|
|
self.assertTrue(quality['pass'], quality)
|
|
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
|
|
self.assertEqual(len(results), 1)
|
|
self.assertEqual(results[0]['toolCallId'], 'ordinary-call')
|
|
self.assertTrue(results[0]['isError'])
|
|
self.assertEqual(results[0]['details']['bench_stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
|
|
self.assertEqual(results, [v['message'] for v in raw if v.get('type') == 'message_end'
|
|
and v['message']['role'] == 'toolResult'])
|
|
observer = monitor.Observer(fixture.project)
|
|
observer.sample(); detail = observer.detail('r', 'j0', 0)
|
|
self.assertEqual(detail['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(detail['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
self.assertEqual(detail['activity_details']['cap_s'], .12)
|
|
self.assertEqual(detail['activity_details']['tool_call_id'], 'ordinary-call')
|
|
receipts.append({'requested': timeout, 'outcome': state['execution_outcome']})
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-cap', 'variants': receipts}))
|
|
|
|
def test_short_silent_tool_and_shorter_native_timeout_keep_original_semantics(self):
|
|
for arguments, error in [({'command': 'sleep .02'}, False),
|
|
({'command': 'sleep .6', 'timeout': .04}, True)]:
|
|
with self.subTest(arguments=arguments), self.collection(arguments, cap=1) as fixture:
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_outcome'], 'COMPLETED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(len(fixture.joint.provider.requests), 2)
|
|
view = trajectory.reconstruct(trajectory.freeze(rd))
|
|
self.assertTrue(trajectory.validate(trajectory.freeze(rd), view)['pass'])
|
|
result = next(m for m in view['messages'] if m.get('role') == 'toolResult')
|
|
self.assertEqual(result['isError'], error)
|
|
self.assertNotIn('bench_stop_reason', result.get('details', {}))
|
|
# An old execution without a frozen cap cannot silently resume
|
|
# under the new policy; its source/session must remain untouched.
|
|
from harness import collection_runtime
|
|
state['config'].pop('ordinary_tool_deadline_s')
|
|
execution.atomic_json(rd / 'execution.json', state)
|
|
original = (rd / 'rollout.json').read_bytes()
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
|
|
collection_runtime.configure_pi(rd, execution.Execution(rd).attach(), 'teacher', resume=True)
|
|
self.assertEqual((rd / 'rollout.json').read_bytes(), original)
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-native-timeout',
|
|
'short_silent_completed': True, 'shorter_native_timeout_can_continue': True}))
|
|
|
|
def test_parent_deadline_reaps_detached_child_even_when_sdk_is_stopped(self):
|
|
# The child clears its environment and starts a new session; no marker or
|
|
# process-group scan could prove this set empty. Only the owned subreaper can.
|
|
program = ('import json,subprocess,sys;from pathlib import Path;'
|
|
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
|
|
'env={},start_new_session=True);'
|
|
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
|
|
'Path("detached.json").write_text(json.dumps({"pid":p.pid,"start":int(s[19])}))')
|
|
command = shlex.join([sys.executable, '-c', program]) + '; kill -STOP "$PPID"; sleep 20'
|
|
with self.collection({'command': command}, cap=.15) as fixture:
|
|
started = time.monotonic()
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
child = json.loads((rd / 'workspace/detached.json').read_text())
|
|
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
|
|
self.assertEqual(execution.process_status(identity), 'DEAD')
|
|
for scope in state['process_scopes'].values():
|
|
receipt = execution.scope_receipt(scope)
|
|
self.assertEqual(receipt['binding']['proof'], 'waitpid:ECHILD')
|
|
self.assertTrue(receipt['all_children_reaped'])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
|
|
checks = trajectory.validate(source, view)['checks']
|
|
self.assertTrue(checks['VALID_MESSAGE_COVERAGE']['pass'])
|
|
self.assertFalse(checks['CALL_TOOL_CLOSURE']['pass'])
|
|
self.assertFalse(any(m.get('role') == 'toolResult' for m in view['messages']))
|
|
from export import to_agentrl
|
|
stats = to_agentrl.materialize(['r'], 'stopped-sdk')['stats']
|
|
self.assertEqual(stats['sft_eligible'], 0)
|
|
self.assertEqual([r['state'] for r in fixture.rows()], ['DEADLINE_EXCEEDED'])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-parent-fallback',
|
|
'sdk_stopped': True, 'detached_child': 'DEAD', 'cleanup': 'VERIFIED',
|
|
'proof': 'waitpid:ECHILD', 'tool_result_synthesized': False, 'sft_eligible': 0}))
|
|
|
|
def test_cancel_and_deadline_use_first_committed_stop(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for first in ('cancel', 'deadline'):
|
|
owner = execution.Execution(Path(tmp) / first)
|
|
owner.start(policy())
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='t1', tool_name='bash', started_monotonic=started, cap_s=.01, state='RUNNING')
|
|
owner.observe_tools([event])
|
|
if first == 'cancel': owner.cancel('user request')
|
|
time.sleep(.02)
|
|
owner.observe_tools([])
|
|
if first == 'deadline': owner.cancel('late user request')
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with owner.admission(): pass
|
|
final = owner.finish(0, protocol_complete=False)
|
|
self.assertEqual(final['execution_outcome'], 'CANCELLED' if first == 'cancel' else 'DEADLINE_EXCEEDED')
|
|
|
|
def test_reused_call_id_fences_stopped_sdk_without_rewriting_first_result(self):
|
|
from harness import collection_runtime
|
|
program = ('import json,subprocess,sys;from pathlib import Path;'
|
|
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
|
|
'env={},start_new_session=True);'
|
|
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
|
|
'Path("reused-detached.json").write_text(json.dumps('
|
|
'{"pid":p.pid,"start":int(s[19]),"sdk_pid":int(sys.argv[1])}))')
|
|
command = shlex.join([sys.executable, '-c', program]) + ' "$PPID"; kill -STOP "$PPID"; sleep 20'
|
|
with self.collection({'command': 'true'}, cap=.2) as fixture:
|
|
fixture.joint.provider.responder = lambda n, body: (
|
|
tool_response({'command': 'true'}) if n == 1 else
|
|
tool_response({'command': command}) if n == 2 else ok(n, body))
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
fallback_cleanup = False
|
|
poll = collection_runtime.ToolDeadlines.poll
|
|
barrier = {'released': False, 'sdk_stopped': False}
|
|
def after_sdk_stop(observer):
|
|
# Force the difficult interleaving: consume the second start only
|
|
# after the owned SDK is stopped and its detached child exists.
|
|
if not barrier['released']:
|
|
until = time.monotonic() + 3
|
|
while time.monotonic() < until:
|
|
try:
|
|
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
|
|
stat = Path(f"/proc/{child['sdk_pid']}/stat").read_text().rsplit(')', 1)[1].split()
|
|
owned = observer.execution.snapshot()['processes']
|
|
if stat[0] == 'T' and any(p['role'] == 'agent' and p['pid'] == child['sdk_pid']
|
|
and p['proc_start_ticks'] == int(stat[19]) for p in owned):
|
|
barrier['sdk_stopped'] = True
|
|
break
|
|
except (OSError, ValueError):
|
|
pass # The fixture has not committed the birth record yet.
|
|
time.sleep(.01)
|
|
barrier['released'] = True
|
|
return poll(observer)
|
|
with patch.object(collection_runtime.ToolDeadlines, 'poll', new=after_sdk_stop), \
|
|
ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(pipeline.collect_rollouts, 'r', limit=1, log=lambda _: None)
|
|
try:
|
|
future.result(timeout=5)
|
|
except TimeoutError:
|
|
fallback_cleanup = True
|
|
finally:
|
|
if not future.done():
|
|
rollout.settle_execution('r', 'j0', 0, execution.Execution(rd).attach(),
|
|
'test-owned repeated-call cleanup', cancel=True)
|
|
future.result(timeout=6)
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertTrue(barrier['sdk_stopped'], 'fixture never observed the owned SDK in SIGSTOP')
|
|
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
|
|
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
|
|
self.assertEqual(execution.process_status(identity), 'DEAD')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
for scope in state['process_scopes'].values():
|
|
self.assertEqual(execution.scope_receipt(scope)['binding']['proof'], 'waitpid:ECHILD')
|
|
self.assertFalse(fallback_cleanup, 'reused call ID escaped its parent-side bound')
|
|
self.assertEqual(state['execution_outcome'], 'INFRA_ERROR')
|
|
self.assertEqual(state['stop_reason'], 'INFRA_TOOL_IDENTITY_CONFLICT')
|
|
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
|
|
starts = [e['bench_tool'] for e in raw if e.get('bench_tool', {}).get('state') == 'RUNNING']
|
|
self.assertEqual(len(starts), 2)
|
|
self.assertEqual({s['tool_call_id'] for s in starts}, {'ordinary-call'})
|
|
self.assertNotEqual(starts[0]['started_monotonic'], starts[1]['started_monotonic'])
|
|
first = state['tool_calls']['ordinary-call']
|
|
self.assertEqual(first['state'], 'DONE')
|
|
self.assertEqual(first['started_monotonic'], starts[0]['started_monotonic'])
|
|
self.assertEqual(first['deadline_monotonic'], starts[0]['deadline_monotonic'])
|
|
self.assertEqual(state['tool_conflict'], starts[1])
|
|
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
|
|
quality = trajectory.validate(source, view)
|
|
self.assertTrue(quality['checks']['VALID_MESSAGE_COVERAGE']['pass'])
|
|
self.assertFalse(quality['checks']['CALL_TOOL_CLOSURE']['pass'])
|
|
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
|
|
self.assertEqual(len(results), 1)
|
|
self.assertEqual(results, [e['message'] for e in raw if e.get('type') == 'message_end'
|
|
and e['message']['role'] == 'toolResult'])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 2)
|
|
self.assertEqual([r['state'] for r in fixture.rows()], ['INFRA_ERROR', 'QUEUED'])
|
|
# A system stop no longer holds the queued repeat.
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-reused-id',
|
|
'real_sdk_generations': 2, 'sdk_stopped': True, 'detached_child': 'DEAD',
|
|
'cleanup': 'VERIFIED', 'proof': 'waitpid:ECHILD', 'first_result_preserved': True,
|
|
'second_result_synthesized': False, 'repeat_claimable': True,
|
|
'outcome': state['execution_outcome'], 'reason': state['stop_reason']}))
|
|
|
|
def test_tool_occurrence_duplicates_never_reset_or_close_another_occurrence(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for variant in ('duplicate_start', 'new_start', 'foreign_end'):
|
|
with self.subTest(variant=variant):
|
|
owner = execution.Execution(Path(tmp) / variant); owner.start(policy())
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='t1', tool_name='bash', started_monotonic=started,
|
|
deadline_monotonic=started + .03, cap_s=.03, state='RUNNING')
|
|
owner.observe_tools([event])
|
|
original = owner.snapshot()['tool_calls']['t1']
|
|
if variant == 'duplicate_start':
|
|
time.sleep(.02)
|
|
self.assertIsNone(owner.observe_tools([event]))
|
|
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
|
|
time.sleep(.02)
|
|
self.assertEqual(owner.observe_tools([]), 'TOOL_DEADLINE_EXCEEDED')
|
|
else:
|
|
other = dict(event, started_monotonic=started + .001,
|
|
deadline_monotonic=started + .031)
|
|
if variant == 'foreign_end':
|
|
other.update(state='DONE', observed_monotonic=time.monotonic())
|
|
self.assertEqual(owner.observe_tools([other]), 'INFRA_TOOL_IDENTITY_CONFLICT')
|
|
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
|
|
self.assertEqual(owner.snapshot()['tool_conflict'], other)
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with owner.admission(): pass
|
|
|
|
def test_verify_preserves_all_control_holds_and_queued_repeats(self):
|
|
from harness.verify import verify_rollout
|
|
with self.collection({'command': 'true'}, cap=1) as fixture:
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
owner = execution.Execution(rd).attach()
|
|
original = owner.snapshot()
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
measurements = verify_rollout(rd, write=False)['gates']
|
|
variants = [('TERMINAL', outcome, 'VERIFIED', outcome) for outcome in
|
|
('CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST')]
|
|
variants += [('PAUSED_INFRA', None, 'VERIFIED', 'PAUSED_INFRA'),
|
|
('CLEANING', None, 'PENDING', 'CLEANING'),
|
|
('ACTIVE', None, 'PENDING', 'RUNNING'),
|
|
('TERMINAL', 'AGENT_ERROR', 'QUARANTINED', 'INFRA_ERROR')]
|
|
for status, outcome, cleanup, expected in variants:
|
|
with self.subTest(status=status, outcome=outcome, cleanup=cleanup):
|
|
execution.atomic_json(owner.path, dict(original, execution_state=status,
|
|
execution_outcome=outcome, cleanup_state=cleanup))
|
|
for _ in range(2):
|
|
verdict = verify_rollout(rd)
|
|
self.assertEqual(verdict['gates'], measurements)
|
|
self.assertEqual(verdict['terminal'], 'UNVERIFIED')
|
|
self.assertEqual(verdict['execution_control']['db_state'], expected)
|
|
self.assertEqual(fixture.rows()[0]['state'], expected)
|
|
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
|
|
held = expected in ('BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING')
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]],
|
|
[] if held else [('j0', 1)])
|
|
for fault in ('mismatch', 'malformed', 'missing'):
|
|
if fault == 'mismatch':
|
|
execution.atomic_json(owner.path, dict(original, execution_id='different'))
|
|
elif fault == 'malformed':
|
|
execution.atomic_json(owner.path, [])
|
|
else:
|
|
owner.path.unlink() # execution_managed source must fail closed.
|
|
verdict = verify_rollout(rd)
|
|
self.assertEqual(verdict['gates'], measurements)
|
|
self.assertEqual(fixture.rows()[0]['state'], 'INFRA_ERROR')
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'verify-control-holds',
|
|
'variants': len(variants), 'identity_faults': 3, 'hard_gates_unchanged': True,
|
|
'repeat_held_only_for': ['BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING']}))
|
|
|
|
def test_legacy_resume_is_rejected_before_execution_quota_or_db_changes(self):
|
|
from harness import cli, collection_runtime as runtime
|
|
fixture = test_collection_crash.CollectionCrash()
|
|
try:
|
|
fixture.setup_collection(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
|
|
self.assertEqual(pipeline.collect_rollouts('r', limit=1, log=lambda _: None), {'PAUSED_INFRA': 1})
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
owner = execution.Execution(rd).attach()
|
|
state = owner.snapshot(); state['config'].pop('ordinary_tool_deadline_s')
|
|
execution.atomic_json(owner.path, state)
|
|
originals = {p: p.read_bytes() for p in (owner.path, rd / 'rollout.json',
|
|
rd / 'agent/session-checkpoint.json', rd / 'agent/native-session.jsonl')}
|
|
rows = fixture.rows()
|
|
with fixture.joint.broker._db() as con:
|
|
ledger = list(con.iterdump())
|
|
with patch.object(runtime, 'broker', side_effect=AssertionError('broker touched before policy check')):
|
|
for action in [owner.resume, lambda: rollout.resume_rollout('r', 'j0', 0),
|
|
lambda: cli.main(['execution', 'resume', '--run', 'r', '--job', 'j0', '--kk', '0']),
|
|
lambda: runtime.configure_pi(rd, owner, 'teacher', resume=True)]:
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
|
|
action()
|
|
self.assertEqual(pipeline._resume_snapshot('r'), ([], False))
|
|
self.assertEqual(fixture.rows(), rows)
|
|
self.assertEqual({p: p.read_bytes() for p in originals}, originals)
|
|
with fixture.joint.broker._db() as con:
|
|
self.assertEqual(list(con.iterdump()), ledger)
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'legacy-resume-policy',
|
|
'entrypoints': ['execution', 'rollout', 'CLI', 'configure', 'collector'],
|
|
'source_and_db_unchanged': True, 'broker_calls': 0}))
|
|
finally:
|
|
fixture.doCleanups()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|