data-gen-bpu/scripts/test_ordinary_tool_deadline.py
ming01.wang 10b070e05d fix: stop holding a task's queued rollouts after a system stop
A queued rollout was never claimed once any sibling ended DEADLINE_EXCEEDED,
CANCELLED, INFRA_ERROR, LOST or BUDGET_EXHAUSTED. Almost all of those were
system stops (tool cap, context rejected by the local quota, board cleanup
or request outcome unknown, pauses that expired, operator restarts), so
1107 queued rollouts across r20260929-04/05/06 sat behind them.

Siblings still run one at a time, and two stops still hold the rest: an
exhausted turn/token budget, and a DEADLINE_EXCEEDED whose verdict says
the trajectory really worked until the 24 h execution deadline. Agent-
finished FAIL/UNVERIFIED keep not holding siblings, as before.

Tests that pinned the old hold now assert the repeat is claimable; the
lease-recovery waves no longer depend on sibling holds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 09:30:27 +08:00

562 lines
35 KiB
Python

"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
import io
import json
import shlex
import sys
import tempfile
import time
import threading
import unittest
from contextlib import contextmanager
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from pathlib import Path
from unittest.mock import patch
from export import trajectory
from harness import db, execution, monitor, paths, pipeline, rollout
from scripts import test_collection_crash, test_pi_session
from scripts.test_execution import policy
from scripts.test_quota import ok
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
class ToolBoundaryCommit(unittest.TestCase):
@contextmanager
def first_reader_bytes(self, path, raw):
"""Expose a stale first read without changing the real event evidence."""
original = Path.open
opened = []
def opening(target, mode='r', *args, **kwargs):
if target == path and mode == 'rb':
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
opened.append(stream)
return stream
return original(target, mode, *args, **kwargs)
with patch.object(Path, 'open', new=opening):
yield opened
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
from harness import collection_runtime
for bad in (b'{"type":"note","value":"\x00"}\n',
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
path = rd / 'agent/events.jsonl'
prefix = b'{"type":"already_consumed"}\n'
path.write_bytes(prefix)
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot(); started = time.monotonic()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='completed', tool_name='bash', state='RUNNING',
started_monotonic=started, cap_s=10)
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
done = dict(start, state='DONE', observed_monotonic=started + 1)
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
original = prefix + start_line + good + done_line
path.write_bytes(original)
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
with self.first_reader_bytes(path, stale) as readers, \
patch.object(execution.time, 'monotonic', return_value=started + 11), \
patch.object(collection_runtime, 'broker') as broker, \
self.assertLogs(level='WARNING') as logs:
self.assertIsNone(observer.poll())
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
broker.assert_not_called()
self.assertEqual(len(logs.output), 1)
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
self.assertNotIn('完整原始证据', logs.output[0])
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
self.assertEqual(observer.offset, len(original))
self.assertEqual(path.read_bytes(), original)
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
from harness import collection_runtime
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
started_monotonic=time.monotonic(), cap_s=10)
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
observer.path.write_bytes(raw)
with self.first_reader_bytes(observer.path, raw) as readers, \
self.assertLogs(level='WARNING'), self.assertRaises(error):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
self.assertEqual(observer.path.read_bytes(), raw)
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
original = b'{"type":"note","value":"complete JSON but no newline"}'
observer.path.write_bytes(original)
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
self.assertLogs(level='WARNING'), \
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(observer.path.read_bytes(), original)
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot()
clock = [time.monotonic()]
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='completed', tool_name='bash', state='RUNNING',
started_monotonic=clock[0], cap_s=10)
observer.path.write_text(json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n')
ready = threading.Event()
observe = owner.observe_tools
def waiting(events):
ready.set()
return observe(events)
with patch.object(owner, 'observe_tools', side_effect=waiting), \
patch.object(execution.time, 'monotonic', side_effect=lambda: clock[0]), \
patch.object(collection_runtime, 'broker') as broker, \
ThreadPoolExecutor(max_workers=1) as pool:
with owner.locked():
result = pool.submit(observer.poll)
self.assertTrue(ready.wait(2))
done = dict(start, state='DONE', observed_monotonic=clock[0] + 1)
with observer.path.open('a') as stream:
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
clock[0] += 11
self.assertIsNone(result.result(timeout=2))
broker.assert_not_called()
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
self.assertEqual(observer.offset, observer.path.stat().st_size)
def test_failed_apply_keeps_cursor_for_retry(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
observer.path.write_text('{"type":"tool_execution_start","bench_tool":{"state":"RUNNING"}}\n')
def fail_after_read(events):
list(events)
raise OSError('fixture commit failed')
with patch.object(owner, 'observe_tools', side_effect=fail_after_read):
with self.assertRaises(OSError):
observer.poll()
self.assertEqual(observer.offset, 0)
def test_partial_boundary_retried_and_late_done_still_times_out(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='late', tool_name='bash', state='RUNNING',
started_monotonic=started, cap_s=10)
raw = json.dumps({'type': 'tool_execution_start', 'bench_tool': event})
observer.path.write_text(raw)
self.assertIsNone(observer.poll())
self.assertEqual(observer.offset, 0)
with observer.path.open('a') as stream:
stream.write('\n')
self.assertIsNone(observer.poll())
done = dict(event, state='DONE', observed_monotonic=started + 11)
with observer.path.open('a') as stream:
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
with patch.object(collection_runtime, 'broker') as broker:
self.assertEqual(observer.poll(), 'TOOL_DEADLINE_EXCEEDED')
broker.return_value.cancel.assert_called_once_with(state['execution_id'])
def tool_response(arguments):
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'ordinary-call',
'type': 'function', 'function': {'name': 'bash', 'arguments': json.dumps(arguments)}}]}
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
return 200, {'Content-Type': 'text/event-stream'}, (
''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks) + 'data: [DONE]\n\n').encode()
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and Pi SDK required')
class OrdinaryToolDeadline(unittest.TestCase):
@contextmanager
def collection(self, arguments, cap=.12):
fixture = test_collection_crash.CollectionCrash()
try:
fixture.setup_collection(lambda n, body: tool_response(arguments) if n == 1 else ok(n, body))
cfg = paths.config()
cfg['rollouts']['tools'] = ['bash']
cfg['collection_v1']['lifecycle']['ordinary_tool_deadline_s'] = cap
# Fixture mutates policy before the first real Execution is created.
from harness import collection_runtime
collection_runtime._configuration = json.dumps(cfg['collection_v1'], sort_keys=True)
original_prepare = fixture.prepare
def prepare(*args, **kwargs):
rd, job, prompt = original_prepare(*args, **kwargs)
job['gates'] = {'required': ['G1', 'G3'], 'G1': {}, 'G2': {}, 'G4': {},
'G3': {'metric': 'top1', 'kind': 'rel_pct', 'tol': 5, 'require_source': 'board'}}
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
return rd, job, prompt
from harness.verify import verify_rollout
with patch.object(rollout, 'prepare', side_effect=prepare), patch.object(rollout, 'verify_rollout', verify_rollout):
yield fixture
finally:
fixture.doCleanups()
def test_requested_timeout_cannot_bypass_frozen_cap(self):
receipts = []
for timeout in (None, 0, -1, 9999):
arguments = {'command': 'sleep .6'}
if timeout is not None:
arguments['timeout'] = timeout
with self.subTest(timeout=timeout), self.collection(arguments) as fixture:
if timeout is None:
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
started = time.monotonic()
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(state['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual(len(fixture.joint.provider.requests), 1)
self.assertLess(time.monotonic() - started, 5)
self.assertEqual(fixture.rows()[0]['state'], 'DEADLINE_EXCEEDED')
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
# A tool-cap stop no longer holds the queued repeat.
self.assertEqual([r['kk'] for r in pipeline._collection_snapshot('r', None, None)[0]],
[1] if timeout is None else [])
self.assertEqual(len(fixture.joint.provider.requests), 1)
if timeout is None:
self.assertEqual(fixture.rows()[1]['state'], 'QUEUED')
self.assertFalse((rd.parent / '01').exists())
source = trajectory.freeze(rd)
view = trajectory.reconstruct(source)
quality = trajectory.validate(source, view)
self.assertTrue(quality['pass'], quality)
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['toolCallId'], 'ordinary-call')
self.assertTrue(results[0]['isError'])
self.assertEqual(results[0]['details']['bench_stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
self.assertEqual(results, [v['message'] for v in raw if v.get('type') == 'message_end'
and v['message']['role'] == 'toolResult'])
observer = monitor.Observer(fixture.project)
observer.sample(); detail = observer.detail('r', 'j0', 0)
self.assertEqual(detail['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(detail['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
self.assertEqual(detail['activity_details']['cap_s'], .12)
self.assertEqual(detail['activity_details']['tool_call_id'], 'ordinary-call')
receipts.append({'requested': timeout, 'outcome': state['execution_outcome']})
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-cap', 'variants': receipts}))
def test_short_silent_tool_and_shorter_native_timeout_keep_original_semantics(self):
for arguments, error in [({'command': 'sleep .02'}, False),
({'command': 'sleep .6', 'timeout': .04}, True)]:
with self.subTest(arguments=arguments), self.collection(arguments, cap=1) as fixture:
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertEqual(state['execution_outcome'], 'COMPLETED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(len(fixture.joint.provider.requests), 2)
view = trajectory.reconstruct(trajectory.freeze(rd))
self.assertTrue(trajectory.validate(trajectory.freeze(rd), view)['pass'])
result = next(m for m in view['messages'] if m.get('role') == 'toolResult')
self.assertEqual(result['isError'], error)
self.assertNotIn('bench_stop_reason', result.get('details', {}))
# An old execution without a frozen cap cannot silently resume
# under the new policy; its source/session must remain untouched.
from harness import collection_runtime
state['config'].pop('ordinary_tool_deadline_s')
execution.atomic_json(rd / 'execution.json', state)
original = (rd / 'rollout.json').read_bytes()
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
collection_runtime.configure_pi(rd, execution.Execution(rd).attach(), 'teacher', resume=True)
self.assertEqual((rd / 'rollout.json').read_bytes(), original)
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-native-timeout',
'short_silent_completed': True, 'shorter_native_timeout_can_continue': True}))
def test_parent_deadline_reaps_detached_child_even_when_sdk_is_stopped(self):
# The child clears its environment and starts a new session; no marker or
# process-group scan could prove this set empty. Only the owned subreaper can.
program = ('import json,subprocess,sys;from pathlib import Path;'
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
'env={},start_new_session=True);'
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
'Path("detached.json").write_text(json.dumps({"pid":p.pid,"start":int(s[19])}))')
command = shlex.join([sys.executable, '-c', program]) + '; kill -STOP "$PPID"; sleep 20'
with self.collection({'command': command}, cap=.15) as fixture:
started = time.monotonic()
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertLess(time.monotonic() - started, 5)
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
child = json.loads((rd / 'workspace/detached.json').read_text())
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
self.assertEqual(execution.process_status(identity), 'DEAD')
for scope in state['process_scopes'].values():
receipt = execution.scope_receipt(scope)
self.assertEqual(receipt['binding']['proof'], 'waitpid:ECHILD')
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(len(fixture.joint.provider.requests), 1)
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
checks = trajectory.validate(source, view)['checks']
self.assertTrue(checks['VALID_MESSAGE_COVERAGE']['pass'])
self.assertFalse(checks['CALL_TOOL_CLOSURE']['pass'])
self.assertFalse(any(m.get('role') == 'toolResult' for m in view['messages']))
from export import to_agentrl
stats = to_agentrl.materialize(['r'], 'stopped-sdk')['stats']
self.assertEqual(stats['sft_eligible'], 0)
self.assertEqual([r['state'] for r in fixture.rows()], ['DEADLINE_EXCEEDED'])
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-parent-fallback',
'sdk_stopped': True, 'detached_child': 'DEAD', 'cleanup': 'VERIFIED',
'proof': 'waitpid:ECHILD', 'tool_result_synthesized': False, 'sft_eligible': 0}))
def test_cancel_and_deadline_use_first_committed_stop(self):
with tempfile.TemporaryDirectory() as tmp:
for first in ('cancel', 'deadline'):
owner = execution.Execution(Path(tmp) / first)
owner.start(policy())
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='t1', tool_name='bash', started_monotonic=started, cap_s=.01, state='RUNNING')
owner.observe_tools([event])
if first == 'cancel': owner.cancel('user request')
time.sleep(.02)
owner.observe_tools([])
if first == 'deadline': owner.cancel('late user request')
with self.assertRaises(execution.ExecutionStopped):
with owner.admission(): pass
final = owner.finish(0, protocol_complete=False)
self.assertEqual(final['execution_outcome'], 'CANCELLED' if first == 'cancel' else 'DEADLINE_EXCEEDED')
def test_reused_call_id_fences_stopped_sdk_without_rewriting_first_result(self):
from harness import collection_runtime
program = ('import json,subprocess,sys;from pathlib import Path;'
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
'env={},start_new_session=True);'
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
'Path("reused-detached.json").write_text(json.dumps('
'{"pid":p.pid,"start":int(s[19]),"sdk_pid":int(sys.argv[1])}))')
command = shlex.join([sys.executable, '-c', program]) + ' "$PPID"; kill -STOP "$PPID"; sleep 20'
with self.collection({'command': 'true'}, cap=.2) as fixture:
fixture.joint.provider.responder = lambda n, body: (
tool_response({'command': 'true'}) if n == 1 else
tool_response({'command': command}) if n == 2 else ok(n, body))
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
rd = fixture.project / 'runs/r/j0/rollouts/00'
fallback_cleanup = False
poll = collection_runtime.ToolDeadlines.poll
barrier = {'released': False, 'sdk_stopped': False}
def after_sdk_stop(observer):
# Force the difficult interleaving: consume the second start only
# after the owned SDK is stopped and its detached child exists.
if not barrier['released']:
until = time.monotonic() + 3
while time.monotonic() < until:
try:
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
stat = Path(f"/proc/{child['sdk_pid']}/stat").read_text().rsplit(')', 1)[1].split()
owned = observer.execution.snapshot()['processes']
if stat[0] == 'T' and any(p['role'] == 'agent' and p['pid'] == child['sdk_pid']
and p['proc_start_ticks'] == int(stat[19]) for p in owned):
barrier['sdk_stopped'] = True
break
except (OSError, ValueError):
pass # The fixture has not committed the birth record yet.
time.sleep(.01)
barrier['released'] = True
return poll(observer)
with patch.object(collection_runtime.ToolDeadlines, 'poll', new=after_sdk_stop), \
ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(pipeline.collect_rollouts, 'r', limit=1, log=lambda _: None)
try:
future.result(timeout=5)
except TimeoutError:
fallback_cleanup = True
finally:
if not future.done():
rollout.settle_execution('r', 'j0', 0, execution.Execution(rd).attach(),
'test-owned repeated-call cleanup', cancel=True)
future.result(timeout=6)
state = execution.Execution(rd).snapshot()
self.assertTrue(barrier['sdk_stopped'], 'fixture never observed the owned SDK in SIGSTOP')
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
self.assertEqual(execution.process_status(identity), 'DEAD')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual(state['cleanup_state'], 'VERIFIED')
for scope in state['process_scopes'].values():
self.assertEqual(execution.scope_receipt(scope)['binding']['proof'], 'waitpid:ECHILD')
self.assertFalse(fallback_cleanup, 'reused call ID escaped its parent-side bound')
self.assertEqual(state['execution_outcome'], 'INFRA_ERROR')
self.assertEqual(state['stop_reason'], 'INFRA_TOOL_IDENTITY_CONFLICT')
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
starts = [e['bench_tool'] for e in raw if e.get('bench_tool', {}).get('state') == 'RUNNING']
self.assertEqual(len(starts), 2)
self.assertEqual({s['tool_call_id'] for s in starts}, {'ordinary-call'})
self.assertNotEqual(starts[0]['started_monotonic'], starts[1]['started_monotonic'])
first = state['tool_calls']['ordinary-call']
self.assertEqual(first['state'], 'DONE')
self.assertEqual(first['started_monotonic'], starts[0]['started_monotonic'])
self.assertEqual(first['deadline_monotonic'], starts[0]['deadline_monotonic'])
self.assertEqual(state['tool_conflict'], starts[1])
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
quality = trajectory.validate(source, view)
self.assertTrue(quality['checks']['VALID_MESSAGE_COVERAGE']['pass'])
self.assertFalse(quality['checks']['CALL_TOOL_CLOSURE']['pass'])
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
self.assertEqual(len(results), 1)
self.assertEqual(results, [e['message'] for e in raw if e.get('type') == 'message_end'
and e['message']['role'] == 'toolResult'])
self.assertEqual(len(fixture.joint.provider.requests), 2)
self.assertEqual([r['state'] for r in fixture.rows()], ['INFRA_ERROR', 'QUEUED'])
# A system stop no longer holds the queued repeat.
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-reused-id',
'real_sdk_generations': 2, 'sdk_stopped': True, 'detached_child': 'DEAD',
'cleanup': 'VERIFIED', 'proof': 'waitpid:ECHILD', 'first_result_preserved': True,
'second_result_synthesized': False, 'repeat_claimable': True,
'outcome': state['execution_outcome'], 'reason': state['stop_reason']}))
def test_tool_occurrence_duplicates_never_reset_or_close_another_occurrence(self):
with tempfile.TemporaryDirectory() as tmp:
for variant in ('duplicate_start', 'new_start', 'foreign_end'):
with self.subTest(variant=variant):
owner = execution.Execution(Path(tmp) / variant); owner.start(policy())
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='t1', tool_name='bash', started_monotonic=started,
deadline_monotonic=started + .03, cap_s=.03, state='RUNNING')
owner.observe_tools([event])
original = owner.snapshot()['tool_calls']['t1']
if variant == 'duplicate_start':
time.sleep(.02)
self.assertIsNone(owner.observe_tools([event]))
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
time.sleep(.02)
self.assertEqual(owner.observe_tools([]), 'TOOL_DEADLINE_EXCEEDED')
else:
other = dict(event, started_monotonic=started + .001,
deadline_monotonic=started + .031)
if variant == 'foreign_end':
other.update(state='DONE', observed_monotonic=time.monotonic())
self.assertEqual(owner.observe_tools([other]), 'INFRA_TOOL_IDENTITY_CONFLICT')
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
self.assertEqual(owner.snapshot()['tool_conflict'], other)
with self.assertRaises(execution.ExecutionStopped):
with owner.admission(): pass
def test_verify_preserves_all_control_holds_and_queued_repeats(self):
from harness.verify import verify_rollout
with self.collection({'command': 'true'}, cap=1) as fixture:
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
owner = execution.Execution(rd).attach()
original = owner.snapshot()
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
measurements = verify_rollout(rd, write=False)['gates']
variants = [('TERMINAL', outcome, 'VERIFIED', outcome) for outcome in
('CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST')]
variants += [('PAUSED_INFRA', None, 'VERIFIED', 'PAUSED_INFRA'),
('CLEANING', None, 'PENDING', 'CLEANING'),
('ACTIVE', None, 'PENDING', 'RUNNING'),
('TERMINAL', 'AGENT_ERROR', 'QUARANTINED', 'INFRA_ERROR')]
for status, outcome, cleanup, expected in variants:
with self.subTest(status=status, outcome=outcome, cleanup=cleanup):
execution.atomic_json(owner.path, dict(original, execution_state=status,
execution_outcome=outcome, cleanup_state=cleanup))
for _ in range(2):
verdict = verify_rollout(rd)
self.assertEqual(verdict['gates'], measurements)
self.assertEqual(verdict['terminal'], 'UNVERIFIED')
self.assertEqual(verdict['execution_control']['db_state'], expected)
self.assertEqual(fixture.rows()[0]['state'], expected)
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
held = expected in ('BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING')
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]],
[] if held else [('j0', 1)])
for fault in ('mismatch', 'malformed', 'missing'):
if fault == 'mismatch':
execution.atomic_json(owner.path, dict(original, execution_id='different'))
elif fault == 'malformed':
execution.atomic_json(owner.path, [])
else:
owner.path.unlink() # execution_managed source must fail closed.
verdict = verify_rollout(rd)
self.assertEqual(verdict['gates'], measurements)
self.assertEqual(fixture.rows()[0]['state'], 'INFRA_ERROR')
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
print('JOINT_RECEIPT ' + json.dumps({'case': 'verify-control-holds',
'variants': len(variants), 'identity_faults': 3, 'hard_gates_unchanged': True,
'repeat_held_only_for': ['BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING']}))
def test_legacy_resume_is_rejected_before_execution_quota_or_db_changes(self):
from harness import cli, collection_runtime as runtime
fixture = test_collection_crash.CollectionCrash()
try:
fixture.setup_collection(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
self.assertEqual(pipeline.collect_rollouts('r', limit=1, log=lambda _: None), {'PAUSED_INFRA': 1})
rd = fixture.project / 'runs/r/j0/rollouts/00'
owner = execution.Execution(rd).attach()
state = owner.snapshot(); state['config'].pop('ordinary_tool_deadline_s')
execution.atomic_json(owner.path, state)
originals = {p: p.read_bytes() for p in (owner.path, rd / 'rollout.json',
rd / 'agent/session-checkpoint.json', rd / 'agent/native-session.jsonl')}
rows = fixture.rows()
with fixture.joint.broker._db() as con:
ledger = list(con.iterdump())
with patch.object(runtime, 'broker', side_effect=AssertionError('broker touched before policy check')):
for action in [owner.resume, lambda: rollout.resume_rollout('r', 'j0', 0),
lambda: cli.main(['execution', 'resume', '--run', 'r', '--job', 'j0', '--kk', '0']),
lambda: runtime.configure_pi(rd, owner, 'teacher', resume=True)]:
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
action()
self.assertEqual(pipeline._resume_snapshot('r'), ([], False))
self.assertEqual(fixture.rows(), rows)
self.assertEqual({p: p.read_bytes() for p in originals}, originals)
with fixture.joint.broker._db() as con:
self.assertEqual(list(con.iterdump()), ledger)
print('JOINT_RECEIPT ' + json.dumps({'case': 'legacy-resume-policy',
'entrypoints': ['execution', 'rollout', 'CLI', 'configure', 'collector'],
'source_and_db_unchanged': True, 'broker_calls': 0}))
finally:
fixture.doCleanups()
if __name__ == '__main__':
unittest.main()