data-gen-bpu/scripts/test_ordinary_tool_deadline.py
bench f191baf520 暂停轨迹在等时这一轮只续跑不领新题;额度组暂停或对账中一律算在等
10-15 11:21、11:33:一条暂停轨迹续上的同一轮里,新题跟着进来,把还在等额度的几条的名额占了。
它们只能等别的轨迹结束才能排上,期间 24 小时时钟照走(两条自带图的只剩 7.5 小时)。
- _claimable:有暂停轨迹已就绪或在等额度时,这一轮只续跑;都续上了下一轮再领新题。
- _resume_snapshot:额度组 PAUSED/RECONCILING 不论原因都算在等(00:56 是连接被切,原来不算)。
collector 重启后生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 11:43:53 +08:00

572 lines
36 KiB
Python

"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
import io
import json
import shlex
import sys
import tempfile
import time
import threading
import unittest
from contextlib import contextmanager
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from pathlib import Path
from unittest.mock import patch
from export import trajectory
from harness import db, execution, monitor, paths, pipeline, rollout
from scripts import test_collection_crash, test_pi_session
from scripts.test_execution import policy
from scripts.test_quota import ok
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
class ToolBoundaryCommit(unittest.TestCase):
@contextmanager
def first_reader_bytes(self, path, raw):
"""Expose a stale first read without changing the real event evidence."""
original = Path.open
opened = []
def opening(target, mode='r', *args, **kwargs):
if target == path and mode == 'rb':
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
opened.append(stream)
return stream
return original(target, mode, *args, **kwargs)
with patch.object(Path, 'open', new=opening):
yield opened
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
from harness import collection_runtime
for bad in (b'{"type":"note","value":"\x00"}\n',
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
path = rd / 'agent/events.jsonl'
prefix = b'{"type":"already_consumed"}\n'
path.write_bytes(prefix)
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot(); started = time.monotonic()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='completed', tool_name='bash', state='RUNNING',
started_monotonic=started, cap_s=10)
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
done = dict(start, state='DONE', observed_monotonic=started + 1)
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
original = prefix + start_line + good + done_line
path.write_bytes(original)
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
with self.first_reader_bytes(path, stale) as readers, \
patch.object(execution.time, 'monotonic', return_value=started + 11), \
patch.object(collection_runtime, 'broker') as broker, \
self.assertLogs(level='WARNING') as logs:
self.assertIsNone(observer.poll())
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
broker.assert_not_called()
self.assertEqual(len(logs.output), 1)
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
self.assertNotIn('完整原始证据', logs.output[0])
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
self.assertEqual(observer.offset, len(original))
self.assertEqual(path.read_bytes(), original)
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
from harness import collection_runtime
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
started_monotonic=time.monotonic(), cap_s=10)
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
observer.path.write_bytes(raw)
with self.first_reader_bytes(observer.path, raw) as readers, \
self.assertLogs(level='WARNING'), self.assertRaises(error):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
self.assertEqual(observer.path.read_bytes(), raw)
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
original = b'{"type":"note","value":"complete JSON but no newline"}'
observer.path.write_bytes(original)
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
self.assertLogs(level='WARNING'), \
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(observer.path.read_bytes(), original)
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot()
clock = [time.monotonic()]
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='completed', tool_name='bash', state='RUNNING',
started_monotonic=clock[0], cap_s=10)
observer.path.write_text(json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n')
ready = threading.Event()
observe = owner.observe_tools
def waiting(events):
ready.set()
return observe(events)
with patch.object(owner, 'observe_tools', side_effect=waiting), \
patch.object(execution.time, 'monotonic', side_effect=lambda: clock[0]), \
patch.object(collection_runtime, 'broker') as broker, \
ThreadPoolExecutor(max_workers=1) as pool:
with owner.locked():
result = pool.submit(observer.poll)
self.assertTrue(ready.wait(2))
done = dict(start, state='DONE', observed_monotonic=clock[0] + 1)
with observer.path.open('a') as stream:
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
clock[0] += 11
self.assertIsNone(result.result(timeout=2))
broker.assert_not_called()
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
self.assertEqual(observer.offset, observer.path.stat().st_size)
def test_failed_apply_keeps_cursor_for_retry(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
observer.path.write_text('{"type":"tool_execution_start","bench_tool":{"state":"RUNNING"}}\n')
def fail_after_read(events):
list(events)
raise OSError('fixture commit failed')
with patch.object(owner, 'observe_tools', side_effect=fail_after_read):
with self.assertRaises(OSError):
observer.poll()
self.assertEqual(observer.offset, 0)
def test_partial_boundary_retried_and_late_done_still_times_out(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='late', tool_name='bash', state='RUNNING',
started_monotonic=started, cap_s=10)
raw = json.dumps({'type': 'tool_execution_start', 'bench_tool': event})
observer.path.write_text(raw)
self.assertIsNone(observer.poll())
self.assertEqual(observer.offset, 0)
with observer.path.open('a') as stream:
stream.write('\n')
self.assertIsNone(observer.poll())
done = dict(event, state='DONE', observed_monotonic=started + 11)
with observer.path.open('a') as stream:
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
with patch.object(collection_runtime, 'broker') as broker:
self.assertEqual(observer.poll(), 'TOOL_DEADLINE_EXCEEDED')
broker.return_value.cancel.assert_called_once_with(state['execution_id'])
def tool_response(arguments):
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'ordinary-call',
'type': 'function', 'function': {'name': 'bash', 'arguments': json.dumps(arguments)}}]}
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
return 200, {'Content-Type': 'text/event-stream'}, (
''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks) + 'data: [DONE]\n\n').encode()
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and Pi SDK required')
class OrdinaryToolDeadline(unittest.TestCase):
@contextmanager
def collection(self, arguments, cap=.12):
fixture = test_collection_crash.CollectionCrash()
try:
fixture.setup_collection(lambda n, body: tool_response(arguments) if n == 1 else ok(n, body))
cfg = paths.config()
cfg['rollouts']['tools'] = ['bash']
cfg['collection_v1']['lifecycle']['ordinary_tool_deadline_s'] = cap
# Fixture mutates policy before the first real Execution is created.
from harness import collection_runtime
collection_runtime._configuration = json.dumps(cfg['collection_v1'], sort_keys=True)
original_prepare = fixture.prepare
def prepare(*args, **kwargs):
rd, job, prompt = original_prepare(*args, **kwargs)
job['gates'] = {'required': ['G1', 'G3'], 'G1': {}, 'G2': {}, 'G4': {},
'G3': {'metric': 'top1', 'kind': 'rel_pct', 'tol': 5, 'require_source': 'board'}}
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
return rd, job, prompt
from harness.verify import verify_rollout
with patch.object(rollout, 'prepare', side_effect=prepare), patch.object(rollout, 'verify_rollout', verify_rollout):
yield fixture
finally:
fixture.doCleanups()
def test_requested_timeout_cannot_bypass_frozen_cap(self):
receipts = []
for timeout in (None, 0, -1, 9999):
arguments = {'command': 'sleep .6'}
if timeout is not None:
arguments['timeout'] = timeout
with self.subTest(timeout=timeout), self.collection(arguments) as fixture:
if timeout is None:
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
started = time.monotonic()
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(state['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual(len(fixture.joint.provider.requests), 1)
self.assertLess(time.monotonic() - started, 5)
self.assertEqual(fixture.rows()[0]['state'], 'DEADLINE_EXCEEDED')
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
# A tool-cap stop no longer holds the queued repeat.
self.assertEqual([r['kk'] for r in pipeline._collection_snapshot('r', None, None)[0]],
[1] if timeout is None else [])
self.assertEqual(len(fixture.joint.provider.requests), 1)
if timeout is None:
self.assertEqual(fixture.rows()[1]['state'], 'QUEUED')
self.assertFalse((rd.parent / '01').exists())
source = trajectory.freeze(rd)
view = trajectory.reconstruct(source)
quality = trajectory.validate(source, view)
self.assertTrue(quality['pass'], quality)
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['toolCallId'], 'ordinary-call')
self.assertTrue(results[0]['isError'])
self.assertEqual(results[0]['details']['bench_stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
self.assertEqual(results, [v['message'] for v in raw if v.get('type') == 'message_end'
and v['message']['role'] == 'toolResult'])
observer = monitor.Observer(fixture.project)
observer.sample(); detail = observer.detail('r', 'j0', 0)
self.assertEqual(detail['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(detail['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
self.assertEqual(detail['activity_details']['cap_s'], .12)
self.assertEqual(detail['activity_details']['tool_call_id'], 'ordinary-call')
receipts.append({'requested': timeout, 'outcome': state['execution_outcome']})
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-cap', 'variants': receipts}))
def test_short_silent_tool_and_shorter_native_timeout_keep_original_semantics(self):
for arguments, error in [({'command': 'sleep .02'}, False),
({'command': 'sleep .6', 'timeout': .04}, True)]:
with self.subTest(arguments=arguments), self.collection(arguments, cap=1) as fixture:
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertEqual(state['execution_outcome'], 'COMPLETED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(len(fixture.joint.provider.requests), 2)
view = trajectory.reconstruct(trajectory.freeze(rd))
self.assertTrue(trajectory.validate(trajectory.freeze(rd), view)['pass'])
result = next(m for m in view['messages'] if m.get('role') == 'toolResult')
self.assertEqual(result['isError'], error)
self.assertNotIn('bench_stop_reason', result.get('details', {}))
# An old execution without a frozen cap cannot silently resume
# under the new policy; its source/session must remain untouched.
from harness import collection_runtime
state['config'].pop('ordinary_tool_deadline_s')
execution.atomic_json(rd / 'execution.json', state)
original = (rd / 'rollout.json').read_bytes()
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
collection_runtime.configure_pi(rd, execution.Execution(rd).attach(), 'teacher', resume=True)
self.assertEqual((rd / 'rollout.json').read_bytes(), original)
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-native-timeout',
'short_silent_completed': True, 'shorter_native_timeout_can_continue': True}))
def test_parent_deadline_reaps_detached_child_even_when_sdk_is_stopped(self):
# The child clears its environment and starts a new session; no marker or
# process-group scan could prove this set empty. Only the owned subreaper can.
program = ('import json,subprocess,sys;from pathlib import Path;'
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
'env={},start_new_session=True);'
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
'Path("detached.json").write_text(json.dumps({"pid":p.pid,"start":int(s[19])}))')
command = shlex.join([sys.executable, '-c', program]) + '; kill -STOP "$PPID"; sleep 20'
with self.collection({'command': command}, cap=.15) as fixture:
started = time.monotonic()
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
state = execution.Execution(rd).snapshot()
self.assertLess(time.monotonic() - started, 5)
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
self.assertEqual(state['cleanup_state'], 'VERIFIED')
self.assertEqual(execution.owned_processes(state), [])
child = json.loads((rd / 'workspace/detached.json').read_text())
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
self.assertEqual(execution.process_status(identity), 'DEAD')
for scope in state['process_scopes'].values():
receipt = execution.scope_receipt(scope)
self.assertEqual(receipt['binding']['proof'], 'waitpid:ECHILD')
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(len(fixture.joint.provider.requests), 1)
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
checks = trajectory.validate(source, view)['checks']
self.assertTrue(checks['VALID_MESSAGE_COVERAGE']['pass'])
self.assertFalse(checks['CALL_TOOL_CLOSURE']['pass'])
self.assertFalse(any(m.get('role') == 'toolResult' for m in view['messages']))
from export import to_agentrl
stats = to_agentrl.materialize(['r'], 'stopped-sdk')['stats']
self.assertEqual(stats['sft_eligible'], 0)
self.assertEqual([r['state'] for r in fixture.rows()], ['DEADLINE_EXCEEDED'])
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-parent-fallback',
'sdk_stopped': True, 'detached_child': 'DEAD', 'cleanup': 'VERIFIED',
'proof': 'waitpid:ECHILD', 'tool_result_synthesized': False, 'sft_eligible': 0}))
def test_cancel_and_deadline_use_first_committed_stop(self):
with tempfile.TemporaryDirectory() as tmp:
for first in ('cancel', 'deadline'):
owner = execution.Execution(Path(tmp) / first)
owner.start(policy())
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='t1', tool_name='bash', started_monotonic=started, cap_s=.01, state='RUNNING')
owner.observe_tools([event])
if first == 'cancel': owner.cancel('user request')
time.sleep(.02)
owner.observe_tools([])
if first == 'deadline': owner.cancel('late user request')
with self.assertRaises(execution.ExecutionStopped):
with owner.admission(): pass
final = owner.finish(0, protocol_complete=False)
self.assertEqual(final['execution_outcome'], 'CANCELLED' if first == 'cancel' else 'DEADLINE_EXCEEDED')
def test_reused_call_id_fences_stopped_sdk_without_rewriting_first_result(self):
from harness import collection_runtime
program = ('import json,subprocess,sys;from pathlib import Path;'
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
'env={},start_new_session=True);'
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
'Path("reused-detached.json").write_text(json.dumps('
'{"pid":p.pid,"start":int(s[19]),"sdk_pid":int(sys.argv[1])}))')
command = shlex.join([sys.executable, '-c', program]) + ' "$PPID"; kill -STOP "$PPID"; sleep 20'
with self.collection({'command': 'true'}, cap=.2) as fixture:
fixture.joint.provider.responder = lambda n, body: (
tool_response({'command': 'true'}) if n == 1 else
tool_response({'command': command}) if n == 2 else ok(n, body))
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
rd = fixture.project / 'runs/r/j0/rollouts/00'
fallback_cleanup = False
poll = collection_runtime.ToolDeadlines.poll
barrier = {'released': False, 'sdk_stopped': False}
def after_sdk_stop(observer):
# Force the difficult interleaving: consume the second start only
# after the owned SDK is stopped and its detached child exists.
if not barrier['released']:
until = time.monotonic() + 3
while time.monotonic() < until:
try:
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
stat = Path(f"/proc/{child['sdk_pid']}/stat").read_text().rsplit(')', 1)[1].split()
owned = observer.execution.snapshot()['processes']
if stat[0] == 'T' and any(p['role'] == 'agent' and p['pid'] == child['sdk_pid']
and p['proc_start_ticks'] == int(stat[19]) for p in owned):
barrier['sdk_stopped'] = True
break
except (OSError, ValueError):
pass # The fixture has not committed the birth record yet.
time.sleep(.01)
barrier['released'] = True
return poll(observer)
with patch.object(collection_runtime.ToolDeadlines, 'poll', new=after_sdk_stop), \
ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(pipeline.collect_rollouts, 'r', limit=1, log=lambda _: None)
try:
future.result(timeout=5)
except TimeoutError:
fallback_cleanup = True
finally:
if not future.done():
rollout.settle_execution('r', 'j0', 0, execution.Execution(rd).attach(),
'test-owned repeated-call cleanup', cancel=True)
future.result(timeout=6)
state = execution.Execution(rd).snapshot()
self.assertTrue(barrier['sdk_stopped'], 'fixture never observed the owned SDK in SIGSTOP')
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
self.assertEqual(execution.process_status(identity), 'DEAD')
self.assertEqual(execution.owned_processes(state), [])
self.assertEqual(state['cleanup_state'], 'VERIFIED')
for scope in state['process_scopes'].values():
self.assertEqual(execution.scope_receipt(scope)['binding']['proof'], 'waitpid:ECHILD')
self.assertFalse(fallback_cleanup, 'reused call ID escaped its parent-side bound')
self.assertEqual(state['execution_outcome'], 'INFRA_ERROR')
self.assertEqual(state['stop_reason'], 'INFRA_TOOL_IDENTITY_CONFLICT')
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
starts = [e['bench_tool'] for e in raw if e.get('bench_tool', {}).get('state') == 'RUNNING']
self.assertEqual(len(starts), 2)
self.assertEqual({s['tool_call_id'] for s in starts}, {'ordinary-call'})
self.assertNotEqual(starts[0]['started_monotonic'], starts[1]['started_monotonic'])
first = state['tool_calls']['ordinary-call']
self.assertEqual(first['state'], 'DONE')
self.assertEqual(first['started_monotonic'], starts[0]['started_monotonic'])
self.assertEqual(first['deadline_monotonic'], starts[0]['deadline_monotonic'])
self.assertEqual(state['tool_conflict'], starts[1])
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
quality = trajectory.validate(source, view)
self.assertTrue(quality['checks']['VALID_MESSAGE_COVERAGE']['pass'])
self.assertFalse(quality['checks']['CALL_TOOL_CLOSURE']['pass'])
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
self.assertEqual(len(results), 1)
self.assertEqual(results, [e['message'] for e in raw if e.get('type') == 'message_end'
and e['message']['role'] == 'toolResult'])
self.assertEqual(len(fixture.joint.provider.requests), 2)
self.assertEqual([r['state'] for r in fixture.rows()], ['INFRA_ERROR', 'QUEUED'])
# A system stop no longer holds the queued repeat.
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-reused-id',
'real_sdk_generations': 2, 'sdk_stopped': True, 'detached_child': 'DEAD',
'cleanup': 'VERIFIED', 'proof': 'waitpid:ECHILD', 'first_result_preserved': True,
'second_result_synthesized': False, 'repeat_claimable': True,
'outcome': state['execution_outcome'], 'reason': state['stop_reason']}))
def test_tool_occurrence_duplicates_never_reset_or_close_another_occurrence(self):
with tempfile.TemporaryDirectory() as tmp:
for variant in ('duplicate_start', 'new_start', 'foreign_end'):
with self.subTest(variant=variant):
owner = execution.Execution(Path(tmp) / variant); owner.start(policy())
state = owner.snapshot(); started = time.monotonic()
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='t1', tool_name='bash', started_monotonic=started,
deadline_monotonic=started + .03, cap_s=.03, state='RUNNING')
owner.observe_tools([event])
original = owner.snapshot()['tool_calls']['t1']
if variant == 'duplicate_start':
time.sleep(.02)
self.assertIsNone(owner.observe_tools([event]))
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
time.sleep(.02)
self.assertEqual(owner.observe_tools([]), 'TOOL_DEADLINE_EXCEEDED')
else:
other = dict(event, started_monotonic=started + .001,
deadline_monotonic=started + .031)
if variant == 'foreign_end':
other.update(state='DONE', observed_monotonic=time.monotonic())
self.assertEqual(owner.observe_tools([other]), 'INFRA_TOOL_IDENTITY_CONFLICT')
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
self.assertEqual(owner.snapshot()['tool_conflict'], other)
with self.assertRaises(execution.ExecutionStopped):
with owner.admission(): pass
def test_verify_preserves_all_control_holds_and_queued_repeats(self):
from harness.verify import verify_rollout
with self.collection({'command': 'true'}, cap=1) as fixture:
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
rd = fixture.project / 'runs/r/j0/rollouts/00'
owner = execution.Execution(rd).attach()
original = owner.snapshot()
with db.connect() as con:
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
measurements = verify_rollout(rd, write=False)['gates']
variants = [('TERMINAL', outcome, 'VERIFIED', outcome) for outcome in
('CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST')]
variants += [('PAUSED_INFRA', None, 'VERIFIED', 'PAUSED_INFRA'),
('CLEANING', None, 'PENDING', 'CLEANING'),
('ACTIVE', None, 'PENDING', 'RUNNING'),
('TERMINAL', 'AGENT_ERROR', 'QUARANTINED', 'INFRA_ERROR')]
for status, outcome, cleanup, expected in variants:
with self.subTest(status=status, outcome=outcome, cleanup=cleanup):
execution.atomic_json(owner.path, dict(original, execution_state=status,
execution_outcome=outcome, cleanup_state=cleanup))
for _ in range(2):
verdict = verify_rollout(rd)
self.assertEqual(verdict['gates'], measurements)
self.assertEqual(verdict['terminal'], 'UNVERIFIED')
self.assertEqual(verdict['execution_control']['db_state'], expected)
self.assertEqual(fixture.rows()[0]['state'], expected)
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
held = expected in ('BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING')
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]],
[] if held else [('j0', 1)])
for fault in ('mismatch', 'malformed', 'missing'):
if fault == 'mismatch':
execution.atomic_json(owner.path, dict(original, execution_id='different'))
elif fault == 'malformed':
execution.atomic_json(owner.path, [])
else:
owner.path.unlink() # execution_managed source must fail closed.
verdict = verify_rollout(rd)
self.assertEqual(verdict['gates'], measurements)
self.assertEqual(fixture.rows()[0]['state'], 'INFRA_ERROR')
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
print('JOINT_RECEIPT ' + json.dumps({'case': 'verify-control-holds',
'variants': len(variants), 'identity_faults': 3, 'hard_gates_unchanged': True,
'repeat_held_only_for': ['BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING']}))
def test_legacy_resume_is_rejected_before_execution_quota_or_db_changes(self):
from harness import cli, collection_runtime as runtime
fixture = test_collection_crash.CollectionCrash()
try:
fixture.setup_collection(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
self.assertEqual(pipeline.collect_rollouts('r', limit=1, log=lambda _: None), {'PAUSED_INFRA': 1})
rd = fixture.project / 'runs/r/j0/rollouts/00'
owner = execution.Execution(rd).attach()
state = owner.snapshot(); state['config'].pop('ordinary_tool_deadline_s')
execution.atomic_json(owner.path, state)
originals = {p: p.read_bytes() for p in (owner.path, rd / 'rollout.json',
rd / 'agent/session-checkpoint.json', rd / 'agent/native-session.jsonl')}
rows = fixture.rows()
with fixture.joint.broker._db() as con:
ledger = list(con.iterdump())
with patch.object(runtime, 'broker', side_effect=AssertionError('broker touched before policy check')):
for action in [owner.resume, lambda: rollout.resume_rollout('r', 'j0', 0),
lambda: cli.main(['execution', 'resume', '--run', 'r', '--job', 'j0', '--kk', '0']),
lambda: runtime.configure_pi(rd, owner, 'teacher', resume=True)]:
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
action()
self.assertEqual(pipeline._resume_snapshot('r'), ([], False))
self.assertEqual(fixture.rows(), rows)
self.assertEqual({p: p.read_bytes() for p in originals}, originals)
with fixture.joint.broker._db() as con:
self.assertEqual(list(con.iterdump()), ledger)
print('JOINT_RECEIPT ' + json.dumps({'case': 'legacy-resume-policy',
'entrypoints': ['execution', 'rollout', 'CLI', 'configure', 'collector'],
'source_and_db_unchanged': True, 'broker_calls': 0}))
finally:
fixture.doCleanups()
class ResumeSlots(unittest.TestCase):
def test_waiting_paused_rollouts_keep_the_free_slots(self):
# 10-15: a ready resume let new claims in the same cycle take the slots of those still waiting for quota.
resume, new = {'job_id': 'p', '_resume': True}, {'job_id': 'n'}
self.assertEqual(pipeline._claimable([resume], True, [new]), [resume])
self.assertEqual(pipeline._claimable([], True, [new]), [])
self.assertEqual(pipeline._claimable([resume], False, [new]), [resume])
self.assertEqual(pipeline._claimable([], False, [new]), [new])
if __name__ == '__main__':
unittest.main()