10-15 11:21、11:33:一条暂停轨迹续上的同一轮里,新题跟着进来,把还在等额度的几条的名额占了。 它们只能等别的轨迹结束才能排上,期间 24 小时时钟照走(两条自带图的只剩 7.5 小时)。 - _claimable:有暂停轨迹已就绪或在等额度时,这一轮只续跑;都续上了下一轮再领新题。 - _resume_snapshot:额度组 PAUSED/RECONCILING 不论原因都算在等(00:56 是连接被切,原来不算)。 collector 重启后生效。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
572 lines
36 KiB
Python
572 lines
36 KiB
Python
"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
|
|
import io
|
|
import json
|
|
import shlex
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import threading
|
|
import unittest
|
|
from contextlib import contextmanager
|
|
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from export import trajectory
|
|
from harness import db, execution, monitor, paths, pipeline, rollout
|
|
from scripts import test_collection_crash, test_pi_session
|
|
from scripts.test_execution import policy
|
|
from scripts.test_quota import ok
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
|
|
class ToolBoundaryCommit(unittest.TestCase):
|
|
@contextmanager
|
|
def first_reader_bytes(self, path, raw):
|
|
"""Expose a stale first read without changing the real event evidence."""
|
|
original = Path.open
|
|
opened = []
|
|
def opening(target, mode='r', *args, **kwargs):
|
|
if target == path and mode == 'rb':
|
|
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
|
|
opened.append(stream)
|
|
return stream
|
|
return original(target, mode, *args, **kwargs)
|
|
with patch.object(Path, 'open', new=opening):
|
|
yield opened
|
|
|
|
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
|
|
from harness import collection_runtime
|
|
for bad in (b'{"type":"note","value":"\x00"}\n',
|
|
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
|
|
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
path = rd / 'agent/events.jsonl'
|
|
prefix = b'{"type":"already_consumed"}\n'
|
|
path.write_bytes(prefix)
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='completed', tool_name='bash', state='RUNNING',
|
|
started_monotonic=started, cap_s=10)
|
|
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
|
|
done = dict(start, state='DONE', observed_monotonic=started + 1)
|
|
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
|
|
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
|
|
original = prefix + start_line + good + done_line
|
|
path.write_bytes(original)
|
|
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
|
|
with self.first_reader_bytes(path, stale) as readers, \
|
|
patch.object(execution.time, 'monotonic', return_value=started + 11), \
|
|
patch.object(collection_runtime, 'broker') as broker, \
|
|
self.assertLogs(level='WARNING') as logs:
|
|
self.assertIsNone(observer.poll())
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
broker.assert_not_called()
|
|
self.assertEqual(len(logs.output), 1)
|
|
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
|
|
self.assertNotIn('完整原始证据', logs.output[0])
|
|
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
|
|
self.assertEqual(observer.offset, len(original))
|
|
self.assertEqual(path.read_bytes(), original)
|
|
|
|
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
|
|
from harness import collection_runtime
|
|
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
|
|
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
|
|
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot()
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
|
|
started_monotonic=time.monotonic(), cap_s=10)
|
|
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
|
|
observer.path.write_bytes(raw)
|
|
with self.first_reader_bytes(observer.path, raw) as readers, \
|
|
self.assertLogs(level='WARNING'), self.assertRaises(error):
|
|
observer.poll()
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
self.assertEqual(observer.offset, 0)
|
|
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
|
|
self.assertEqual(observer.path.read_bytes(), raw)
|
|
|
|
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
original = b'{"type":"note","value":"complete JSON but no newline"}'
|
|
observer.path.write_bytes(original)
|
|
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
|
|
self.assertLogs(level='WARNING'), \
|
|
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
|
|
observer.poll()
|
|
self.assertEqual(len(readers), 2)
|
|
self.assertTrue(all(reader.closed for reader in readers))
|
|
self.assertEqual(observer.offset, 0)
|
|
self.assertEqual(observer.path.read_bytes(), original)
|
|
|
|
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot()
|
|
clock = [time.monotonic()]
|
|
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='completed', tool_name='bash', state='RUNNING',
|
|
started_monotonic=clock[0], cap_s=10)
|
|
observer.path.write_text(json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n')
|
|
ready = threading.Event()
|
|
observe = owner.observe_tools
|
|
def waiting(events):
|
|
ready.set()
|
|
return observe(events)
|
|
with patch.object(owner, 'observe_tools', side_effect=waiting), \
|
|
patch.object(execution.time, 'monotonic', side_effect=lambda: clock[0]), \
|
|
patch.object(collection_runtime, 'broker') as broker, \
|
|
ThreadPoolExecutor(max_workers=1) as pool:
|
|
with owner.locked():
|
|
result = pool.submit(observer.poll)
|
|
self.assertTrue(ready.wait(2))
|
|
done = dict(start, state='DONE', observed_monotonic=clock[0] + 1)
|
|
with observer.path.open('a') as stream:
|
|
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
|
|
clock[0] += 11
|
|
self.assertIsNone(result.result(timeout=2))
|
|
broker.assert_not_called()
|
|
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
|
|
self.assertEqual(observer.offset, observer.path.stat().st_size)
|
|
|
|
def test_failed_apply_keeps_cursor_for_retry(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
observer.path.write_text('{"type":"tool_execution_start","bench_tool":{"state":"RUNNING"}}\n')
|
|
def fail_after_read(events):
|
|
list(events)
|
|
raise OSError('fixture commit failed')
|
|
with patch.object(owner, 'observe_tools', side_effect=fail_after_read):
|
|
with self.assertRaises(OSError):
|
|
observer.poll()
|
|
self.assertEqual(observer.offset, 0)
|
|
|
|
def test_partial_boundary_retried_and_late_done_still_times_out(self):
|
|
from harness import collection_runtime
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
rd = Path(tmp)
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
(rd / 'agent').mkdir()
|
|
observer = collection_runtime.ToolDeadlines(rd, owner)
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='late', tool_name='bash', state='RUNNING',
|
|
started_monotonic=started, cap_s=10)
|
|
raw = json.dumps({'type': 'tool_execution_start', 'bench_tool': event})
|
|
observer.path.write_text(raw)
|
|
self.assertIsNone(observer.poll())
|
|
self.assertEqual(observer.offset, 0)
|
|
with observer.path.open('a') as stream:
|
|
stream.write('\n')
|
|
self.assertIsNone(observer.poll())
|
|
done = dict(event, state='DONE', observed_monotonic=started + 11)
|
|
with observer.path.open('a') as stream:
|
|
stream.write(json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n')
|
|
with patch.object(collection_runtime, 'broker') as broker:
|
|
self.assertEqual(observer.poll(), 'TOOL_DEADLINE_EXCEEDED')
|
|
broker.return_value.cancel.assert_called_once_with(state['execution_id'])
|
|
|
|
|
|
def tool_response(arguments):
|
|
delta = {'role': 'assistant', 'tool_calls': [{'index': 0, 'id': 'ordinary-call',
|
|
'type': 'function', 'function': {'name': 'bash', 'arguments': json.dumps(arguments)}}]}
|
|
chunks = [{'choices': [{'index': 0, 'delta': delta, 'finish_reason': None}]},
|
|
{'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'tool_calls'}],
|
|
'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}}]
|
|
return 200, {'Content-Type': 'text/event-stream'}, (
|
|
''.join('data: ' + json.dumps(c) + '\n\n' for c in chunks) + 'data: [DONE]\n\n').encode()
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux' and test_pi_session.NODE.is_file(), 'Linux and Pi SDK required')
|
|
class OrdinaryToolDeadline(unittest.TestCase):
|
|
@contextmanager
|
|
def collection(self, arguments, cap=.12):
|
|
fixture = test_collection_crash.CollectionCrash()
|
|
try:
|
|
fixture.setup_collection(lambda n, body: tool_response(arguments) if n == 1 else ok(n, body))
|
|
cfg = paths.config()
|
|
cfg['rollouts']['tools'] = ['bash']
|
|
cfg['collection_v1']['lifecycle']['ordinary_tool_deadline_s'] = cap
|
|
# Fixture mutates policy before the first real Execution is created.
|
|
from harness import collection_runtime
|
|
collection_runtime._configuration = json.dumps(cfg['collection_v1'], sort_keys=True)
|
|
original_prepare = fixture.prepare
|
|
def prepare(*args, **kwargs):
|
|
rd, job, prompt = original_prepare(*args, **kwargs)
|
|
job['gates'] = {'required': ['G1', 'G3'], 'G1': {}, 'G2': {}, 'G4': {},
|
|
'G3': {'metric': 'top1', 'kind': 'rel_pct', 'tol': 5, 'require_source': 'board'}}
|
|
(rd.parent.parent / 'job.json').write_text(json.dumps(job))
|
|
return rd, job, prompt
|
|
from harness.verify import verify_rollout
|
|
with patch.object(rollout, 'prepare', side_effect=prepare), patch.object(rollout, 'verify_rollout', verify_rollout):
|
|
yield fixture
|
|
finally:
|
|
fixture.doCleanups()
|
|
|
|
def test_requested_timeout_cannot_bypass_frozen_cap(self):
|
|
receipts = []
|
|
for timeout in (None, 0, -1, 9999):
|
|
arguments = {'command': 'sleep .6'}
|
|
if timeout is not None:
|
|
arguments['timeout'] = timeout
|
|
with self.subTest(timeout=timeout), self.collection(arguments) as fixture:
|
|
if timeout is None:
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
started = time.monotonic()
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertEqual(fixture.rows()[0]['state'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
|
|
# A tool-cap stop no longer holds the queued repeat.
|
|
self.assertEqual([r['kk'] for r in pipeline._collection_snapshot('r', None, None)[0]],
|
|
[1] if timeout is None else [])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
if timeout is None:
|
|
self.assertEqual(fixture.rows()[1]['state'], 'QUEUED')
|
|
self.assertFalse((rd.parent / '01').exists())
|
|
source = trajectory.freeze(rd)
|
|
view = trajectory.reconstruct(source)
|
|
quality = trajectory.validate(source, view)
|
|
self.assertTrue(quality['pass'], quality)
|
|
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
|
|
self.assertEqual(len(results), 1)
|
|
self.assertEqual(results[0]['toolCallId'], 'ordinary-call')
|
|
self.assertTrue(results[0]['isError'])
|
|
self.assertEqual(results[0]['details']['bench_stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
|
|
self.assertEqual(results, [v['message'] for v in raw if v.get('type') == 'message_end'
|
|
and v['message']['role'] == 'toolResult'])
|
|
observer = monitor.Observer(fixture.project)
|
|
observer.sample(); detail = observer.detail('r', 'j0', 0)
|
|
self.assertEqual(detail['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(detail['stop_reason'], 'TOOL_DEADLINE_EXCEEDED')
|
|
self.assertEqual(detail['activity_details']['cap_s'], .12)
|
|
self.assertEqual(detail['activity_details']['tool_call_id'], 'ordinary-call')
|
|
receipts.append({'requested': timeout, 'outcome': state['execution_outcome']})
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-cap', 'variants': receipts}))
|
|
|
|
def test_short_silent_tool_and_shorter_native_timeout_keep_original_semantics(self):
|
|
for arguments, error in [({'command': 'sleep .02'}, False),
|
|
({'command': 'sleep .6', 'timeout': .04}, True)]:
|
|
with self.subTest(arguments=arguments), self.collection(arguments, cap=1) as fixture:
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertEqual(state['execution_outcome'], 'COMPLETED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(len(fixture.joint.provider.requests), 2)
|
|
view = trajectory.reconstruct(trajectory.freeze(rd))
|
|
self.assertTrue(trajectory.validate(trajectory.freeze(rd), view)['pass'])
|
|
result = next(m for m in view['messages'] if m.get('role') == 'toolResult')
|
|
self.assertEqual(result['isError'], error)
|
|
self.assertNotIn('bench_stop_reason', result.get('details', {}))
|
|
# An old execution without a frozen cap cannot silently resume
|
|
# under the new policy; its source/session must remain untouched.
|
|
from harness import collection_runtime
|
|
state['config'].pop('ordinary_tool_deadline_s')
|
|
execution.atomic_json(rd / 'execution.json', state)
|
|
original = (rd / 'rollout.json').read_bytes()
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
|
|
collection_runtime.configure_pi(rd, execution.Execution(rd).attach(), 'teacher', resume=True)
|
|
self.assertEqual((rd / 'rollout.json').read_bytes(), original)
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-native-timeout',
|
|
'short_silent_completed': True, 'shorter_native_timeout_can_continue': True}))
|
|
|
|
def test_parent_deadline_reaps_detached_child_even_when_sdk_is_stopped(self):
|
|
# The child clears its environment and starts a new session; no marker or
|
|
# process-group scan could prove this set empty. Only the owned subreaper can.
|
|
program = ('import json,subprocess,sys;from pathlib import Path;'
|
|
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
|
|
'env={},start_new_session=True);'
|
|
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
|
|
'Path("detached.json").write_text(json.dumps({"pid":p.pid,"start":int(s[19])}))')
|
|
command = shlex.join([sys.executable, '-c', program]) + '; kill -STOP "$PPID"; sleep 20'
|
|
with self.collection({'command': command}, cap=.15) as fixture:
|
|
started = time.monotonic()
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertLess(time.monotonic() - started, 5)
|
|
self.assertEqual(state['execution_outcome'], 'DEADLINE_EXCEEDED')
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
child = json.loads((rd / 'workspace/detached.json').read_text())
|
|
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
|
|
self.assertEqual(execution.process_status(identity), 'DEAD')
|
|
for scope in state['process_scopes'].values():
|
|
receipt = execution.scope_receipt(scope)
|
|
self.assertEqual(receipt['binding']['proof'], 'waitpid:ECHILD')
|
|
self.assertTrue(receipt['all_children_reaped'])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 1)
|
|
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
|
|
checks = trajectory.validate(source, view)['checks']
|
|
self.assertTrue(checks['VALID_MESSAGE_COVERAGE']['pass'])
|
|
self.assertFalse(checks['CALL_TOOL_CLOSURE']['pass'])
|
|
self.assertFalse(any(m.get('role') == 'toolResult' for m in view['messages']))
|
|
from export import to_agentrl
|
|
stats = to_agentrl.materialize(['r'], 'stopped-sdk')['stats']
|
|
self.assertEqual(stats['sft_eligible'], 0)
|
|
self.assertEqual([r['state'] for r in fixture.rows()], ['DEADLINE_EXCEEDED'])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-parent-fallback',
|
|
'sdk_stopped': True, 'detached_child': 'DEAD', 'cleanup': 'VERIFIED',
|
|
'proof': 'waitpid:ECHILD', 'tool_result_synthesized': False, 'sft_eligible': 0}))
|
|
|
|
def test_cancel_and_deadline_use_first_committed_stop(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for first in ('cancel', 'deadline'):
|
|
owner = execution.Execution(Path(tmp) / first)
|
|
owner.start(policy())
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='t1', tool_name='bash', started_monotonic=started, cap_s=.01, state='RUNNING')
|
|
owner.observe_tools([event])
|
|
if first == 'cancel': owner.cancel('user request')
|
|
time.sleep(.02)
|
|
owner.observe_tools([])
|
|
if first == 'deadline': owner.cancel('late user request')
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with owner.admission(): pass
|
|
final = owner.finish(0, protocol_complete=False)
|
|
self.assertEqual(final['execution_outcome'], 'CANCELLED' if first == 'cancel' else 'DEADLINE_EXCEEDED')
|
|
|
|
def test_reused_call_id_fences_stopped_sdk_without_rewriting_first_result(self):
|
|
from harness import collection_runtime
|
|
program = ('import json,subprocess,sys;from pathlib import Path;'
|
|
'p=subprocess.Popen([sys.executable,"-c","import time;time.sleep(20)"],'
|
|
'env={},start_new_session=True);'
|
|
's=Path(f"/proc/{p.pid}/stat").read_text().rsplit(")",1)[1].split();'
|
|
'Path("reused-detached.json").write_text(json.dumps('
|
|
'{"pid":p.pid,"start":int(s[19]),"sdk_pid":int(sys.argv[1])}))')
|
|
command = shlex.join([sys.executable, '-c', program]) + ' "$PPID"; kill -STOP "$PPID"; sleep 20'
|
|
with self.collection({'command': 'true'}, cap=.2) as fixture:
|
|
fixture.joint.provider.responder = lambda n, body: (
|
|
tool_response({'command': 'true'}) if n == 1 else
|
|
tool_response({'command': command}) if n == 2 else ok(n, body))
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
fallback_cleanup = False
|
|
poll = collection_runtime.ToolDeadlines.poll
|
|
barrier = {'released': False, 'sdk_stopped': False}
|
|
def after_sdk_stop(observer):
|
|
# Force the difficult interleaving: consume the second start only
|
|
# after the owned SDK is stopped and its detached child exists.
|
|
if not barrier['released']:
|
|
until = time.monotonic() + 3
|
|
while time.monotonic() < until:
|
|
try:
|
|
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
|
|
stat = Path(f"/proc/{child['sdk_pid']}/stat").read_text().rsplit(')', 1)[1].split()
|
|
owned = observer.execution.snapshot()['processes']
|
|
if stat[0] == 'T' and any(p['role'] == 'agent' and p['pid'] == child['sdk_pid']
|
|
and p['proc_start_ticks'] == int(stat[19]) for p in owned):
|
|
barrier['sdk_stopped'] = True
|
|
break
|
|
except (OSError, ValueError):
|
|
pass # The fixture has not committed the birth record yet.
|
|
time.sleep(.01)
|
|
barrier['released'] = True
|
|
return poll(observer)
|
|
with patch.object(collection_runtime.ToolDeadlines, 'poll', new=after_sdk_stop), \
|
|
ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(pipeline.collect_rollouts, 'r', limit=1, log=lambda _: None)
|
|
try:
|
|
future.result(timeout=5)
|
|
except TimeoutError:
|
|
fallback_cleanup = True
|
|
finally:
|
|
if not future.done():
|
|
rollout.settle_execution('r', 'j0', 0, execution.Execution(rd).attach(),
|
|
'test-owned repeated-call cleanup', cancel=True)
|
|
future.result(timeout=6)
|
|
state = execution.Execution(rd).snapshot()
|
|
self.assertTrue(barrier['sdk_stopped'], 'fixture never observed the owned SDK in SIGSTOP')
|
|
child = json.loads((rd / 'workspace/reused-detached.json').read_text())
|
|
identity = dict(state['owner'], pid=child['pid'], proc_start_ticks=child['start'])
|
|
self.assertEqual(execution.process_status(identity), 'DEAD')
|
|
self.assertEqual(execution.owned_processes(state), [])
|
|
self.assertEqual(state['cleanup_state'], 'VERIFIED')
|
|
for scope in state['process_scopes'].values():
|
|
self.assertEqual(execution.scope_receipt(scope)['binding']['proof'], 'waitpid:ECHILD')
|
|
self.assertFalse(fallback_cleanup, 'reused call ID escaped its parent-side bound')
|
|
self.assertEqual(state['execution_outcome'], 'INFRA_ERROR')
|
|
self.assertEqual(state['stop_reason'], 'INFRA_TOOL_IDENTITY_CONFLICT')
|
|
raw = [json.loads(line) for line in (rd / 'agent/events.jsonl').read_text().splitlines()]
|
|
starts = [e['bench_tool'] for e in raw if e.get('bench_tool', {}).get('state') == 'RUNNING']
|
|
self.assertEqual(len(starts), 2)
|
|
self.assertEqual({s['tool_call_id'] for s in starts}, {'ordinary-call'})
|
|
self.assertNotEqual(starts[0]['started_monotonic'], starts[1]['started_monotonic'])
|
|
first = state['tool_calls']['ordinary-call']
|
|
self.assertEqual(first['state'], 'DONE')
|
|
self.assertEqual(first['started_monotonic'], starts[0]['started_monotonic'])
|
|
self.assertEqual(first['deadline_monotonic'], starts[0]['deadline_monotonic'])
|
|
self.assertEqual(state['tool_conflict'], starts[1])
|
|
source = trajectory.freeze(rd); view = trajectory.reconstruct(source)
|
|
quality = trajectory.validate(source, view)
|
|
self.assertTrue(quality['checks']['VALID_MESSAGE_COVERAGE']['pass'])
|
|
self.assertFalse(quality['checks']['CALL_TOOL_CLOSURE']['pass'])
|
|
results = [m for m in view['messages'] if m.get('role') == 'toolResult']
|
|
self.assertEqual(len(results), 1)
|
|
self.assertEqual(results, [e['message'] for e in raw if e.get('type') == 'message_end'
|
|
and e['message']['role'] == 'toolResult'])
|
|
self.assertEqual(len(fixture.joint.provider.requests), 2)
|
|
self.assertEqual([r['state'] for r in fixture.rows()], ['INFRA_ERROR', 'QUEUED'])
|
|
# A system stop no longer holds the queued repeat.
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'ordinary-tool-reused-id',
|
|
'real_sdk_generations': 2, 'sdk_stopped': True, 'detached_child': 'DEAD',
|
|
'cleanup': 'VERIFIED', 'proof': 'waitpid:ECHILD', 'first_result_preserved': True,
|
|
'second_result_synthesized': False, 'repeat_claimable': True,
|
|
'outcome': state['execution_outcome'], 'reason': state['stop_reason']}))
|
|
|
|
def test_tool_occurrence_duplicates_never_reset_or_close_another_occurrence(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for variant in ('duplicate_start', 'new_start', 'foreign_end'):
|
|
with self.subTest(variant=variant):
|
|
owner = execution.Execution(Path(tmp) / variant); owner.start(policy())
|
|
state = owner.snapshot(); started = time.monotonic()
|
|
event = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
|
tool_call_id='t1', tool_name='bash', started_monotonic=started,
|
|
deadline_monotonic=started + .03, cap_s=.03, state='RUNNING')
|
|
owner.observe_tools([event])
|
|
original = owner.snapshot()['tool_calls']['t1']
|
|
if variant == 'duplicate_start':
|
|
time.sleep(.02)
|
|
self.assertIsNone(owner.observe_tools([event]))
|
|
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
|
|
time.sleep(.02)
|
|
self.assertEqual(owner.observe_tools([]), 'TOOL_DEADLINE_EXCEEDED')
|
|
else:
|
|
other = dict(event, started_monotonic=started + .001,
|
|
deadline_monotonic=started + .031)
|
|
if variant == 'foreign_end':
|
|
other.update(state='DONE', observed_monotonic=time.monotonic())
|
|
self.assertEqual(owner.observe_tools([other]), 'INFRA_TOOL_IDENTITY_CONFLICT')
|
|
self.assertEqual(owner.snapshot()['tool_calls']['t1'], original)
|
|
self.assertEqual(owner.snapshot()['tool_conflict'], other)
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with owner.admission(): pass
|
|
|
|
def test_verify_preserves_all_control_holds_and_queued_repeats(self):
|
|
from harness.verify import verify_rollout
|
|
with self.collection({'command': 'true'}, cap=1) as fixture:
|
|
pipeline.collect_rollouts('r', limit=1, log=lambda _: None)
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
owner = execution.Execution(rd).attach()
|
|
original = owner.snapshot()
|
|
with db.connect() as con:
|
|
db.rollout_upsert(con, 'r', 'j0', 1, 'e2e', 'teacher')
|
|
measurements = verify_rollout(rd, write=False)['gates']
|
|
variants = [('TERMINAL', outcome, 'VERIFIED', outcome) for outcome in
|
|
('CANCELLED', 'DEADLINE_EXCEEDED', 'INFRA_ERROR', 'BUDGET_EXHAUSTED', 'LOST')]
|
|
variants += [('PAUSED_INFRA', None, 'VERIFIED', 'PAUSED_INFRA'),
|
|
('CLEANING', None, 'PENDING', 'CLEANING'),
|
|
('ACTIVE', None, 'PENDING', 'RUNNING'),
|
|
('TERMINAL', 'AGENT_ERROR', 'QUARANTINED', 'INFRA_ERROR')]
|
|
for status, outcome, cleanup, expected in variants:
|
|
with self.subTest(status=status, outcome=outcome, cleanup=cleanup):
|
|
execution.atomic_json(owner.path, dict(original, execution_state=status,
|
|
execution_outcome=outcome, cleanup_state=cleanup))
|
|
for _ in range(2):
|
|
verdict = verify_rollout(rd)
|
|
self.assertEqual(verdict['gates'], measurements)
|
|
self.assertEqual(verdict['terminal'], 'UNVERIFIED')
|
|
self.assertEqual(verdict['execution_control']['db_state'], expected)
|
|
self.assertEqual(fixture.rows()[0]['state'], expected)
|
|
self.assertEqual(fixture.rows()[0]['verdict_terminal'], 'UNVERIFIED')
|
|
held = expected in ('BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING')
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]],
|
|
[] if held else [('j0', 1)])
|
|
for fault in ('mismatch', 'malformed', 'missing'):
|
|
if fault == 'mismatch':
|
|
execution.atomic_json(owner.path, dict(original, execution_id='different'))
|
|
elif fault == 'malformed':
|
|
execution.atomic_json(owner.path, [])
|
|
else:
|
|
owner.path.unlink() # execution_managed source must fail closed.
|
|
verdict = verify_rollout(rd)
|
|
self.assertEqual(verdict['gates'], measurements)
|
|
self.assertEqual(fixture.rows()[0]['state'], 'INFRA_ERROR')
|
|
self.assertEqual([(r['job_id'], r['kk']) for r in pipeline._collection_snapshot('r', None, None)[0]], [('j0', 1)])
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'verify-control-holds',
|
|
'variants': len(variants), 'identity_faults': 3, 'hard_gates_unchanged': True,
|
|
'repeat_held_only_for': ['BUDGET_EXHAUSTED', 'PAUSED_INFRA', 'CLEANING', 'RUNNING']}))
|
|
|
|
def test_legacy_resume_is_rejected_before_execution_quota_or_db_changes(self):
|
|
from harness import cli, collection_runtime as runtime
|
|
fixture = test_collection_crash.CollectionCrash()
|
|
try:
|
|
fixture.setup_collection(lambda *_: (429, {'Retry-After': '.08'}, b'{"error":{"type":"rate_limit"}}'))
|
|
self.assertEqual(pipeline.collect_rollouts('r', limit=1, log=lambda _: None), {'PAUSED_INFRA': 1})
|
|
rd = fixture.project / 'runs/r/j0/rollouts/00'
|
|
owner = execution.Execution(rd).attach()
|
|
state = owner.snapshot(); state['config'].pop('ordinary_tool_deadline_s')
|
|
execution.atomic_json(owner.path, state)
|
|
originals = {p: p.read_bytes() for p in (owner.path, rd / 'rollout.json',
|
|
rd / 'agent/session-checkpoint.json', rd / 'agent/native-session.jsonl')}
|
|
rows = fixture.rows()
|
|
with fixture.joint.broker._db() as con:
|
|
ledger = list(con.iterdump())
|
|
with patch.object(runtime, 'broker', side_effect=AssertionError('broker touched before policy check')):
|
|
for action in [owner.resume, lambda: rollout.resume_rollout('r', 'j0', 0),
|
|
lambda: cli.main(['execution', 'resume', '--run', 'r', '--job', 'j0', '--kk', '0']),
|
|
lambda: runtime.configure_pi(rd, owner, 'teacher', resume=True)]:
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'RESUME_TOOL_POLICY_MISSING'):
|
|
action()
|
|
self.assertEqual(pipeline._resume_snapshot('r'), ([], False))
|
|
self.assertEqual(fixture.rows(), rows)
|
|
self.assertEqual({p: p.read_bytes() for p in originals}, originals)
|
|
with fixture.joint.broker._db() as con:
|
|
self.assertEqual(list(con.iterdump()), ledger)
|
|
print('JOINT_RECEIPT ' + json.dumps({'case': 'legacy-resume-policy',
|
|
'entrypoints': ['execution', 'rollout', 'CLI', 'configure', 'collector'],
|
|
'source_and_db_unchanged': True, 'broker_calls': 0}))
|
|
finally:
|
|
fixture.doCleanups()
|
|
|
|
|
|
class ResumeSlots(unittest.TestCase):
|
|
def test_waiting_paused_rollouts_keep_the_free_slots(self):
|
|
# 10-15: a ready resume let new claims in the same cycle take the slots of those still waiting for quota.
|
|
resume, new = {'job_id': 'p', '_resume': True}, {'job_id': 'n'}
|
|
self.assertEqual(pipeline._claimable([resume], True, [new]), [resume])
|
|
self.assertEqual(pipeline._claimable([], True, [new]), [])
|
|
self.assertEqual(pipeline._claimable([resume], False, [new]), [resume])
|
|
self.assertEqual(pipeline._claimable([], False, [new]), [new])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|