fix: retry transient tool event reads once
This commit is contained in:
parent
b398495d54
commit
8bb1694499
@ -2,8 +2,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
from contextlib import ExitStack
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import threading
|
||||
|
||||
@ -162,12 +164,26 @@ class ToolDeadlines:
|
||||
# a DONE arriving during lock contention must precede timeout judgment.
|
||||
nonlocal offset
|
||||
if self.path.exists():
|
||||
with self.path.open('rb') as stream:
|
||||
with ExitStack() as readers:
|
||||
stream = readers.enter_context(self.path.open('rb'))
|
||||
stream.seek(offset)
|
||||
while line := stream.readline():
|
||||
if not line.endswith(b'\n'):
|
||||
break # Partial boundaries are retried on the next poll.
|
||||
event = json.loads(line)
|
||||
try:
|
||||
event = json.loads(line.decode('utf-8'))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s',
|
||||
self.path, offset, len(line), hashlib.sha256(line).hexdigest(),
|
||||
type(error).__name__)
|
||||
# Discard buffered bytes, including any following lines.
|
||||
stream.close()
|
||||
stream = readers.enter_context(self.path.open('rb'))
|
||||
stream.seek(offset)
|
||||
line = stream.readline()
|
||||
if not line.endswith(b'\n'):
|
||||
raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error
|
||||
event = json.loads(line.decode('utf-8'))
|
||||
offset = stream.tell()
|
||||
if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'):
|
||||
yield event['bench_tool']
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
|
||||
import io
|
||||
import json
|
||||
import shlex
|
||||
import sys
|
||||
@ -20,6 +21,100 @@ from scripts.test_quota import ok
|
||||
|
||||
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
|
||||
class ToolBoundaryCommit(unittest.TestCase):
|
||||
@contextmanager
|
||||
def first_reader_bytes(self, path, raw):
|
||||
"""Expose a stale first read without changing the real event evidence."""
|
||||
original = Path.open
|
||||
opened = []
|
||||
def opening(target, mode='r', *args, **kwargs):
|
||||
if target == path and mode == 'rb':
|
||||
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
|
||||
opened.append(stream)
|
||||
return stream
|
||||
return original(target, mode, *args, **kwargs)
|
||||
with patch.object(Path, 'open', new=opening):
|
||||
yield opened
|
||||
|
||||
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
|
||||
from harness import collection_runtime
|
||||
for bad in (b'{"type":"note","value":"\x00"}\n',
|
||||
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
|
||||
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
|
||||
rd = Path(tmp)
|
||||
owner = execution.Execution(rd); owner.start(policy())
|
||||
(rd / 'agent').mkdir()
|
||||
path = rd / 'agent/events.jsonl'
|
||||
prefix = b'{"type":"already_consumed"}\n'
|
||||
path.write_bytes(prefix)
|
||||
observer = collection_runtime.ToolDeadlines(rd, owner)
|
||||
state = owner.snapshot(); started = time.monotonic()
|
||||
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
||||
tool_call_id='completed', tool_name='bash', state='RUNNING',
|
||||
started_monotonic=started, cap_s=10)
|
||||
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
|
||||
done = dict(start, state='DONE', observed_monotonic=started + 1)
|
||||
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
|
||||
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
|
||||
original = prefix + start_line + good + done_line
|
||||
path.write_bytes(original)
|
||||
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
|
||||
with self.first_reader_bytes(path, stale) as readers, \
|
||||
patch.object(execution.time, 'monotonic', return_value=started + 11), \
|
||||
patch.object(collection_runtime, 'broker') as broker, \
|
||||
self.assertLogs(level='WARNING') as logs:
|
||||
self.assertIsNone(observer.poll())
|
||||
self.assertEqual(len(readers), 2)
|
||||
self.assertTrue(all(reader.closed for reader in readers))
|
||||
broker.assert_not_called()
|
||||
self.assertEqual(len(logs.output), 1)
|
||||
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
|
||||
self.assertNotIn('完整原始证据', logs.output[0])
|
||||
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
|
||||
self.assertEqual(observer.offset, len(original))
|
||||
self.assertEqual(path.read_bytes(), original)
|
||||
|
||||
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
|
||||
from harness import collection_runtime
|
||||
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
|
||||
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
|
||||
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
|
||||
rd = Path(tmp)
|
||||
owner = execution.Execution(rd); owner.start(policy())
|
||||
(rd / 'agent').mkdir()
|
||||
observer = collection_runtime.ToolDeadlines(rd, owner)
|
||||
state = owner.snapshot()
|
||||
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
|
||||
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
|
||||
started_monotonic=time.monotonic(), cap_s=10)
|
||||
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
|
||||
observer.path.write_bytes(raw)
|
||||
with self.first_reader_bytes(observer.path, raw) as readers, \
|
||||
self.assertLogs(level='WARNING'), self.assertRaises(error):
|
||||
observer.poll()
|
||||
self.assertEqual(len(readers), 2)
|
||||
self.assertTrue(all(reader.closed for reader in readers))
|
||||
self.assertEqual(observer.offset, 0)
|
||||
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
|
||||
self.assertEqual(observer.path.read_bytes(), raw)
|
||||
|
||||
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
|
||||
from harness import collection_runtime
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
rd = Path(tmp)
|
||||
owner = execution.Execution(rd); owner.start(policy())
|
||||
(rd / 'agent').mkdir()
|
||||
observer = collection_runtime.ToolDeadlines(rd, owner)
|
||||
original = b'{"type":"note","value":"complete JSON but no newline"}'
|
||||
observer.path.write_bytes(original)
|
||||
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
|
||||
self.assertLogs(level='WARNING'), \
|
||||
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
|
||||
observer.poll()
|
||||
self.assertEqual(len(readers), 2)
|
||||
self.assertTrue(all(reader.closed for reader in readers))
|
||||
self.assertEqual(observer.offset, 0)
|
||||
self.assertEqual(observer.path.read_bytes(), original)
|
||||
|
||||
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
|
||||
from harness import collection_runtime
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user