fix: retry transient tool event reads once

This commit is contained in:
ming01.wang 2026-09-24 02:00:12 +08:00
parent b398495d54
commit 8bb1694499
2 changed files with 113 additions and 2 deletions

View File

@ -2,8 +2,10 @@
from __future__ import annotations
import atexit
from contextlib import ExitStack
import hashlib
import json
import logging
from pathlib import Path
import threading
@ -162,12 +164,26 @@ class ToolDeadlines:
# a DONE arriving during lock contention must precede timeout judgment.
nonlocal offset
if self.path.exists():
with self.path.open('rb') as stream:
with ExitStack() as readers:
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
while line := stream.readline():
if not line.endswith(b'\n'):
break # Partial boundaries are retried on the next poll.
event = json.loads(line)
try:
event = json.loads(line.decode('utf-8'))
except (UnicodeDecodeError, json.JSONDecodeError) as error:
logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s',
self.path, offset, len(line), hashlib.sha256(line).hexdigest(),
type(error).__name__)
# Discard buffered bytes, including any following lines.
stream.close()
stream = readers.enter_context(self.path.open('rb'))
stream.seek(offset)
line = stream.readline()
if not line.endswith(b'\n'):
raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error
event = json.loads(line.decode('utf-8'))
offset = stream.tell()
if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'):
yield event['bench_tool']

View File

@ -1,4 +1,5 @@
"""Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only."""
import io
import json
import shlex
import sys
@ -20,6 +21,100 @@ from scripts.test_quota import ok
@unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required')
class ToolBoundaryCommit(unittest.TestCase):
@contextmanager
def first_reader_bytes(self, path, raw):
"""Expose a stale first read without changing the real event evidence."""
original = Path.open
opened = []
def opening(target, mode='r', *args, **kwargs):
if target == path and mode == 'rb':
stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs)
opened.append(stream)
return stream
return original(target, mode, *args, **kwargs)
with patch.object(Path, 'open', new=opening):
yield opened
def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self):
from harness import collection_runtime
for bad in (b'{"type":"note","value":"\x00"}\n',
b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'):
with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
path = rd / 'agent/events.jsonl'
prefix = b'{"type":"already_consumed"}\n'
path.write_bytes(prefix)
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot(); started = time.monotonic()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='completed', tool_name='bash', state='RUNNING',
started_monotonic=started, cap_s=10)
start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode()
done = dict(start, state='DONE', observed_monotonic=started + 1)
done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode()
good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode()
original = prefix + start_line + good + done_line
path.write_bytes(original)
stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n'
with self.first_reader_bytes(path, stale) as readers, \
patch.object(execution.time, 'monotonic', return_value=started + 11), \
patch.object(collection_runtime, 'broker') as broker, \
self.assertLogs(level='WARNING') as logs:
self.assertIsNone(observer.poll())
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
broker.assert_not_called()
self.assertEqual(len(logs.output), 1)
self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0])
self.assertNotIn('完整原始证据', logs.output[0])
self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE')
self.assertEqual(observer.offset, len(original))
self.assertEqual(path.read_bytes(), original)
def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self):
from harness import collection_runtime
for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError),
(b'{"value":"\xff"}\n', UnicodeDecodeError)):
with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
state = owner.snapshot()
start = dict(execution_id=state['execution_id'], epoch=state['epoch'],
tool_call_id='uncommitted', tool_name='bash', state='RUNNING',
started_monotonic=time.monotonic(), cap_s=10)
raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad
observer.path.write_bytes(raw)
with self.first_reader_bytes(observer.path, raw) as readers, \
self.assertLogs(level='WARNING'), self.assertRaises(error):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(owner.snapshot().get('tool_calls', {}), {})
self.assertEqual(observer.path.read_bytes(), raw)
def test_reread_without_complete_newline_fails_without_committing_cursor(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp:
rd = Path(tmp)
owner = execution.Execution(rd); owner.start(policy())
(rd / 'agent').mkdir()
observer = collection_runtime.ToolDeadlines(rd, owner)
original = b'{"type":"note","value":"complete JSON but no newline"}'
observer.path.write_bytes(original)
with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \
self.assertLogs(level='WARNING'), \
self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'):
observer.poll()
self.assertEqual(len(readers), 2)
self.assertTrue(all(reader.closed for reader in readers))
self.assertEqual(observer.offset, 0)
self.assertEqual(observer.path.read_bytes(), original)
def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self):
from harness import collection_runtime
with tempfile.TemporaryDirectory() as tmp: