From 8bb16944995685eafdf622bf12ef7aa8e136a29a Mon Sep 17 00:00:00 2001 From: "ming01.wang" Date: Thu, 24 Sep 2026 02:00:12 +0800 Subject: [PATCH] fix: retry transient tool event reads once --- harness/collection_runtime.py | 20 +++++- scripts/test_ordinary_tool_deadline.py | 95 ++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/harness/collection_runtime.py b/harness/collection_runtime.py index 6f756c0..d1fb312 100644 --- a/harness/collection_runtime.py +++ b/harness/collection_runtime.py @@ -2,8 +2,10 @@ from __future__ import annotations import atexit +from contextlib import ExitStack import hashlib import json +import logging from pathlib import Path import threading @@ -162,12 +164,26 @@ class ToolDeadlines: # a DONE arriving during lock contention must precede timeout judgment. nonlocal offset if self.path.exists(): - with self.path.open('rb') as stream: + with ExitStack() as readers: + stream = readers.enter_context(self.path.open('rb')) stream.seek(offset) while line := stream.readline(): if not line.endswith(b'\n'): break # Partial boundaries are retried on the next poll. - event = json.loads(line) + try: + event = json.loads(line.decode('utf-8')) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + logging.warning('TOOL_EVENT_REREAD path=%s offset=%d bytes=%d sha256=%s error=%s', + self.path, offset, len(line), hashlib.sha256(line).hexdigest(), + type(error).__name__) + # Discard buffered bytes, including any following lines. + stream.close() + stream = readers.enter_context(self.path.open('rb')) + stream.seek(offset) + line = stream.readline() + if not line.endswith(b'\n'): + raise ValueError('INCOMPLETE_TOOL_EVENT_REREAD') from error + event = json.loads(line.decode('utf-8')) offset = stream.tell() if event.get('type') in ('tool_execution_start', 'tool_execution_end') and event.get('bench_tool'): yield event['bench_tool'] diff --git a/scripts/test_ordinary_tool_deadline.py b/scripts/test_ordinary_tool_deadline.py index 33b0ad4..9c864f9 100644 --- a/scripts/test_ordinary_tool_deadline.py +++ b/scripts/test_ordinary_tool_deadline.py @@ -1,4 +1,5 @@ """Ordinary Pi tools: real SDK/collector and owned Linux scopes, local HTTP only.""" +import io import json import shlex import sys @@ -20,6 +21,100 @@ from scripts.test_quota import ok @unittest.skipUnless(sys.platform == 'linux', 'Linux execution identity required') class ToolBoundaryCommit(unittest.TestCase): + @contextmanager + def first_reader_bytes(self, path, raw): + """Expose a stale first read without changing the real event evidence.""" + original = Path.open + opened = [] + def opening(target, mode='r', *args, **kwargs): + if target == path and mode == 'rb': + stream = io.BytesIO(raw) if not opened else original(target, mode, *args, **kwargs) + opened.append(stream) + return stream + return original(target, mode, *args, **kwargs) + with patch.object(Path, 'open', new=opening): + yield opened + + def test_transient_bad_read_reopens_at_boundary_and_discards_stale_tail(self): + from harness import collection_runtime + for bad in (b'{"type":"note","value":"\x00"}\n', + b'{"type":"note","value":"' + b'x' * 1000 + b'\xff"}\n'): + with self.subTest(bad_bytes=len(bad)), tempfile.TemporaryDirectory() as tmp: + rd = Path(tmp) + owner = execution.Execution(rd); owner.start(policy()) + (rd / 'agent').mkdir() + path = rd / 'agent/events.jsonl' + prefix = b'{"type":"already_consumed"}\n' + path.write_bytes(prefix) + observer = collection_runtime.ToolDeadlines(rd, owner) + state = owner.snapshot(); started = time.monotonic() + start = dict(execution_id=state['execution_id'], epoch=state['epoch'], + tool_call_id='completed', tool_name='bash', state='RUNNING', + started_monotonic=started, cap_s=10) + start_line = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + done = dict(start, state='DONE', observed_monotonic=started + 1) + done_line = (json.dumps({'type': 'tool_execution_end', 'bench_tool': done}) + '\n').encode() + good = '{"type":"note","value":"完整原始证据保持不变"}\n'.encode() + original = prefix + start_line + good + done_line + path.write_bytes(original) + stale = prefix + start_line + bad + b'{"type":"stale_tail_without_done"}\n' + with self.first_reader_bytes(path, stale) as readers, \ + patch.object(execution.time, 'monotonic', return_value=started + 11), \ + patch.object(collection_runtime, 'broker') as broker, \ + self.assertLogs(level='WARNING') as logs: + self.assertIsNone(observer.poll()) + self.assertEqual(len(readers), 2) + self.assertTrue(all(reader.closed for reader in readers)) + broker.assert_not_called() + self.assertEqual(len(logs.output), 1) + self.assertIn(f'offset={len(prefix + start_line)}', logs.output[0]) + self.assertNotIn('完整原始证据', logs.output[0]) + self.assertEqual(owner.snapshot()['tool_calls']['completed']['state'], 'DONE') + self.assertEqual(observer.offset, len(original)) + self.assertEqual(path.read_bytes(), original) + + def test_persistent_bad_read_fails_once_without_committing_cursor_or_tools(self): + from harness import collection_runtime + for bad, error in ((b'{"value":"\x00"}\n', json.JSONDecodeError), + (b'{"value":"\xff"}\n', UnicodeDecodeError)): + with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as tmp: + rd = Path(tmp) + owner = execution.Execution(rd); owner.start(policy()) + (rd / 'agent').mkdir() + observer = collection_runtime.ToolDeadlines(rd, owner) + state = owner.snapshot() + start = dict(execution_id=state['execution_id'], epoch=state['epoch'], + tool_call_id='uncommitted', tool_name='bash', state='RUNNING', + started_monotonic=time.monotonic(), cap_s=10) + raw = (json.dumps({'type': 'tool_execution_start', 'bench_tool': start}) + '\n').encode() + bad + observer.path.write_bytes(raw) + with self.first_reader_bytes(observer.path, raw) as readers, \ + self.assertLogs(level='WARNING'), self.assertRaises(error): + observer.poll() + self.assertEqual(len(readers), 2) + self.assertTrue(all(reader.closed for reader in readers)) + self.assertEqual(observer.offset, 0) + self.assertEqual(owner.snapshot().get('tool_calls', {}), {}) + self.assertEqual(observer.path.read_bytes(), raw) + + def test_reread_without_complete_newline_fails_without_committing_cursor(self): + from harness import collection_runtime + with tempfile.TemporaryDirectory() as tmp: + rd = Path(tmp) + owner = execution.Execution(rd); owner.start(policy()) + (rd / 'agent').mkdir() + observer = collection_runtime.ToolDeadlines(rd, owner) + original = b'{"type":"note","value":"complete JSON but no newline"}' + observer.path.write_bytes(original) + with self.first_reader_bytes(observer.path, b'{"value":"\xff"}\n') as readers, \ + self.assertLogs(level='WARNING'), \ + self.assertRaisesRegex(ValueError, 'INCOMPLETE_TOOL_EVENT_REREAD'): + observer.poll() + self.assertEqual(len(readers), 2) + self.assertTrue(all(reader.closed for reader in readers)) + self.assertEqual(observer.offset, 0) + self.assertEqual(observer.path.read_bytes(), original) + def test_done_arriving_while_observer_waits_for_lock_is_not_timed_out(self): from harness import collection_runtime with tempfile.TemporaryDirectory() as tmp: