diff --git a/harness/collection_runtime.py b/harness/collection_runtime.py index a9da14b..f8337f6 100644 --- a/harness/collection_runtime.py +++ b/harness/collection_runtime.py @@ -207,7 +207,8 @@ def freeze_checkpoint(rd, execution): path = Path(checkpoint['session_path']) if not path.resolve().is_relative_to((rd / 'workspace/.pi-sessions').resolve()): raise ValueError('CHECKPOINT_PATH_INVALID') - data = config_bytes(rd / 'workspace', str(path)) + # A long rollout's session passes 1 MiB (10-15..10-17: 17 rollouts, 4 ended as INFRA_ERROR); no size cap here. + data = config_bytes(rd / 'workspace', str(path), limit=None) if hashlib.sha256(data).hexdigest() != checkpoint.get('sha256'): raise ValueError('CHECKPOINT_CHANGED') # Binary session snapshots require fsync, like JSON lifecycle records. diff --git a/harness/stageworker.py b/harness/stageworker.py index e41778e..d7413b2 100644 --- a/harness/stageworker.py +++ b/harness/stageworker.py @@ -19,17 +19,20 @@ from . import stagerun, execution as lifecycle, watchdog from .stages import AGENT_RUNNABLE -def read_regular(parent_fd, name): +def read_regular(parent_fd, name, limit=1024 * 1024): fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent_fd) try: - if not stat.S_ISREG(os.fstat(fd).st_mode) or os.fstat(fd).st_size > 1024 * 1024: + if not stat.S_ISREG(os.fstat(fd).st_mode) or (limit is not None and os.fstat(fd).st_size > limit): raise ValueError('ticket/config must be a regular file <= 1 MiB') - return os.read(fd, 1024 * 1024 + 1) + if limit is not None: + return os.read(fd, limit + 1) + with os.fdopen(os.dup(fd), 'rb') as stream: + return stream.read() finally: os.close(fd) -def config_bytes(ws, value): +def config_bytes(ws, value, limit=1024 * 1024): p = Path(value) rel = p.relative_to(ws) if not rel.parts or '..' in rel.parts: @@ -39,7 +42,7 @@ def config_bytes(ws, value): for part in rel.parts[:-1]: nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) os.close(fd); fd = nxt - return read_regular(fd, rel.name) + return read_regular(fd, rel.name, limit) finally: os.close(fd) diff --git a/scripts/test_codex.py b/scripts/test_codex.py index 83c9f91..06cf36f 100644 --- a/scripts/test_codex.py +++ b/scripts/test_codex.py @@ -43,6 +43,11 @@ class Regression(unittest.TestCase): self.assertEqual(config_bytes(ws,str(ws/'ok')),b'x') for p in [ws/'link', ws/'../outside']: with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p)) + with self.assertRaises((OSError,ValueError)): config_bytes(ws,str(p),limit=None) + # A long Pi session is several MiB: capped for configs, whole for the frozen checkpoint. + (ws/'session.jsonl').write_bytes(b'x'*(3<<20)) + with self.assertRaises(ValueError): config_bytes(ws,str(ws/'session.jsonl')) + self.assertEqual(len(config_bytes(ws,str(ws/'session.jsonl'),limit=None)),3<<20) def test_worker_config_file_paths(self): with tempfile.TemporaryDirectory() as d: