diff --git a/harness/salty_dev.py b/harness/salty_dev.py index bd685ce..877bcfd 100644 --- a/harness/salty_dev.py +++ b/harness/salty_dev.py @@ -28,7 +28,9 @@ from . import execution, paths, salty_remote from .salty_client import TransportError from .watchdog import RunResult -EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'} +# .bench-session.json.tmp: pi_session's atomic write; listed, then renamed away before tar read it +# (10-20 20:17 and twice on 10-16: FileNotFoundError, the dev command died as STAGE_EXECUTOR_EXIT). +EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'} UPLOAD_LIMIT = 256 << 20 # the Mac mini uplink is ~1 MB/s; big inputs are made inside bench dev GRACE_S = 300 FETCH_MARGIN_S = 120 # the command ends this long before the request deadline so its results still come back @@ -68,6 +70,22 @@ def synced(rd: Path): os.replace(tmp, path) +def memory_gi(policy, ctx, rd: Path) -> int: + """The heaviest dev use is compiling this model: at least what its formal compiles needed (salty_remote._memory + doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). A dev job of this rollout that failed (an OOM + gives no reason) doubles what it had, cap 128Gi: the agent cannot raise the limit, and retrying at the same size + failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi).""" + gi = max(int(str(m).removesuffix('Gi')) for m in ( + (policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')), + salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx))) + # ponytail: this rollout only; a new rollout of the same model starts from the formal compiles' floor again. + for f in (rd / 'stages' / '_queue').glob('terminal-*/salty/remote.json'): + state = json.loads(f.read_text()) + if state.get('job_state') == 'failed' and state.get('memory'): + gi = max(gi, 2 * int(state['memory'].removesuffix('Gi'))) + return min(gi, 128) + + def bundle(path: Path, ws: Path, uploads: list, request: dict) -> None: with tarfile.open(path, 'x') as tar: data = json.dumps(request).encode() @@ -201,9 +219,7 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat bundle(adapter / 'bundle.tar', ws, uploads, request) shutil.copyfile(salty_remote.WORKER, adapter / 'worker.py') try: - # The heaviest dev use is compiling this model: give it at least what its formal compiles needed - # (salty_remote._memory doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). - memory = f"{max(int(str(m).removesuffix('Gi')) for m in ((policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')), salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))}Gi" + memory = state['memory'] = f'{memory_gi(policy, ctx, rd)}Gi' state['job_id'] = salty_remote._submit(cli, state, policy, SimpleNamespace(name='terminal'), adapter, memory) execution.atomic_json(state_file, state) with synced(rd) as manifest: # the remote copy has them now, whatever the command does @@ -216,11 +232,14 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat out = cli.call('describe', state['job_id'])[1] job_state = (re.search(r'^ηŠΆζ€:\s*(\w+)', out, re.MULTILINE) or [None, None])[1] if job_state != 'succeeded': + state['job_state'] = job_state + execution.atomic_json(state_file, state) salty_remote._remote_logs(policy, state['job_id'], stderr) # A container OOM gives no reason; its memory log shows it (the agent cannot raise the limit itself). if cli.once(['storage', 'cp', state['remote'] + '/usage.log', str(adapter / 'usage.log')])[0] == 0: + more = f"; the next bench dev command gets {min(2 * int(memory.removesuffix('Gi')), 128)}Gi" if job_state == 'failed' else '' with stderr.open('a') as f: - f.write(f'\n[bench dev container: {memory} memory; usage every 15s β€” a peak near the limit means it ran out of memory]\n' + f.write(f'\n[bench dev container: {memory} memory; usage every 15s β€” a peak near the limit means it ran out of memory{more}]\n' + (adapter / 'usage.log').read_text()[-4000:]) return done('FAIL', None, 'TERMINAL_DEADLINE_EXCEEDED' if remaining() < 0 else 'REMOTE_WORKER_FAILED') for f in ('result.json', 'outputs.tar', 'stdout.log', 'stderr.log'): diff --git a/harness/terminal.py b/harness/terminal.py index eef657b..aeb4e18 100644 --- a/harness/terminal.py +++ b/harness/terminal.py @@ -120,7 +120,7 @@ def freeze(source, destination): source = Path(source) if source.is_symlink() or source.resolve() != source.absolute() or not source.is_dir(): raise ValueError('terminal source must be a real directory') - excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'} + excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'} files, total = {}, 0 destination.mkdir() def copy(directory_fd, target, rel=Path('.')): diff --git a/scripts/test_salty_remote.py b/scripts/test_salty_remote.py index 3486931..3bc2e2c 100644 --- a/scripts/test_salty_remote.py +++ b/scripts/test_salty_remote.py @@ -92,6 +92,7 @@ def main(): failure_meta_check() usage_log_check() memory_escalation_check() + dev_memory_check() partial_describe_check() async_cleanup_check() cleanup_result_check() @@ -230,6 +231,31 @@ def memory_escalation_check(): assert salty_remote._memory(policy, stage, ctx) == expected, path +def dev_memory_check(): + """bench dev: a failed dev job of this rollout doubles what it had (the agent cannot raise it); the pi checkpoint's + temp file is never uploaded (it vanishes mid-bundle).""" + from harness import salty_dev + with tempfile.TemporaryDirectory() as tmp: + ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/cls--m--nash-p--ptq--fp16int8") + rd = ctx.job_dir / "rollouts/02" + policy = {"memory": "8Gi", "memory_by_stage": {"terminal": "16Gi"}} + assert salty_dev.memory_gi(policy, ctx, rd) == 16 + for name, state, expected in (("terminal-a", {"memory": "16Gi", "job_state": "stopped"}, 16), + ("terminal-b", {"memory": "16Gi"}, 16), # succeeded: nothing recorded + ("terminal-c", {"memory": "16Gi", "job_state": "failed"}, 32), + ("terminal-d", {"memory": "128Gi", "job_state": "failed"}, 128)): + d = rd / "stages/_queue" / name / "salty" + d.mkdir(parents=True) + (d / "remote.json").write_text(json.dumps(state)) + assert salty_dev.memory_gi(policy, ctx, rd) == expected, name + assert salty_dev.memory_gi(policy, ctx, ctx.job_dir / "rollouts/03") == 16 # other rollout + ws = Path(tmp) / "ws" + ws.mkdir() + for f in (".bench-session.json", ".bench-session.json.tmp", "a.py"): + (ws / f).write_text("x") + assert list(salty_dev.local_files(ws)) == ["a.py"] + + def usage_log_check(): import threading, time with tempfile.TemporaryDirectory() as tmp: