bench dev: double memory after a failed dev job; never upload the pi checkpoint temp file

A dev container OOM (no reason given, the agent cannot raise the limit) was retried
at the same size and failed again every time (10-20: efficientnet.b6 3x at 16Gi,
unidepth.v2.vit.l 2x at 32Gi). remote.json now records memory and job_state; a failed
dev job of the rollout doubles the next one's memory (cap 128Gi) and stderr says so.

.bench-session.json.tmp (pi_session atomic write) was listed then renamed away before
tar read it: FileNotFoundError, dev command died as STAGE_EXECUTOR_EXIT (3x since 10-16).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bench 2026-10-20 21:02:23 +08:00
parent 16b99ee1ed
commit 50f163e993
3 changed files with 51 additions and 6 deletions

View File

@ -28,7 +28,9 @@ from . import execution, paths, salty_remote
from .salty_client import TransportError
from .watchdog import RunResult
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'}
# .bench-session.json.tmp: pi_session's atomic write; listed, then renamed away before tar read it
# (10-20 20:17 and twice on 10-16: FileNotFoundError, the dev command died as STAGE_EXECUTOR_EXIT).
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
UPLOAD_LIMIT = 256 << 20 # the Mac mini uplink is ~1 MB/s; big inputs are made inside bench dev
GRACE_S = 300
FETCH_MARGIN_S = 120 # the command ends this long before the request deadline so its results still come back
@ -68,6 +70,22 @@ def synced(rd: Path):
os.replace(tmp, path)
def memory_gi(policy, ctx, rd: Path) -> int:
"""The heaviest dev use is compiling this model: at least what its formal compiles needed (salty_remote._memory
doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). A dev job of this rollout that failed (an OOM
gives no reason) doubles what it had, cap 128Gi: the agent cannot raise the limit, and retrying at the same size
failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi)."""
gi = max(int(str(m).removesuffix('Gi')) for m in (
(policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')),
salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))
# ponytail: this rollout only; a new rollout of the same model starts from the formal compiles' floor again.
for f in (rd / 'stages' / '_queue').glob('terminal-*/salty/remote.json'):
state = json.loads(f.read_text())
if state.get('job_state') == 'failed' and state.get('memory'):
gi = max(gi, 2 * int(state['memory'].removesuffix('Gi')))
return min(gi, 128)
def bundle(path: Path, ws: Path, uploads: list, request: dict) -> None:
with tarfile.open(path, 'x') as tar:
data = json.dumps(request).encode()
@ -201,9 +219,7 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat
bundle(adapter / 'bundle.tar', ws, uploads, request)
shutil.copyfile(salty_remote.WORKER, adapter / 'worker.py')
try:
# The heaviest dev use is compiling this model: give it at least what its formal compiles needed
# (salty_remote._memory doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev).
memory = f"{max(int(str(m).removesuffix('Gi')) for m in ((policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')), salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))}Gi"
memory = state['memory'] = f'{memory_gi(policy, ctx, rd)}Gi'
state['job_id'] = salty_remote._submit(cli, state, policy, SimpleNamespace(name='terminal'), adapter, memory)
execution.atomic_json(state_file, state)
with synced(rd) as manifest: # the remote copy has them now, whatever the command does
@ -216,11 +232,14 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat
out = cli.call('describe', state['job_id'])[1]
job_state = (re.search(r'^状态:\s*(\w+)', out, re.MULTILINE) or [None, None])[1]
if job_state != 'succeeded':
state['job_state'] = job_state
execution.atomic_json(state_file, state)
salty_remote._remote_logs(policy, state['job_id'], stderr)
# A container OOM gives no reason; its memory log shows it (the agent cannot raise the limit itself).
if cli.once(['storage', 'cp', state['remote'] + '/usage.log', str(adapter / 'usage.log')])[0] == 0:
more = f"; the next bench dev command gets {min(2 * int(memory.removesuffix('Gi')), 128)}Gi" if job_state == 'failed' else ''
with stderr.open('a') as f:
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory]\n'
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory{more}]\n'
+ (adapter / 'usage.log').read_text()[-4000:])
return done('FAIL', None, 'TERMINAL_DEADLINE_EXCEEDED' if remaining() < 0 else 'REMOTE_WORKER_FAILED')
for f in ('result.json', 'outputs.tar', 'stdout.log', 'stderr.log'):

View File

@ -120,7 +120,7 @@ def freeze(source, destination):
source = Path(source)
if source.is_symlink() or source.resolve() != source.absolute() or not source.is_dir():
raise ValueError('terminal source must be a real directory')
excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'}
excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
files, total = {}, 0
destination.mkdir()
def copy(directory_fd, target, rel=Path('.')):

View File

@ -92,6 +92,7 @@ def main():
failure_meta_check()
usage_log_check()
memory_escalation_check()
dev_memory_check()
partial_describe_check()
async_cleanup_check()
cleanup_result_check()
@ -230,6 +231,31 @@ def memory_escalation_check():
assert salty_remote._memory(policy, stage, ctx) == expected, path
def dev_memory_check():
"""bench dev: a failed dev job of this rollout doubles what it had (the agent cannot raise it); the pi checkpoint's
temp file is never uploaded (it vanishes mid-bundle)."""
from harness import salty_dev
with tempfile.TemporaryDirectory() as tmp:
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/cls--m--nash-p--ptq--fp16int8")
rd = ctx.job_dir / "rollouts/02"
policy = {"memory": "8Gi", "memory_by_stage": {"terminal": "16Gi"}}
assert salty_dev.memory_gi(policy, ctx, rd) == 16
for name, state, expected in (("terminal-a", {"memory": "16Gi", "job_state": "stopped"}, 16),
("terminal-b", {"memory": "16Gi"}, 16), # succeeded: nothing recorded
("terminal-c", {"memory": "16Gi", "job_state": "failed"}, 32),
("terminal-d", {"memory": "128Gi", "job_state": "failed"}, 128)):
d = rd / "stages/_queue" / name / "salty"
d.mkdir(parents=True)
(d / "remote.json").write_text(json.dumps(state))
assert salty_dev.memory_gi(policy, ctx, rd) == expected, name
assert salty_dev.memory_gi(policy, ctx, ctx.job_dir / "rollouts/03") == 16 # other rollout
ws = Path(tmp) / "ws"
ws.mkdir()
for f in (".bench-session.json", ".bench-session.json.tmp", "a.py"):
(ws / f).write_text("x")
assert list(salty_dev.local_files(ws)) == ["a.py"]
def usage_log_check():
import threading, time
with tempfile.TemporaryDirectory() as tmp: