bench dev: double memory after a failed dev job; never upload the pi checkpoint temp file
A dev container OOM (no reason given, the agent cannot raise the limit) was retried at the same size and failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi). remote.json now records memory and job_state; a failed dev job of the rollout doubles the next one's memory (cap 128Gi) and stderr says so. .bench-session.json.tmp (pi_session atomic write) was listed then renamed away before tar read it: FileNotFoundError, dev command died as STAGE_EXECUTOR_EXIT (3x since 10-16). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
16b99ee1ed
commit
50f163e993
@ -28,7 +28,9 @@ from . import execution, paths, salty_remote
|
||||
from .salty_client import TransportError
|
||||
from .watchdog import RunResult
|
||||
|
||||
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'}
|
||||
# .bench-session.json.tmp: pi_session's atomic write; listed, then renamed away before tar read it
|
||||
# (10-20 20:17 and twice on 10-16: FileNotFoundError, the dev command died as STAGE_EXECUTOR_EXIT).
|
||||
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
|
||||
UPLOAD_LIMIT = 256 << 20 # the Mac mini uplink is ~1 MB/s; big inputs are made inside bench dev
|
||||
GRACE_S = 300
|
||||
FETCH_MARGIN_S = 120 # the command ends this long before the request deadline so its results still come back
|
||||
@ -68,6 +70,22 @@ def synced(rd: Path):
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def memory_gi(policy, ctx, rd: Path) -> int:
|
||||
"""The heaviest dev use is compiling this model: at least what its formal compiles needed (salty_remote._memory
|
||||
doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). A dev job of this rollout that failed (an OOM
|
||||
gives no reason) doubles what it had, cap 128Gi: the agent cannot raise the limit, and retrying at the same size
|
||||
failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi)."""
|
||||
gi = max(int(str(m).removesuffix('Gi')) for m in (
|
||||
(policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')),
|
||||
salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))
|
||||
# ponytail: this rollout only; a new rollout of the same model starts from the formal compiles' floor again.
|
||||
for f in (rd / 'stages' / '_queue').glob('terminal-*/salty/remote.json'):
|
||||
state = json.loads(f.read_text())
|
||||
if state.get('job_state') == 'failed' and state.get('memory'):
|
||||
gi = max(gi, 2 * int(state['memory'].removesuffix('Gi')))
|
||||
return min(gi, 128)
|
||||
|
||||
|
||||
def bundle(path: Path, ws: Path, uploads: list, request: dict) -> None:
|
||||
with tarfile.open(path, 'x') as tar:
|
||||
data = json.dumps(request).encode()
|
||||
@ -201,9 +219,7 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat
|
||||
bundle(adapter / 'bundle.tar', ws, uploads, request)
|
||||
shutil.copyfile(salty_remote.WORKER, adapter / 'worker.py')
|
||||
try:
|
||||
# The heaviest dev use is compiling this model: give it at least what its formal compiles needed
|
||||
# (salty_remote._memory doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev).
|
||||
memory = f"{max(int(str(m).removesuffix('Gi')) for m in ((policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')), salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))}Gi"
|
||||
memory = state['memory'] = f'{memory_gi(policy, ctx, rd)}Gi'
|
||||
state['job_id'] = salty_remote._submit(cli, state, policy, SimpleNamespace(name='terminal'), adapter, memory)
|
||||
execution.atomic_json(state_file, state)
|
||||
with synced(rd) as manifest: # the remote copy has them now, whatever the command does
|
||||
@ -216,11 +232,14 @@ def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Pat
|
||||
out = cli.call('describe', state['job_id'])[1]
|
||||
job_state = (re.search(r'^状态:\s*(\w+)', out, re.MULTILINE) or [None, None])[1]
|
||||
if job_state != 'succeeded':
|
||||
state['job_state'] = job_state
|
||||
execution.atomic_json(state_file, state)
|
||||
salty_remote._remote_logs(policy, state['job_id'], stderr)
|
||||
# A container OOM gives no reason; its memory log shows it (the agent cannot raise the limit itself).
|
||||
if cli.once(['storage', 'cp', state['remote'] + '/usage.log', str(adapter / 'usage.log')])[0] == 0:
|
||||
more = f"; the next bench dev command gets {min(2 * int(memory.removesuffix('Gi')), 128)}Gi" if job_state == 'failed' else ''
|
||||
with stderr.open('a') as f:
|
||||
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory]\n'
|
||||
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory{more}]\n'
|
||||
+ (adapter / 'usage.log').read_text()[-4000:])
|
||||
return done('FAIL', None, 'TERMINAL_DEADLINE_EXCEEDED' if remaining() < 0 else 'REMOTE_WORKER_FAILED')
|
||||
for f in ('result.json', 'outputs.tar', 'stdout.log', 'stderr.log'):
|
||||
|
||||
@ -120,7 +120,7 @@ def freeze(source, destination):
|
||||
source = Path(source)
|
||||
if source.is_symlink() or source.resolve() != source.absolute() or not source.is_dir():
|
||||
raise ValueError('terminal source must be a real directory')
|
||||
excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.env', '.env.local', 'guard.log'}
|
||||
excluded = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
|
||||
files, total = {}, 0
|
||||
destination.mkdir()
|
||||
def copy(directory_fd, target, rel=Path('.')):
|
||||
|
||||
@ -92,6 +92,7 @@ def main():
|
||||
failure_meta_check()
|
||||
usage_log_check()
|
||||
memory_escalation_check()
|
||||
dev_memory_check()
|
||||
partial_describe_check()
|
||||
async_cleanup_check()
|
||||
cleanup_result_check()
|
||||
@ -230,6 +231,31 @@ def memory_escalation_check():
|
||||
assert salty_remote._memory(policy, stage, ctx) == expected, path
|
||||
|
||||
|
||||
def dev_memory_check():
|
||||
"""bench dev: a failed dev job of this rollout doubles what it had (the agent cannot raise it); the pi checkpoint's
|
||||
temp file is never uploaded (it vanishes mid-bundle)."""
|
||||
from harness import salty_dev
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/cls--m--nash-p--ptq--fp16int8")
|
||||
rd = ctx.job_dir / "rollouts/02"
|
||||
policy = {"memory": "8Gi", "memory_by_stage": {"terminal": "16Gi"}}
|
||||
assert salty_dev.memory_gi(policy, ctx, rd) == 16
|
||||
for name, state, expected in (("terminal-a", {"memory": "16Gi", "job_state": "stopped"}, 16),
|
||||
("terminal-b", {"memory": "16Gi"}, 16), # succeeded: nothing recorded
|
||||
("terminal-c", {"memory": "16Gi", "job_state": "failed"}, 32),
|
||||
("terminal-d", {"memory": "128Gi", "job_state": "failed"}, 128)):
|
||||
d = rd / "stages/_queue" / name / "salty"
|
||||
d.mkdir(parents=True)
|
||||
(d / "remote.json").write_text(json.dumps(state))
|
||||
assert salty_dev.memory_gi(policy, ctx, rd) == expected, name
|
||||
assert salty_dev.memory_gi(policy, ctx, ctx.job_dir / "rollouts/03") == 16 # other rollout
|
||||
ws = Path(tmp) / "ws"
|
||||
ws.mkdir()
|
||||
for f in (".bench-session.json", ".bench-session.json.tmp", "a.py"):
|
||||
(ws / f).write_text("x")
|
||||
assert list(salty_dev.local_files(ws)) == ["a.py"]
|
||||
|
||||
|
||||
def usage_log_check():
|
||||
import threading, time
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user