feat: salty_remote 按题记内存需求:同一 job 任何轨迹里该阶段被 OOM 杀过,就从当时内存的两倍起步

内存需要是模型的属性;之前按单条轨迹计数,补号重跑的轨迹又从 16Gi 起,重复被杀。
meta 里记 salty_memory。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bench 2026-10-11 10:36:40 +08:00
parent 45cff52749
commit c8a93c9605
2 changed files with 26 additions and 17 deletions

View File

@ -156,15 +156,16 @@ class _Cli:
def _memory(policy, stage, ctx) -> str:
"""Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. Each
earlier attempt of this stage whose job failed without a result doubles the request (cap 128Gi, near the old compile server), so
only the models that need it hold more of the shared queue."""
base = (policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi"))
died = 0
for meta in (ctx.stage_root / stage.name).glob("a*/meta.json"):
"""Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. The need is the
model's, so a failed (not stopped) job of this stage in any rollout of this job sets the floor: twice the
memory it had, cap 128Gi (near the old compile server). Only the models that need it hold more of the queue."""
base = int((policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")).removesuffix("Gi"))
gi = base
for meta in ctx.job_dir.glob(f"rollouts/*/stages/{stage.name}/a*/meta.json"):
m = json.loads(meta.read_text())
died += m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed"
return f"{min(int(base.removesuffix('Gi')) * 2 ** died, 128)}Gi"
if m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed":
gi = max(gi, 2 * int(str(m.get("salty_memory") or f"{base}Gi").removesuffix("Gi")))
return f"{min(gi, 128)}Gi"
def _submit(cli, state, policy, stage, adapter, memory) -> str:
@ -259,10 +260,11 @@ def _run(stage, ctx, *, sleep, clock):
state = {"name": name, "remote": "personal:/" + name, "job_id": None}
atomic_json(state_file, state)
if not state["job_id"]:
state["job_id"] = _submit(cli, state, policy, stage, adapter, _memory(policy, stage, ctx))
state["memory"] = _memory(policy, stage, ctx)
state["job_id"] = _submit(cli, state, policy, stage, adapter, state["memory"])
atomic_json(state_file, state)
job_state = _wait(cli, state["job_id"], policy, started, clock, sleep)
meta = {"executor": "salty", "salty_job_id": state["job_id"]}
meta = {"executor": "salty", "salty_job_id": state["job_id"], "salty_memory": state.get("memory")}
if job_state != "succeeded":
# The worker never reached a result: no stage outcome exists, board receipts decide quarantine.
_remote_logs(policy, state["job_id"], sd / "stderr.log")

View File

@ -81,18 +81,25 @@ def main():
def memory_escalation_check():
"""Only attempts whose job failed (not stopped) double the next request; capped at 128Gi."""
"""A failed (not stopped) job in any rollout of the job doubles the memory it had; capped at 128Gi."""
with tempfile.TemporaryDirectory() as tmp:
ctx = SimpleNamespace(stage_root=Path(tmp))
ctx = SimpleNamespace(job_dir=Path(tmp))
stage = SimpleNamespace(name="05-compile")
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
for i, job_state in enumerate(("failed", "stopped", "failed", "failed", "failed")):
(Path(tmp) / f"05-compile/a{i:02d}").mkdir(parents=True)
(Path(tmp) / f"05-compile/a{i:02d}/meta.json").write_text(json.dumps(
{"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state}))
assert salty_remote._memory(policy, stage, ctx) == ("32Gi", "32Gi", "64Gi", "128Gi", "128Gi")[i], i
cases = (("00/stages/05-compile/a00", "failed", None, "32Gi"), # old meta: no memory recorded
("00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
("01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # another rollout of the same job
("01/stages/06-hbm/a00", "failed", "64Gi", "64Gi"), # another stage does not count
("01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
("02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
for path, job_state, memory, expected in cases:
d = Path(tmp) / "rollouts" / path
d.mkdir(parents=True)
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
"salty_memory": memory}))
assert salty_remote._memory(policy, stage, ctx) == expected, path
def usage_log_check():