feat: salty_remote 按题记内存需求:同一 job 任何轨迹里该阶段被 OOM 杀过,就从当时内存的两倍起步
内存需要是模型的属性;之前按单条轨迹计数,补号重跑的轨迹又从 16Gi 起,重复被杀。 meta 里记 salty_memory。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
45cff52749
commit
c8a93c9605
@ -156,15 +156,16 @@ class _Cli:
|
||||
|
||||
|
||||
def _memory(policy, stage, ctx) -> str:
|
||||
"""Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. Each
|
||||
earlier attempt of this stage whose job failed without a result doubles the request (cap 128Gi, near the old compile server), so
|
||||
only the models that need it hold more of the shared queue."""
|
||||
base = (policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi"))
|
||||
died = 0
|
||||
for meta in (ctx.stage_root / stage.name).glob("a*/meta.json"):
|
||||
"""Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. The need is the
|
||||
model's, so a failed (not stopped) job of this stage in any rollout of this job sets the floor: twice the
|
||||
memory it had, cap 128Gi (near the old compile server). Only the models that need it hold more of the queue."""
|
||||
base = int((policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")).removesuffix("Gi"))
|
||||
gi = base
|
||||
for meta in ctx.job_dir.glob(f"rollouts/*/stages/{stage.name}/a*/meta.json"):
|
||||
m = json.loads(meta.read_text())
|
||||
died += m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed"
|
||||
return f"{min(int(base.removesuffix('Gi')) * 2 ** died, 128)}Gi"
|
||||
if m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed":
|
||||
gi = max(gi, 2 * int(str(m.get("salty_memory") or f"{base}Gi").removesuffix("Gi")))
|
||||
return f"{min(gi, 128)}Gi"
|
||||
|
||||
|
||||
def _submit(cli, state, policy, stage, adapter, memory) -> str:
|
||||
@ -259,10 +260,11 @@ def _run(stage, ctx, *, sleep, clock):
|
||||
state = {"name": name, "remote": "personal:/" + name, "job_id": None}
|
||||
atomic_json(state_file, state)
|
||||
if not state["job_id"]:
|
||||
state["job_id"] = _submit(cli, state, policy, stage, adapter, _memory(policy, stage, ctx))
|
||||
state["memory"] = _memory(policy, stage, ctx)
|
||||
state["job_id"] = _submit(cli, state, policy, stage, adapter, state["memory"])
|
||||
atomic_json(state_file, state)
|
||||
job_state = _wait(cli, state["job_id"], policy, started, clock, sleep)
|
||||
meta = {"executor": "salty", "salty_job_id": state["job_id"]}
|
||||
meta = {"executor": "salty", "salty_job_id": state["job_id"], "salty_memory": state.get("memory")}
|
||||
if job_state != "succeeded":
|
||||
# The worker never reached a result: no stage outcome exists, board receipts decide quarantine.
|
||||
_remote_logs(policy, state["job_id"], sd / "stderr.log")
|
||||
|
||||
@ -81,18 +81,25 @@ def main():
|
||||
|
||||
|
||||
def memory_escalation_check():
|
||||
"""Only attempts whose job failed (not stopped) double the next request; capped at 128Gi."""
|
||||
"""A failed (not stopped) job in any rollout of the job doubles the memory it had; capped at 128Gi."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
ctx = SimpleNamespace(stage_root=Path(tmp))
|
||||
ctx = SimpleNamespace(job_dir=Path(tmp))
|
||||
stage = SimpleNamespace(name="05-compile")
|
||||
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
|
||||
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
|
||||
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
|
||||
for i, job_state in enumerate(("failed", "stopped", "failed", "failed", "failed")):
|
||||
(Path(tmp) / f"05-compile/a{i:02d}").mkdir(parents=True)
|
||||
(Path(tmp) / f"05-compile/a{i:02d}/meta.json").write_text(json.dumps(
|
||||
{"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state}))
|
||||
assert salty_remote._memory(policy, stage, ctx) == ("32Gi", "32Gi", "64Gi", "128Gi", "128Gi")[i], i
|
||||
cases = (("00/stages/05-compile/a00", "failed", None, "32Gi"), # old meta: no memory recorded
|
||||
("00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
|
||||
("01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # another rollout of the same job
|
||||
("01/stages/06-hbm/a00", "failed", "64Gi", "64Gi"), # another stage does not count
|
||||
("01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
|
||||
("02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
|
||||
for path, job_state, memory, expected in cases:
|
||||
d = Path(tmp) / "rollouts" / path
|
||||
d.mkdir(parents=True)
|
||||
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
|
||||
"salty_memory": memory}))
|
||||
assert salty_remote._memory(policy, stage, ctx) == expected, path
|
||||
|
||||
|
||||
def usage_log_check():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user