From c8a93c9605bfbd7ccac97cd0856d749da591b333 Mon Sep 17 00:00:00 2001 From: bench Date: Sun, 11 Oct 2026 10:36:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20salty=5Fremote=20=E6=8C=89=E9=A2=98?= =?UTF-8?q?=E8=AE=B0=E5=86=85=E5=AD=98=E9=9C=80=E6=B1=82=EF=BC=9A=E5=90=8C?= =?UTF-8?q?=E4=B8=80=20job=20=E4=BB=BB=E4=BD=95=E8=BD=A8=E8=BF=B9=E9=87=8C?= =?UTF-8?q?=E8=AF=A5=E9=98=B6=E6=AE=B5=E8=A2=AB=20OOM=20=E6=9D=80=E8=BF=87?= =?UTF-8?q?=EF=BC=8C=E5=B0=B1=E4=BB=8E=E5=BD=93=E6=97=B6=E5=86=85=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E4=B8=A4=E5=80=8D=E8=B5=B7=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 内存需要是模型的属性;之前按单条轨迹计数,补号重跑的轨迹又从 16Gi 起,重复被杀。 meta 里记 salty_memory。 Co-Authored-By: Claude Opus 5.5 --- harness/salty_remote.py | 22 ++++++++++++---------- scripts/test_salty_remote.py | 21 ++++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/harness/salty_remote.py b/harness/salty_remote.py index 3a73df9..ee9962f 100644 --- a/harness/salty_remote.py +++ b/harness/salty_remote.py @@ -156,15 +156,16 @@ class _Cli: def _memory(policy, stage, ctx) -> str: - """Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. Each - earlier attempt of this stage whose job failed without a result doubles the request (cap 128Gi, near the old compile server), so - only the models that need it hold more of the shared queue.""" - base = (policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")) - died = 0 - for meta in (ctx.stage_root / stage.name).glob("a*/meta.json"): + """Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. The need is the + model's, so a failed (not stopped) job of this stage in any rollout of this job sets the floor: twice the + memory it had, cap 128Gi (near the old compile server). Only the models that need it hold more of the queue.""" + base = int((policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")).removesuffix("Gi")) + gi = base + for meta in ctx.job_dir.glob(f"rollouts/*/stages/{stage.name}/a*/meta.json"): m = json.loads(meta.read_text()) - died += m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed" - return f"{min(int(base.removesuffix('Gi')) * 2 ** died, 128)}Gi" + if m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed": + gi = max(gi, 2 * int(str(m.get("salty_memory") or f"{base}Gi").removesuffix("Gi"))) + return f"{min(gi, 128)}Gi" def _submit(cli, state, policy, stage, adapter, memory) -> str: @@ -259,10 +260,11 @@ def _run(stage, ctx, *, sleep, clock): state = {"name": name, "remote": "personal:/" + name, "job_id": None} atomic_json(state_file, state) if not state["job_id"]: - state["job_id"] = _submit(cli, state, policy, stage, adapter, _memory(policy, stage, ctx)) + state["memory"] = _memory(policy, stage, ctx) + state["job_id"] = _submit(cli, state, policy, stage, adapter, state["memory"]) atomic_json(state_file, state) job_state = _wait(cli, state["job_id"], policy, started, clock, sleep) - meta = {"executor": "salty", "salty_job_id": state["job_id"]} + meta = {"executor": "salty", "salty_job_id": state["job_id"], "salty_memory": state.get("memory")} if job_state != "succeeded": # The worker never reached a result: no stage outcome exists, board receipts decide quarantine. _remote_logs(policy, state["job_id"], sd / "stderr.log") diff --git a/scripts/test_salty_remote.py b/scripts/test_salty_remote.py index 543acf7..e7d5679 100644 --- a/scripts/test_salty_remote.py +++ b/scripts/test_salty_remote.py @@ -81,18 +81,25 @@ def main(): def memory_escalation_check(): - """Only attempts whose job failed (not stopped) double the next request; capped at 128Gi.""" + """A failed (not stopped) job in any rollout of the job doubles the memory it had; capped at 128Gi.""" with tempfile.TemporaryDirectory() as tmp: - ctx = SimpleNamespace(stage_root=Path(tmp)) + ctx = SimpleNamespace(job_dir=Path(tmp)) stage = SimpleNamespace(name="05-compile") policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}} assert salty_remote._memory(policy, stage, ctx) == "16Gi" assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi" - for i, job_state in enumerate(("failed", "stopped", "failed", "failed", "failed")): - (Path(tmp) / f"05-compile/a{i:02d}").mkdir(parents=True) - (Path(tmp) / f"05-compile/a{i:02d}/meta.json").write_text(json.dumps( - {"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state})) - assert salty_remote._memory(policy, stage, ctx) == ("32Gi", "32Gi", "64Gi", "128Gi", "128Gi")[i], i + cases = (("00/stages/05-compile/a00", "failed", None, "32Gi"), # old meta: no memory recorded + ("00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"), + ("01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # another rollout of the same job + ("01/stages/06-hbm/a00", "failed", "64Gi", "64Gi"), # another stage does not count + ("01/stages/05-compile/a01", "failed", "64Gi", "128Gi"), + ("02/stages/05-compile/a00", "failed", "128Gi", "128Gi")) + for path, job_state, memory, expected in cases: + d = Path(tmp) / "rollouts" / path + d.mkdir(parents=True) + (d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state, + "salty_memory": memory})) + assert salty_remote._memory(policy, stage, ctx) == expected, path def usage_log_check():