diff --git a/harness/salty_remote.py b/harness/salty_remote.py index 3a73df9..ee9962f 100644 --- a/harness/salty_remote.py +++ b/harness/salty_remote.py @@ -156,15 +156,16 @@ class _Cli: def _memory(policy, stage, ctx) -> str: - """Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. Each - earlier attempt of this stage whose job failed without a result doubles the request (cap 128Gi, near the old compile server), so - only the models that need it hold more of the shared queue.""" - base = (policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")) - died = 0 - for meta in (ctx.stage_root / stage.name).glob("a*/meta.json"): + """Containers run with memory.oom.group=1: an OOM kills the whole job with no reason given. The need is the + model's, so a failed (not stopped) job of this stage in any rollout of this job sets the floor: twice the + memory it had, cap 128Gi (near the old compile server). Only the models that need it hold more of the queue.""" + base = int((policy.get("memory_by_stage") or {}).get(stage.name, policy.get("memory", "16Gi")).removesuffix("Gi")) + gi = base + for meta in ctx.job_dir.glob(f"rollouts/*/stages/{stage.name}/a*/meta.json"): m = json.loads(meta.read_text()) - died += m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed" - return f"{min(int(base.removesuffix('Gi')) * 2 ** died, 128)}Gi" + if m.get("error_class") == "REMOTE_WORKER_FAILED" and m.get("salty_job_state") == "failed": + gi = max(gi, 2 * int(str(m.get("salty_memory") or f"{base}Gi").removesuffix("Gi"))) + return f"{min(gi, 128)}Gi" def _submit(cli, state, policy, stage, adapter, memory) -> str: @@ -259,10 +260,11 @@ def _run(stage, ctx, *, sleep, clock): state = {"name": name, "remote": "personal:/" + name, "job_id": None} atomic_json(state_file, state) if not state["job_id"]: - state["job_id"] = _submit(cli, state, policy, stage, adapter, _memory(policy, stage, ctx)) + state["memory"] = _memory(policy, stage, ctx) + state["job_id"] = _submit(cli, state, policy, stage, adapter, state["memory"]) atomic_json(state_file, state) job_state = _wait(cli, state["job_id"], policy, started, clock, sleep) - meta = {"executor": "salty", "salty_job_id": state["job_id"]} + meta = {"executor": "salty", "salty_job_id": state["job_id"], "salty_memory": state.get("memory")} if job_state != "succeeded": # The worker never reached a result: no stage outcome exists, board receipts decide quarantine. _remote_logs(policy, state["job_id"], sd / "stderr.log") diff --git a/scripts/test_salty_remote.py b/scripts/test_salty_remote.py index 543acf7..e7d5679 100644 --- a/scripts/test_salty_remote.py +++ b/scripts/test_salty_remote.py @@ -81,18 +81,25 @@ def main(): def memory_escalation_check(): - """Only attempts whose job failed (not stopped) double the next request; capped at 128Gi.""" + """A failed (not stopped) job in any rollout of the job doubles the memory it had; capped at 128Gi.""" with tempfile.TemporaryDirectory() as tmp: - ctx = SimpleNamespace(stage_root=Path(tmp)) + ctx = SimpleNamespace(job_dir=Path(tmp)) stage = SimpleNamespace(name="05-compile") policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}} assert salty_remote._memory(policy, stage, ctx) == "16Gi" assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi" - for i, job_state in enumerate(("failed", "stopped", "failed", "failed", "failed")): - (Path(tmp) / f"05-compile/a{i:02d}").mkdir(parents=True) - (Path(tmp) / f"05-compile/a{i:02d}/meta.json").write_text(json.dumps( - {"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state})) - assert salty_remote._memory(policy, stage, ctx) == ("32Gi", "32Gi", "64Gi", "128Gi", "128Gi")[i], i + cases = (("00/stages/05-compile/a00", "failed", None, "32Gi"), # old meta: no memory recorded + ("00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"), + ("01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # another rollout of the same job + ("01/stages/06-hbm/a00", "failed", "64Gi", "64Gi"), # another stage does not count + ("01/stages/05-compile/a01", "failed", "64Gi", "128Gi"), + ("02/stages/05-compile/a00", "failed", "128Gi", "128Gi")) + for path, job_state, memory, expected in cases: + d = Path(tmp) / "rollouts" / path + d.mkdir(parents=True) + (d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state, + "salty_memory": memory})) + assert salty_remote._memory(policy, stage, ctx) == expected, path def usage_log_check():