步骤到了阶段时限被看门狗结束时来不及删 cpu 名额的占用记录,_try_slots 见记录就跳过, 只有调度程序重启才清;10-12 晚到 10-13 上午累计锁死 26/32 个名额。现在占用者已死、 且该步骤已记 DONE、进程清理已确认的名额可以直接接手(板卡名额不变)。 同时远端容器不随本地步骤停,7 个容器空跑最久 7.5 小时;提交时用 timeout 把容器 运行时长限制在该阶段的执行时限内(镜像自带 /usr/bin/timeout,已实测)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
183 lines
9.9 KiB
Python
183 lines
9.9 KiB
Python
"""Round trip: controller bundle → container unpack/collect → controller install. Run: PYTHONPATH=. python3 scripts/test_salty_remote.py"""
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import tarfile
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
|
||
from harness import salty_remote, salty_remote_worker as worker
|
||
|
||
|
||
def main():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp = Path(tmp)
|
||
job_dir = tmp / "runs/r1/j1"
|
||
rd = job_dir / "rollouts/00"
|
||
(rd / "workspace").mkdir(parents=True)
|
||
(job_dir / "job.json").write_text(json.dumps({"job_id": "j1", "run_id": "r1"}))
|
||
(job_dir / "data").symlink_to(tmp / "data")
|
||
(rd / "workspace/compile.yaml").write_text("calibration_parameters: {}\n")
|
||
ctx = SimpleNamespace(run_id="r1", job={"job_id": "j1"}, job_dir=job_dir, stage_root=rd / "stages",
|
||
artifacts=rd / "workspace/artifacts", rollout=0, attempt=0, lane="agent",
|
||
agent_config=rd / "workspace/compile.yaml", extra={"slot": 1, "board": {"name": "b"}})
|
||
stage = SimpleNamespace(name="05-compile")
|
||
bundle = tmp / "bundle.tar"
|
||
salty_remote._work_dir = lambda stage, ctx: None # the real work_root (/home/ming01.wang) is container-only
|
||
salty_remote._bundle(bundle, stage, ctx)
|
||
names = tarfile.open(bundle).getnames()
|
||
assert "request.json" in names and "run-config.yaml" in names
|
||
assert any(n.startswith("code/harness/") for n in names) and not any("__pycache__" in n for n in names)
|
||
|
||
worker.BENCH = tmp / "container-bench"
|
||
(job_dir / "data").unlink() # the worker recreates the link from the request
|
||
request = worker.unpack(bundle)
|
||
assert (worker.BENCH / "harness/salty_remote_worker.py").is_file()
|
||
assert (job_dir / "data").is_symlink() and request["extra"]["board"] == {"name": "b"}
|
||
assert request["agent_config"] == str(rd / "workspace/compile.yaml")
|
||
|
||
sd = rd / "stages/05-compile/a00"
|
||
sd.mkdir(parents=True)
|
||
(sd / "stdout.log").write_text("compiled\n")
|
||
(rd / "stages/05-compile/meta.json").write_text('{"state": "OK"}')
|
||
work = tmp / "work/r1/j1/r00/05-compile-work"
|
||
(work / "out").mkdir(parents=True)
|
||
(work / "out/model.hbm").write_bytes(b"hbm")
|
||
(work / "out/model.bc").write_bytes(b"bc") # only the HBM is returned from the work dir
|
||
request["work_dir"] = str(work)
|
||
art = rd / "workspace/artifacts"; art.mkdir(parents=True)
|
||
(art / "old.hbm").write_bytes(b"old")
|
||
before = worker.snapshot(art)
|
||
(art / "j1.hbm").write_bytes(b"new") # written during the stage (mtime may be old)
|
||
os.utime(art / "j1.hbm", (0, 0))
|
||
result = worker.collect(request, tmp / "result", before)
|
||
name = lambda p: str(p).lstrip("/")
|
||
assert set(result["outputs"]) == {name(sd / "stdout.log"), name(rd / "stages/05-compile/meta.json"),
|
||
name(work / "out/model.hbm"), name(art / "j1.hbm")}
|
||
|
||
(sd / "stdout.log").unlink()
|
||
(work / "out/model.hbm").unlink()
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
|
||
assert (work / "out/model.hbm").read_bytes() == b"hbm"
|
||
try:
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir]) # work dir not allowed
|
||
except ValueError:
|
||
pass
|
||
else:
|
||
raise AssertionError("output outside the allowed roots must be rejected")
|
||
assert (sd / "stdout.log").read_text() == "compiled\n"
|
||
result["outputs"][name(rd / "stages/05-compile/meta.json")]["sha256"] = "0" * 64
|
||
try:
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
|
||
except ValueError:
|
||
pass
|
||
else:
|
||
raise AssertionError("tampered output must be rejected")
|
||
poll_retry_check()
|
||
failure_meta_check()
|
||
usage_log_check()
|
||
memory_escalation_check()
|
||
print("ok")
|
||
|
||
|
||
def memory_escalation_check():
|
||
"""A failed (not stopped) job of the same model (any run/march/rollout) doubles the memory it had; cap 128Gi."""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/flow--m--nash-m--ptq--int8")
|
||
stage = SimpleNamespace(name="05-compile")
|
||
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
|
||
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
|
||
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
|
||
cases = (("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", None, "32Gi"), # no memory recorded
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
|
||
("r1/flow--m2--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", "64Gi", "32Gi"), # other model
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/06-hbm/a00", "failed", "64Gi", "32Gi"), # other stage
|
||
("r2/flow--m--nash-p--ptq--fp16int8/rollouts/01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # other run/march
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
|
||
for path, job_state, memory, expected in cases:
|
||
d = Path(tmp) / path
|
||
d.mkdir(parents=True)
|
||
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
|
||
"salty_memory": memory}))
|
||
assert salty_remote._memory(policy, stage, ctx) == expected, path
|
||
|
||
|
||
def usage_log_check():
|
||
import threading, time
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
log = Path(tmp) / "usage.log"
|
||
threading.Thread(target=worker.usage_log, args=(log, 3600), daemon=True).start() # one tick, then sleeps past the test
|
||
time.sleep(0.5)
|
||
assert "mem=" in log.read_text() and "tmp_free=" in log.read_text() and "top:" in log.read_text()
|
||
|
||
|
||
def failure_meta_check():
|
||
"""A container that never wrote meta: the controller's meta must carry the operation identity
|
||
(stageworker compares it with the receipt) and mirror the stage-level copy."""
|
||
from harness.stages.base import StageOutcome
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
ident = {"operation_id": "05-compile-x", "payload_sha256": "p", "execution_id": "e", "epoch": 1}
|
||
ctx = SimpleNamespace(stage_root=root, attempt=0, rollout=0, extra={"operation_identity": ident})
|
||
stage = SimpleNamespace(name="05-compile", stage_dir=lambda c: (root / "05-compile/a00").mkdir(parents=True) or root / "05-compile/a00")
|
||
real = salty_remote._run
|
||
salty_remote._run = lambda *a, **k: StageOutcome("FAIL", 1, "REMOTE_WORKER_FAILED", 1.0, {})
|
||
try:
|
||
salty_remote.run(stage, ctx)
|
||
finally:
|
||
salty_remote._run = real
|
||
meta = json.loads((root / "05-compile/a00/meta.json").read_text())
|
||
assert {k: meta[k] for k in ident} == ident and meta["error_class"] == "REMOTE_WORKER_FAILED"
|
||
assert (root / "05-compile/meta.json").read_text() == (root / "05-compile/a00/meta.json").read_text()
|
||
|
||
|
||
def poll_retry_check():
|
||
"""Transient CLI failures are retried; a lost `run` reply is recovered by name, never resubmitted."""
|
||
import stat, tempfile
|
||
from harness.salty_client import TransportError
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp = Path(tmp)
|
||
fake = tmp / "salty"
|
||
fake.write_text(f"""#!/bin/sh
|
||
echo "$@" >> {tmp}/calls
|
||
n=$(grep -c . {tmp}/calls)
|
||
case "$1" in
|
||
describe)
|
||
[ -f {tmp}/created ] && {{ echo "ID: salty-00000000-0000-0000-0000-000000000001"; echo "状态: succeeded"; exit 0; }}
|
||
echo "错误:server 返回错误 not_found(HTTP 404)" >&2; exit 1;;
|
||
run) touch {tmp}/created; echo "网络错误" >&2; exit 1;; # created, but the reply is lost
|
||
storage) [ "$n" -le 2 ] && {{ echo "网络错误" >&2; exit 1; }}; exit 0;; # first two uploads fail
|
||
esac
|
||
""")
|
||
fake.chmod(fake.stat().st_mode | stat.S_IEXEC)
|
||
clock = iter(range(0, 10**6, 5)).__next__
|
||
cli = salty_remote._Cli(str(fake), 1800, lambda s: None, clock, lambda m: None)
|
||
adapter = tmp / "a"; adapter.mkdir()
|
||
for f in ("bundle.tar", "worker.py"):
|
||
(adapter / f).write_text("x")
|
||
state = {"name": "bench-stage-t", "remote": "personal:/bench-stage-t"}
|
||
policy = {"image": "r.example/i@sha256:" + "0" * 64}
|
||
from unittest.mock import patch
|
||
phase = SimpleNamespace(policy=lambda name: {"execution_deadline_s": 3600.0} if name == "01-fetch" else None)
|
||
with patch.object(salty_remote.execution, "current", return_value=phase):
|
||
job = salty_remote._submit(cli, state, policy, type("S", (), {"name": "01-fetch"}), adapter, "8Gi")
|
||
assert job == "salty-00000000-0000-0000-0000-000000000001"
|
||
calls = (tmp / "calls").read_text().splitlines()
|
||
assert sum(c.startswith("run") for c in calls) == 1, calls # never a second submit
|
||
run, = [c for c in calls if c.startswith("run")]
|
||
assert "-e OMP_NUM_THREADS=4 -e MKL_NUM_THREADS=4 -e OPENBLAS_NUM_THREADS=4 --cpu 4" in run, run # thread pools sized to the quota
|
||
assert "&& exec timeout 3600 /home/" in run, run # the container stops with the local phase deadline
|
||
assert salty_remote._wait(cli, job, {}, 0, clock, lambda s: None) == "succeeded"
|
||
down = salty_remote._Cli("/nonexistent/salty-down", 60, lambda s: None, iter(range(0, 10**6, 30)).__next__, lambda m: None)
|
||
try:
|
||
down.call("describe", job)
|
||
except (TransportError, OSError):
|
||
pass
|
||
else:
|
||
raise AssertionError("a lasting outage must fail")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|