data-gen-bpu/scripts/test_salty_remote.py
bench 7093c321f0 超时被杀的步骤归还名额、远端容器随阶段时限停止
步骤到了阶段时限被看门狗结束时来不及删 cpu 名额的占用记录,_try_slots 见记录就跳过,
只有调度程序重启才清;10-12 晚到 10-13 上午累计锁死 26/32 个名额。现在占用者已死、
且该步骤已记 DONE、进程清理已确认的名额可以直接接手(板卡名额不变)。
同时远端容器不随本地步骤停,7 个容器空跑最久 7.5 小时;提交时用 timeout 把容器
运行时长限制在该阶段的执行时限内(镜像自带 /usr/bin/timeout,已实测)。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 11:15:05 +08:00

183 lines
9.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Round trip: controller bundle → container unpack/collect → controller install. Run: PYTHONPATH=. python3 scripts/test_salty_remote.py"""
import json
import os
from pathlib import Path
import tarfile
import tempfile
from types import SimpleNamespace
from harness import salty_remote, salty_remote_worker as worker
def main():
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
job_dir = tmp / "runs/r1/j1"
rd = job_dir / "rollouts/00"
(rd / "workspace").mkdir(parents=True)
(job_dir / "job.json").write_text(json.dumps({"job_id": "j1", "run_id": "r1"}))
(job_dir / "data").symlink_to(tmp / "data")
(rd / "workspace/compile.yaml").write_text("calibration_parameters: {}\n")
ctx = SimpleNamespace(run_id="r1", job={"job_id": "j1"}, job_dir=job_dir, stage_root=rd / "stages",
artifacts=rd / "workspace/artifacts", rollout=0, attempt=0, lane="agent",
agent_config=rd / "workspace/compile.yaml", extra={"slot": 1, "board": {"name": "b"}})
stage = SimpleNamespace(name="05-compile")
bundle = tmp / "bundle.tar"
salty_remote._work_dir = lambda stage, ctx: None # the real work_root (/home/ming01.wang) is container-only
salty_remote._bundle(bundle, stage, ctx)
names = tarfile.open(bundle).getnames()
assert "request.json" in names and "run-config.yaml" in names
assert any(n.startswith("code/harness/") for n in names) and not any("__pycache__" in n for n in names)
worker.BENCH = tmp / "container-bench"
(job_dir / "data").unlink() # the worker recreates the link from the request
request = worker.unpack(bundle)
assert (worker.BENCH / "harness/salty_remote_worker.py").is_file()
assert (job_dir / "data").is_symlink() and request["extra"]["board"] == {"name": "b"}
assert request["agent_config"] == str(rd / "workspace/compile.yaml")
sd = rd / "stages/05-compile/a00"
sd.mkdir(parents=True)
(sd / "stdout.log").write_text("compiled\n")
(rd / "stages/05-compile/meta.json").write_text('{"state": "OK"}')
work = tmp / "work/r1/j1/r00/05-compile-work"
(work / "out").mkdir(parents=True)
(work / "out/model.hbm").write_bytes(b"hbm")
(work / "out/model.bc").write_bytes(b"bc") # only the HBM is returned from the work dir
request["work_dir"] = str(work)
art = rd / "workspace/artifacts"; art.mkdir(parents=True)
(art / "old.hbm").write_bytes(b"old")
before = worker.snapshot(art)
(art / "j1.hbm").write_bytes(b"new") # written during the stage (mtime may be old)
os.utime(art / "j1.hbm", (0, 0))
result = worker.collect(request, tmp / "result", before)
name = lambda p: str(p).lstrip("/")
assert set(result["outputs"]) == {name(sd / "stdout.log"), name(rd / "stages/05-compile/meta.json"),
name(work / "out/model.hbm"), name(art / "j1.hbm")}
(sd / "stdout.log").unlink()
(work / "out/model.hbm").unlink()
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
assert (work / "out/model.hbm").read_bytes() == b"hbm"
try:
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir]) # work dir not allowed
except ValueError:
pass
else:
raise AssertionError("output outside the allowed roots must be rejected")
assert (sd / "stdout.log").read_text() == "compiled\n"
result["outputs"][name(rd / "stages/05-compile/meta.json")]["sha256"] = "0" * 64
try:
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
except ValueError:
pass
else:
raise AssertionError("tampered output must be rejected")
poll_retry_check()
failure_meta_check()
usage_log_check()
memory_escalation_check()
print("ok")
def memory_escalation_check():
"""A failed (not stopped) job of the same model (any run/march/rollout) doubles the memory it had; cap 128Gi."""
with tempfile.TemporaryDirectory() as tmp:
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/flow--m--nash-m--ptq--int8")
stage = SimpleNamespace(name="05-compile")
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
cases = (("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", None, "32Gi"), # no memory recorded
("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
("r1/flow--m2--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", "64Gi", "32Gi"), # other model
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/06-hbm/a00", "failed", "64Gi", "32Gi"), # other stage
("r2/flow--m--nash-p--ptq--fp16int8/rollouts/01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # other run/march
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
("r1/flow--m--nash-m--ptq--int8/rollouts/02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
for path, job_state, memory, expected in cases:
d = Path(tmp) / path
d.mkdir(parents=True)
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
"salty_memory": memory}))
assert salty_remote._memory(policy, stage, ctx) == expected, path
def usage_log_check():
import threading, time
with tempfile.TemporaryDirectory() as tmp:
log = Path(tmp) / "usage.log"
threading.Thread(target=worker.usage_log, args=(log, 3600), daemon=True).start() # one tick, then sleeps past the test
time.sleep(0.5)
assert "mem=" in log.read_text() and "tmp_free=" in log.read_text() and "top:" in log.read_text()
def failure_meta_check():
"""A container that never wrote meta: the controller's meta must carry the operation identity
(stageworker compares it with the receipt) and mirror the stage-level copy."""
from harness.stages.base import StageOutcome
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
ident = {"operation_id": "05-compile-x", "payload_sha256": "p", "execution_id": "e", "epoch": 1}
ctx = SimpleNamespace(stage_root=root, attempt=0, rollout=0, extra={"operation_identity": ident})
stage = SimpleNamespace(name="05-compile", stage_dir=lambda c: (root / "05-compile/a00").mkdir(parents=True) or root / "05-compile/a00")
real = salty_remote._run
salty_remote._run = lambda *a, **k: StageOutcome("FAIL", 1, "REMOTE_WORKER_FAILED", 1.0, {})
try:
salty_remote.run(stage, ctx)
finally:
salty_remote._run = real
meta = json.loads((root / "05-compile/a00/meta.json").read_text())
assert {k: meta[k] for k in ident} == ident and meta["error_class"] == "REMOTE_WORKER_FAILED"
assert (root / "05-compile/meta.json").read_text() == (root / "05-compile/a00/meta.json").read_text()
def poll_retry_check():
"""Transient CLI failures are retried; a lost `run` reply is recovered by name, never resubmitted."""
import stat, tempfile
from harness.salty_client import TransportError
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
fake = tmp / "salty"
fake.write_text(f"""#!/bin/sh
echo "$@" >> {tmp}/calls
n=$(grep -c . {tmp}/calls)
case "$1" in
describe)
[ -f {tmp}/created ] && {{ echo "ID: salty-00000000-0000-0000-0000-000000000001"; echo "状态: succeeded"; exit 0; }}
echo "错误:server 返回错误 not_found(HTTP 404)" >&2; exit 1;;
run) touch {tmp}/created; echo "网络错误" >&2; exit 1;; # created, but the reply is lost
storage) [ "$n" -le 2 ] && {{ echo "网络错误" >&2; exit 1; }}; exit 0;; # first two uploads fail
esac
""")
fake.chmod(fake.stat().st_mode | stat.S_IEXEC)
clock = iter(range(0, 10**6, 5)).__next__
cli = salty_remote._Cli(str(fake), 1800, lambda s: None, clock, lambda m: None)
adapter = tmp / "a"; adapter.mkdir()
for f in ("bundle.tar", "worker.py"):
(adapter / f).write_text("x")
state = {"name": "bench-stage-t", "remote": "personal:/bench-stage-t"}
policy = {"image": "r.example/i@sha256:" + "0" * 64}
from unittest.mock import patch
phase = SimpleNamespace(policy=lambda name: {"execution_deadline_s": 3600.0} if name == "01-fetch" else None)
with patch.object(salty_remote.execution, "current", return_value=phase):
job = salty_remote._submit(cli, state, policy, type("S", (), {"name": "01-fetch"}), adapter, "8Gi")
assert job == "salty-00000000-0000-0000-0000-000000000001"
calls = (tmp / "calls").read_text().splitlines()
assert sum(c.startswith("run") for c in calls) == 1, calls # never a second submit
run, = [c for c in calls if c.startswith("run")]
assert "-e OMP_NUM_THREADS=4 -e MKL_NUM_THREADS=4 -e OPENBLAS_NUM_THREADS=4 --cpu 4" in run, run # thread pools sized to the quota
assert "&& exec timeout 3600 /home/" in run, run # the container stops with the local phase deadline
assert salty_remote._wait(cli, job, {}, 0, clock, lambda s: None) == "succeeded"
down = salty_remote._Cli("/nonexistent/salty-down", 60, lambda s: None, iter(range(0, 10**6, 30)).__next__, lambda m: None)
try:
down.call("describe", job)
except (TransportError, OSError):
pass
else:
raise AssertionError("a lasting outage must fail")
if __name__ == "__main__":
main()