The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates included. That did the model-specific half of the deployment for it. - guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site - the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract - SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is replaced by the facts that differ from the docs - e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass - agent/VERSION 24 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
354 lines
21 KiB
Python
354 lines
21 KiB
Python
"""Round trip: controller bundle → container unpack/collect → controller install. Run: PYTHONPATH=. python3 scripts/test_salty_remote.py"""
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import tarfile
|
||
import tempfile
|
||
from types import SimpleNamespace
|
||
|
||
from harness import salty_remote, salty_remote_worker as worker
|
||
|
||
|
||
def main():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp = Path(tmp)
|
||
job_dir = tmp / "runs/r1/j1"
|
||
rd = job_dir / "rollouts/00"
|
||
(rd / "workspace").mkdir(parents=True)
|
||
(job_dir / "job.json").write_text(json.dumps({"job_id": "j1", "run_id": "r1"}))
|
||
(job_dir / "data").symlink_to(tmp / "data")
|
||
(rd / "workspace/compile.yaml").write_text("calibration_parameters: {}\n")
|
||
ctx = SimpleNamespace(run_id="r1", job={"job_id": "j1"}, job_dir=job_dir, stage_root=rd / "stages",
|
||
artifacts=rd / "workspace/artifacts", rollout=0, attempt=0, lane="agent",
|
||
agent_config=rd / "workspace/compile.yaml", extra={"slot": 1, "board": {"name": "b"}})
|
||
stage = SimpleNamespace(name="05-compile")
|
||
bundle = tmp / "bundle.tar"
|
||
salty_remote._work_dir = lambda stage, ctx: None # the real work_root (/home/ming01.wang) is container-only
|
||
salty_remote._bundle(bundle, stage, ctx)
|
||
names = tarfile.open(bundle).getnames()
|
||
assert "request.json" in names and "run-config.yaml" in names
|
||
assert any(n.startswith("code/harness/") for n in names) and not any("__pycache__" in n for n in names)
|
||
|
||
worker.BENCH = tmp / "container-bench"
|
||
(job_dir / "data").unlink() # the worker recreates the link from the request
|
||
request = worker.unpack(bundle)
|
||
assert (worker.BENCH / "harness/salty_remote_worker.py").is_file()
|
||
assert (job_dir / "data").is_symlink() and request["extra"]["board"] == {"name": "b"}
|
||
assert request["agent_config"] == str(rd / "workspace/compile.yaml")
|
||
# A checkout outside the old bench root shares BENCH/runs' rollout storage (bench dev and stages agree).
|
||
from harness import paths
|
||
assert request["bench_root"] == str(paths.BENCH_ROOT)
|
||
worker.PERSONAL = tmp / "personal"
|
||
(worker.PERSONAL / "bench-root/runs").mkdir(parents=True)
|
||
worker.checkout_runs({"bench_root": str(tmp / "staging")})
|
||
assert (tmp / "staging/runs").resolve() == (worker.PERSONAL / "bench-root/runs").resolve()
|
||
worker.checkout_runs({"bench_root": str(worker.BENCH)}) # the old root itself: nothing to add
|
||
assert not (worker.BENCH / "runs").exists()
|
||
|
||
sd = rd / "stages/05-compile/a00"
|
||
sd.mkdir(parents=True)
|
||
(sd / "stdout.log").write_text("compiled\n")
|
||
(rd / "stages/05-compile/meta.json").write_text('{"state": "OK"}')
|
||
work = tmp / "work/r1/j1/r00/05-compile-work"
|
||
(work / "out").mkdir(parents=True)
|
||
(work / "out/model.hbm").write_bytes(b"hbm")
|
||
(work / "out/model.bc").write_bytes(b"bc") # only the HBM is returned from the work dir
|
||
request["work_dir"] = str(work)
|
||
art = rd / "workspace/artifacts"; art.mkdir(parents=True)
|
||
(art / "old.hbm").write_bytes(b"old")
|
||
before = worker.snapshot(art)
|
||
(art / "j1.hbm").write_bytes(b"new") # written during the stage (mtime may be old)
|
||
os.utime(art / "j1.hbm", (0, 0))
|
||
result = worker.collect(request, tmp / "result", before)
|
||
name = lambda p: str(p).lstrip("/")
|
||
assert set(result["outputs"]) == {name(sd / "stdout.log"), name(rd / "stages/05-compile/meta.json"),
|
||
name(work / "out/model.hbm"), name(art / "j1.hbm")}
|
||
# 06 hands back its HBM/BC even when an earlier, failed attempt left them there unchanged (10-20).
|
||
(art / "j1.bc").write_bytes(b"bc")
|
||
r06 = worker.collect({**request, "stage": "06-hbm"}, tmp / "result06", worker.snapshot(art))
|
||
assert {name(art / "j1.hbm"), name(art / "j1.bc")} <= set(r06["outputs"]), r06["outputs"]
|
||
assert worker.collect({**request, "stage": "07-board"}, tmp / "result07", worker.snapshot(art))["outputs"].keys().isdisjoint(
|
||
{name(art / "j1.hbm"), name(art / "j1.bc")}) # other stages still return only what they changed
|
||
|
||
(sd / "stdout.log").unlink()
|
||
(work / "out/model.hbm").unlink()
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
|
||
assert (work / "out/model.hbm").read_bytes() == b"hbm"
|
||
try:
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir]) # work dir not allowed
|
||
except ValueError:
|
||
pass
|
||
else:
|
||
raise AssertionError("output outside the allowed roots must be rejected")
|
||
assert (sd / "stdout.log").read_text() == "compiled\n"
|
||
result["outputs"][name(rd / "stages/05-compile/meta.json")]["sha256"] = "0" * 64
|
||
try:
|
||
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
|
||
except ValueError:
|
||
pass
|
||
else:
|
||
raise AssertionError("tampered output must be rejected")
|
||
poll_retry_check()
|
||
failure_meta_check()
|
||
usage_log_check()
|
||
memory_escalation_check()
|
||
dev_memory_check()
|
||
dev_bundle_check()
|
||
partial_describe_check()
|
||
async_cleanup_check()
|
||
cleanup_result_check()
|
||
print("ok")
|
||
|
||
|
||
def partial_describe_check():
|
||
"""Only the known partial-diagnostics error with complete, matching basics bypasses retries."""
|
||
from unittest.mock import patch
|
||
from harness.salty_client import TransportError
|
||
job = "salty-bab524f1-e95b-40fc-b64b-4371efdad1f6"
|
||
out = f"ID: {job}\n名称: probe\n状态: succeeded\n实时诊断状态: partial\n"
|
||
err = "错误:实时诊断不完整:status=partial,3 项缺失或截断已列在输出中(基础信息已输出;缺失部分不代表任务失败)\n"
|
||
cli = salty_remote._Cli("unused", 0, lambda _: None, lambda: 1, lambda _: None)
|
||
with patch.object(cli, "once", return_value=(1, out, err)) as once:
|
||
assert cli.call("describe", job)[0] == cli.call("describe", "probe")[0] == 0
|
||
assert once.call_count == 2
|
||
for ref, body, error in (("other", out, err), (job, out, "权限不足"),
|
||
(job, out + "状态: failed\n", err), (job, out.replace(job, "partial-id"), err)):
|
||
with patch.object(cli, "once", return_value=(1, body, error)):
|
||
try:
|
||
cli.call("describe", ref)
|
||
except TransportError:
|
||
pass
|
||
else:
|
||
raise AssertionError("a mismatched identity or another CLI error must retain normal failure handling")
|
||
|
||
|
||
def async_cleanup_check():
|
||
"""Accepted is not complete; each pass queries the same persisted task, never resubmits uncertainty."""
|
||
from unittest.mock import patch
|
||
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path, calls = Path(tmp) / "remote.json", []
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
|
||
status = "queued"
|
||
def once(args, timeout):
|
||
assert timeout == 30
|
||
calls.append(args)
|
||
if args[:2] == ["storage", "rm"]:
|
||
assert json.loads(path.read_text())["cleanup"]["status"] == "submission_unknown"
|
||
return 0, f"删除任务已受理:{task}(状态 queued)\n", ""
|
||
if args[:2] == ["storage", "task"]:
|
||
assert args[2] == task and json.loads(path.read_text())["cleanup"]["task_id"] == task
|
||
return 0, f"ID: {task}\n操作: delete\n状态: {status}\n源: personal:/probe\n递归: true\n", ""
|
||
assert args == ["rm", "salty-probe"] or (args == ["storage", "rm-task", task] and status == "succeeded")
|
||
return 0, "", ""
|
||
cli = SimpleNamespace(once=once, log=lambda _: None)
|
||
salty_remote._cleanup(cli, state, path)
|
||
# The job goes first, whatever happens to the dir (10-20: ~8000 kept jobs broke `salty ps`).
|
||
assert calls[0] == ["rm", "salty-probe"] and state["job_removed"]
|
||
assert not state.get("cleaned") and state["cleanup"]["status"] == "queued"
|
||
state = json.loads(path.read_text()) # a later offload pass, including process restart
|
||
status = "running"
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert calls[-1] == ["storage", "task", task] and len(calls) == 4
|
||
status = "failed"
|
||
salty_remote._cleanup(cli, state, path)
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert len(calls) == 5 and not state.get("cleaned") and state["cleanup"]["task_id"] == task
|
||
# Cleaned only once the task's identity/source are confirmed.
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
|
||
status = "succeeded"
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert json.loads(path.read_text())["cleaned"] and calls[-1] == ["storage", "rm-task", task]
|
||
# A refused dir delete still removes the job.
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
|
||
with patch.object(cli, "once", side_effect=lambda a, timeout: (0, "", "") if a[0] == "rm" else (1, "", "错误:server 返回异常状态 405")):
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert state["job_removed"] and state["cleanup"]["status"] == "rejected" and not state.get("cleaned")
|
||
for reply in ((0, f"ID: {task}\n操作: delete\n状态: succeeded\n源: personal:/other\n递归: true\n", ""),
|
||
(1, "", "not_found HTTP 404")):
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True, "cleanup": {"task_id": task, "status": "queued"}}
|
||
with patch.object(cli, "once", return_value=reply) as once:
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert once.call_count == 1 and state["cleanup"]["status"] == "query_unknown" and not state.get("cleaned")
|
||
for reply in ((None, "", "timeout"), (0, "accepted without task ID", ""), (1, "", "not_found HTTP 404")):
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True}
|
||
with patch.object(cli, "once", return_value=reply) as once:
|
||
salty_remote._cleanup(cli, state, path)
|
||
salty_remote._cleanup(cli, json.loads(path.read_text()), path)
|
||
assert once.call_count == 1 and state["cleanup"]["status"] == "submission_unknown"
|
||
# A failed sentinel write must prevent POST; cleanup errors cannot turn an obtained result into failure.
|
||
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True}
|
||
with patch.object(salty_remote.execution, "atomic_json", side_effect=OSError("read only")), \
|
||
patch.object(cli, "log", side_effect=OSError("read only")), patch.object(cli, "once") as once:
|
||
salty_remote._cleanup(cli, state, path)
|
||
assert not once.called
|
||
|
||
|
||
def cleanup_result_check():
|
||
"""A storage deletion error cannot replace a successfully fetched stage result."""
|
||
from unittest.mock import patch
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
sd = root / "stage"; sd.mkdir()
|
||
ctx = SimpleNamespace(cfg={"execution": {"salty_remote": {"image": "x@sha256:0"}}},
|
||
attempt=0, job_dir=root, extra={})
|
||
stage = SimpleNamespace(name="01-fetch", stage_dir=lambda _: sd)
|
||
result = {"identity": {"attempt": 0, "stage": stage.name}, "extra": {},
|
||
"outcome": {"state": "OK", "exit_code": 0, "error_class": None, "duration_s": 1,
|
||
"meta": {}, "measurements": {}}}
|
||
def fetch(cli, remote, dest):
|
||
dest.write_text(json.dumps(result) if dest.name == "result.json" else "")
|
||
cli = SimpleNamespace(once=lambda _, **kw: (1, "", "HTTP 405"), log=lambda _: None)
|
||
with patch.object(salty_remote, "_Cli", return_value=cli), patch.object(salty_remote, "_bundle"), \
|
||
patch.object(salty_remote, "_memory", return_value="8Gi"), patch.object(salty_remote, "_submit", return_value="job"), \
|
||
patch.object(salty_remote, "_wait", return_value="succeeded"), patch.object(salty_remote, "_fetch", side_effect=fetch), \
|
||
patch.object(salty_remote, "_install"):
|
||
outcome = salty_remote._run(stage, ctx, sleep=lambda _: None, clock=lambda: 1)
|
||
assert outcome.state == "OK" and outcome.exit_code == 0
|
||
saved = json.loads((sd / "salty/remote.json").read_text())
|
||
assert not saved.get("cleaned") and saved["cleanup"]["status"] == "submission_unknown"
|
||
|
||
|
||
def memory_escalation_check():
|
||
"""A failed (not stopped) job of the same model (any run/march/rollout) doubles the memory it had; cap 128Gi."""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/flow--m--nash-m--ptq--int8")
|
||
stage = SimpleNamespace(name="05-compile")
|
||
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
|
||
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
|
||
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
|
||
cases = (("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", None, "32Gi"), # no memory recorded
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
|
||
("r1/flow--m2--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", "64Gi", "32Gi"), # other model
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/06-hbm/a00", "failed", "64Gi", "32Gi"), # other stage
|
||
("r2/flow--m--nash-p--ptq--fp16int8/rollouts/01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # other run/march
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
|
||
("r1/flow--m--nash-m--ptq--int8/rollouts/02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
|
||
for path, job_state, memory, expected in cases:
|
||
d = Path(tmp) / path
|
||
d.mkdir(parents=True)
|
||
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
|
||
"salty_memory": memory}))
|
||
assert salty_remote._memory(policy, stage, ctx) == expected, path
|
||
empty = Path(tmp) / "r1/flow--m--nash-m--ptq--int8/rollouts/03/stages/05-compile/a00"
|
||
empty.mkdir(parents=True)
|
||
(empty / "meta.json").write_text("") # cut by a power loss
|
||
assert salty_remote._memory(policy, stage, ctx) == "128Gi"
|
||
|
||
|
||
def dev_memory_check():
|
||
"""bench dev: a failed dev job of this rollout doubles what it had (the agent cannot raise it); the pi checkpoint's
|
||
temp file is never uploaded (it vanishes mid-bundle)."""
|
||
from harness import salty_dev
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/cls--m--nash-p--ptq--fp16int8")
|
||
rd = ctx.job_dir / "rollouts/02"
|
||
policy = {"memory": "8Gi", "memory_by_stage": {"terminal": "16Gi"}}
|
||
assert salty_dev.memory_gi(policy, ctx, rd) == 16
|
||
for name, state, expected in (("terminal-a", {"memory": "16Gi", "job_state": "stopped"}, 16),
|
||
("terminal-b", {"memory": "16Gi"}, 16), # succeeded: nothing recorded
|
||
("terminal-c", {"memory": "16Gi", "job_state": "failed"}, 32),
|
||
("terminal-d", {"memory": "128Gi", "job_state": "failed"}, 128)):
|
||
d = rd / "stages/_queue" / name / "salty"
|
||
d.mkdir(parents=True)
|
||
(d / "remote.json").write_text(json.dumps(state))
|
||
assert salty_dev.memory_gi(policy, ctx, rd) == expected, name
|
||
assert salty_dev.memory_gi(policy, ctx, ctx.job_dir / "rollouts/03") == 16 # other rollout
|
||
ws = Path(tmp) / "ws"
|
||
ws.mkdir()
|
||
for f in (".bench-session.json", ".bench-session.json.tmp", "a.py"):
|
||
(ws / f).write_text("x")
|
||
assert list(salty_dev.local_files(ws)) == ["a.py"]
|
||
|
||
|
||
def dev_bundle_check():
|
||
"""bench dev carries only the sandbox's own sitecustomize: no adapters, export tools, evaluator or gates."""
|
||
from harness import salty_dev
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
ws = Path(tmp) / "ws"
|
||
ws.mkdir()
|
||
(ws / "a.py").write_text("x")
|
||
salty_dev.bundle(Path(tmp) / "b.tar", ws, ["a.py"], {"dev": {}})
|
||
with tarfile.open(Path(tmp) / "b.tar") as tar:
|
||
names = sorted(tar.getnames())
|
||
assert names == ["code/harness/container_site/sitecustomize.py", "request.json", "ws/a.py"], names
|
||
|
||
|
||
def usage_log_check():
|
||
import threading, time
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
log = Path(tmp) / "usage.log"
|
||
threading.Thread(target=worker.usage_log, args=(log, 3600), daemon=True).start() # one tick, then sleeps past the test
|
||
time.sleep(0.5)
|
||
assert "mem=" in log.read_text() and "tmp_free=" in log.read_text() and "top:" in log.read_text()
|
||
|
||
|
||
def failure_meta_check():
|
||
"""A container that never wrote meta: the controller's meta must carry the operation identity
|
||
(stageworker compares it with the receipt) and mirror the stage-level copy."""
|
||
from harness.stages.base import StageOutcome
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
root = Path(tmp)
|
||
ident = {"operation_id": "05-compile-x", "payload_sha256": "p", "execution_id": "e", "epoch": 1}
|
||
ctx = SimpleNamespace(stage_root=root, attempt=0, rollout=0, extra={"operation_identity": ident})
|
||
stage = SimpleNamespace(name="05-compile", stage_dir=lambda c: (root / "05-compile/a00").mkdir(parents=True) or root / "05-compile/a00")
|
||
real = salty_remote._run
|
||
salty_remote._run = lambda *a, **k: StageOutcome("FAIL", 1, "REMOTE_WORKER_FAILED", 1.0, {})
|
||
try:
|
||
salty_remote.run(stage, ctx)
|
||
finally:
|
||
salty_remote._run = real
|
||
meta = json.loads((root / "05-compile/a00/meta.json").read_text())
|
||
assert {k: meta[k] for k in ident} == ident and meta["error_class"] == "REMOTE_WORKER_FAILED"
|
||
assert (root / "05-compile/meta.json").read_text() == (root / "05-compile/a00/meta.json").read_text()
|
||
|
||
|
||
def poll_retry_check():
|
||
"""Transient CLI failures are retried; a lost `run` reply is recovered by name, never resubmitted."""
|
||
import stat, tempfile
|
||
from harness.salty_client import TransportError
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp = Path(tmp)
|
||
fake = tmp / "salty"
|
||
fake.write_text(f"""#!/bin/sh
|
||
echo "$@" >> {tmp}/calls
|
||
n=$(grep -c . {tmp}/calls)
|
||
case "$1" in
|
||
describe)
|
||
[ -f {tmp}/created ] && {{ echo "ID: salty-00000000-0000-0000-0000-000000000001"; echo "状态: succeeded"; exit 0; }}
|
||
echo "错误:server 返回错误 not_found(HTTP 404)" >&2; exit 1;;
|
||
run) touch {tmp}/created; echo "网络错误" >&2; exit 1;; # created, but the reply is lost
|
||
storage) [ "$n" -le 2 ] && {{ echo "网络错误" >&2; exit 1; }}; exit 0;; # first two uploads fail
|
||
esac
|
||
""")
|
||
fake.chmod(fake.stat().st_mode | stat.S_IEXEC)
|
||
clock = iter(range(0, 10**6, 5)).__next__
|
||
cli = salty_remote._Cli(str(fake), 1800, lambda s: None, clock, lambda m: None)
|
||
adapter = tmp / "a"; adapter.mkdir()
|
||
for f in ("bundle.tar", "worker.py"):
|
||
(adapter / f).write_text("x")
|
||
state = {"name": "bench-stage-t", "remote": "personal:/bench-stage-t"}
|
||
policy = {"image": "r.example/i@sha256:" + "0" * 64}
|
||
from unittest.mock import patch
|
||
phase = SimpleNamespace(policy=lambda name: {"execution_deadline_s": 3600.0} if name == "01-fetch" else None)
|
||
with patch.object(salty_remote.execution, "current", return_value=phase):
|
||
job = salty_remote._submit(cli, state, policy, type("S", (), {"name": "01-fetch"}), adapter, "8Gi")
|
||
assert job == "salty-00000000-0000-0000-0000-000000000001"
|
||
calls = (tmp / "calls").read_text().splitlines()
|
||
assert sum(c.startswith("run") for c in calls) == 1, calls # never a second submit
|
||
run, = [c for c in calls if c.startswith("run")]
|
||
assert "-e OMP_NUM_THREADS=4 -e MKL_NUM_THREADS=4 -e OPENBLAS_NUM_THREADS=4 --cpu 4" in run, run # thread pools sized to the quota
|
||
assert "&& exec timeout 3600 /home/" in run, run # the container stops with the local phase deadline
|
||
assert salty_remote._wait(cli, job, {}, 0, clock, lambda s: None) == "succeeded"
|
||
down = salty_remote._Cli("/nonexistent/salty-down", 60, lambda s: None, iter(range(0, 10**6, 30)).__next__, lambda m: None)
|
||
try:
|
||
down.call("describe", job)
|
||
except (TransportError, OSError):
|
||
pass
|
||
else:
|
||
raise AssertionError("a lasting outage must fail")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|