data-gen-bpu/scripts/test_salty_remote.py
bench 8fb9349907 Agent no longer gets the platform's model code; each task states its I/O contract
The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every
bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates
included. That did the model-specific half of the deployment for it.

- guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site
- the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the
  graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract
- SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is
  replaced by the facts that differ from the docs
- e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass
- agent/VERSION 24

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 21:51:17 +08:00

354 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Round trip: controller bundle → container unpack/collect → controller install. Run: PYTHONPATH=. python3 scripts/test_salty_remote.py"""
import json
import os
from pathlib import Path
import tarfile
import tempfile
from types import SimpleNamespace
from harness import salty_remote, salty_remote_worker as worker
def main():
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
job_dir = tmp / "runs/r1/j1"
rd = job_dir / "rollouts/00"
(rd / "workspace").mkdir(parents=True)
(job_dir / "job.json").write_text(json.dumps({"job_id": "j1", "run_id": "r1"}))
(job_dir / "data").symlink_to(tmp / "data")
(rd / "workspace/compile.yaml").write_text("calibration_parameters: {}\n")
ctx = SimpleNamespace(run_id="r1", job={"job_id": "j1"}, job_dir=job_dir, stage_root=rd / "stages",
artifacts=rd / "workspace/artifacts", rollout=0, attempt=0, lane="agent",
agent_config=rd / "workspace/compile.yaml", extra={"slot": 1, "board": {"name": "b"}})
stage = SimpleNamespace(name="05-compile")
bundle = tmp / "bundle.tar"
salty_remote._work_dir = lambda stage, ctx: None # the real work_root (/home/ming01.wang) is container-only
salty_remote._bundle(bundle, stage, ctx)
names = tarfile.open(bundle).getnames()
assert "request.json" in names and "run-config.yaml" in names
assert any(n.startswith("code/harness/") for n in names) and not any("__pycache__" in n for n in names)
worker.BENCH = tmp / "container-bench"
(job_dir / "data").unlink() # the worker recreates the link from the request
request = worker.unpack(bundle)
assert (worker.BENCH / "harness/salty_remote_worker.py").is_file()
assert (job_dir / "data").is_symlink() and request["extra"]["board"] == {"name": "b"}
assert request["agent_config"] == str(rd / "workspace/compile.yaml")
# A checkout outside the old bench root shares BENCH/runs' rollout storage (bench dev and stages agree).
from harness import paths
assert request["bench_root"] == str(paths.BENCH_ROOT)
worker.PERSONAL = tmp / "personal"
(worker.PERSONAL / "bench-root/runs").mkdir(parents=True)
worker.checkout_runs({"bench_root": str(tmp / "staging")})
assert (tmp / "staging/runs").resolve() == (worker.PERSONAL / "bench-root/runs").resolve()
worker.checkout_runs({"bench_root": str(worker.BENCH)}) # the old root itself: nothing to add
assert not (worker.BENCH / "runs").exists()
sd = rd / "stages/05-compile/a00"
sd.mkdir(parents=True)
(sd / "stdout.log").write_text("compiled\n")
(rd / "stages/05-compile/meta.json").write_text('{"state": "OK"}')
work = tmp / "work/r1/j1/r00/05-compile-work"
(work / "out").mkdir(parents=True)
(work / "out/model.hbm").write_bytes(b"hbm")
(work / "out/model.bc").write_bytes(b"bc") # only the HBM is returned from the work dir
request["work_dir"] = str(work)
art = rd / "workspace/artifacts"; art.mkdir(parents=True)
(art / "old.hbm").write_bytes(b"old")
before = worker.snapshot(art)
(art / "j1.hbm").write_bytes(b"new") # written during the stage (mtime may be old)
os.utime(art / "j1.hbm", (0, 0))
result = worker.collect(request, tmp / "result", before)
name = lambda p: str(p).lstrip("/")
assert set(result["outputs"]) == {name(sd / "stdout.log"), name(rd / "stages/05-compile/meta.json"),
name(work / "out/model.hbm"), name(art / "j1.hbm")}
# 06 hands back its HBM/BC even when an earlier, failed attempt left them there unchanged (10-20).
(art / "j1.bc").write_bytes(b"bc")
r06 = worker.collect({**request, "stage": "06-hbm"}, tmp / "result06", worker.snapshot(art))
assert {name(art / "j1.hbm"), name(art / "j1.bc")} <= set(r06["outputs"]), r06["outputs"]
assert worker.collect({**request, "stage": "07-board"}, tmp / "result07", worker.snapshot(art))["outputs"].keys().isdisjoint(
{name(art / "j1.hbm"), name(art / "j1.bc")}) # other stages still return only what they changed
(sd / "stdout.log").unlink()
(work / "out/model.hbm").unlink()
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
assert (work / "out/model.hbm").read_bytes() == b"hbm"
try:
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir]) # work dir not allowed
except ValueError:
pass
else:
raise AssertionError("output outside the allowed roots must be rejected")
assert (sd / "stdout.log").read_text() == "compiled\n"
result["outputs"][name(rd / "stages/05-compile/meta.json")]["sha256"] = "0" * 64
try:
salty_remote._install(result, tmp / "result/outputs.tar", [job_dir, work])
except ValueError:
pass
else:
raise AssertionError("tampered output must be rejected")
poll_retry_check()
failure_meta_check()
usage_log_check()
memory_escalation_check()
dev_memory_check()
dev_bundle_check()
partial_describe_check()
async_cleanup_check()
cleanup_result_check()
print("ok")
def partial_describe_check():
"""Only the known partial-diagnostics error with complete, matching basics bypasses retries."""
from unittest.mock import patch
from harness.salty_client import TransportError
job = "salty-bab524f1-e95b-40fc-b64b-4371efdad1f6"
out = f"ID: {job}\n名称: probe\n状态: succeeded\n实时诊断状态: partial\n"
err = "错误:实时诊断不完整:status=partial,3 项缺失或截断已列在输出中(基础信息已输出;缺失部分不代表任务失败)\n"
cli = salty_remote._Cli("unused", 0, lambda _: None, lambda: 1, lambda _: None)
with patch.object(cli, "once", return_value=(1, out, err)) as once:
assert cli.call("describe", job)[0] == cli.call("describe", "probe")[0] == 0
assert once.call_count == 2
for ref, body, error in (("other", out, err), (job, out, "权限不足"),
(job, out + "状态: failed\n", err), (job, out.replace(job, "partial-id"), err)):
with patch.object(cli, "once", return_value=(1, body, error)):
try:
cli.call("describe", ref)
except TransportError:
pass
else:
raise AssertionError("a mismatched identity or another CLI error must retain normal failure handling")
def async_cleanup_check():
"""Accepted is not complete; each pass queries the same persisted task, never resubmits uncertainty."""
from unittest.mock import patch
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
with tempfile.TemporaryDirectory() as tmp:
path, calls = Path(tmp) / "remote.json", []
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
status = "queued"
def once(args, timeout):
assert timeout == 30
calls.append(args)
if args[:2] == ["storage", "rm"]:
assert json.loads(path.read_text())["cleanup"]["status"] == "submission_unknown"
return 0, f"删除任务已受理:{task}(状态 queued)\n", ""
if args[:2] == ["storage", "task"]:
assert args[2] == task and json.loads(path.read_text())["cleanup"]["task_id"] == task
return 0, f"ID: {task}\n操作: delete\n状态: {status}\n源: personal:/probe\n递归: true\n", ""
assert args == ["rm", "salty-probe"] or (args == ["storage", "rm-task", task] and status == "succeeded")
return 0, "", ""
cli = SimpleNamespace(once=once, log=lambda _: None)
salty_remote._cleanup(cli, state, path)
# The job goes first, whatever happens to the dir (10-20: ~8000 kept jobs broke `salty ps`).
assert calls[0] == ["rm", "salty-probe"] and state["job_removed"]
assert not state.get("cleaned") and state["cleanup"]["status"] == "queued"
state = json.loads(path.read_text()) # a later offload pass, including process restart
status = "running"
salty_remote._cleanup(cli, state, path)
assert calls[-1] == ["storage", "task", task] and len(calls) == 4
status = "failed"
salty_remote._cleanup(cli, state, path)
salty_remote._cleanup(cli, state, path)
assert len(calls) == 5 and not state.get("cleaned") and state["cleanup"]["task_id"] == task
# Cleaned only once the task's identity/source are confirmed.
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
status = "succeeded"
salty_remote._cleanup(cli, state, path)
assert json.loads(path.read_text())["cleaned"] and calls[-1] == ["storage", "rm-task", task]
# A refused dir delete still removes the job.
state = {"remote": "personal:/probe", "job_id": "salty-probe"}
with patch.object(cli, "once", side_effect=lambda a, timeout: (0, "", "") if a[0] == "rm" else (1, "", "错误:server 返回异常状态 405")):
salty_remote._cleanup(cli, state, path)
assert state["job_removed"] and state["cleanup"]["status"] == "rejected" and not state.get("cleaned")
for reply in ((0, f"ID: {task}\n操作: delete\n状态: succeeded\n源: personal:/other\n递归: true\n", ""),
(1, "", "not_found HTTP 404")):
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True, "cleanup": {"task_id": task, "status": "queued"}}
with patch.object(cli, "once", return_value=reply) as once:
salty_remote._cleanup(cli, state, path)
assert once.call_count == 1 and state["cleanup"]["status"] == "query_unknown" and not state.get("cleaned")
for reply in ((None, "", "timeout"), (0, "accepted without task ID", ""), (1, "", "not_found HTTP 404")):
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True}
with patch.object(cli, "once", return_value=reply) as once:
salty_remote._cleanup(cli, state, path)
salty_remote._cleanup(cli, json.loads(path.read_text()), path)
assert once.call_count == 1 and state["cleanup"]["status"] == "submission_unknown"
# A failed sentinel write must prevent POST; cleanup errors cannot turn an obtained result into failure.
state = {"remote": "personal:/probe", "job_id": "salty-probe", "job_removed": True}
with patch.object(salty_remote.execution, "atomic_json", side_effect=OSError("read only")), \
patch.object(cli, "log", side_effect=OSError("read only")), patch.object(cli, "once") as once:
salty_remote._cleanup(cli, state, path)
assert not once.called
def cleanup_result_check():
"""A storage deletion error cannot replace a successfully fetched stage result."""
from unittest.mock import patch
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sd = root / "stage"; sd.mkdir()
ctx = SimpleNamespace(cfg={"execution": {"salty_remote": {"image": "x@sha256:0"}}},
attempt=0, job_dir=root, extra={})
stage = SimpleNamespace(name="01-fetch", stage_dir=lambda _: sd)
result = {"identity": {"attempt": 0, "stage": stage.name}, "extra": {},
"outcome": {"state": "OK", "exit_code": 0, "error_class": None, "duration_s": 1,
"meta": {}, "measurements": {}}}
def fetch(cli, remote, dest):
dest.write_text(json.dumps(result) if dest.name == "result.json" else "")
cli = SimpleNamespace(once=lambda _, **kw: (1, "", "HTTP 405"), log=lambda _: None)
with patch.object(salty_remote, "_Cli", return_value=cli), patch.object(salty_remote, "_bundle"), \
patch.object(salty_remote, "_memory", return_value="8Gi"), patch.object(salty_remote, "_submit", return_value="job"), \
patch.object(salty_remote, "_wait", return_value="succeeded"), patch.object(salty_remote, "_fetch", side_effect=fetch), \
patch.object(salty_remote, "_install"):
outcome = salty_remote._run(stage, ctx, sleep=lambda _: None, clock=lambda: 1)
assert outcome.state == "OK" and outcome.exit_code == 0
saved = json.loads((sd / "salty/remote.json").read_text())
assert not saved.get("cleaned") and saved["cleanup"]["status"] == "submission_unknown"
def memory_escalation_check():
"""A failed (not stopped) job of the same model (any run/march/rollout) doubles the memory it had; cap 128Gi."""
with tempfile.TemporaryDirectory() as tmp:
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/flow--m--nash-m--ptq--int8")
stage = SimpleNamespace(name="05-compile")
policy = {"memory": "8Gi", "memory_by_stage": {"05-compile": "16Gi"}}
assert salty_remote._memory(policy, stage, ctx) == "16Gi"
assert salty_remote._memory(policy, SimpleNamespace(name="06-hbm"), ctx) == "8Gi"
cases = (("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", None, "32Gi"), # no memory recorded
("r1/flow--m--nash-m--ptq--int8/rollouts/00/stages/05-compile/a01", "stopped", "32Gi", "32Gi"),
("r1/flow--m2--nash-m--ptq--int8/rollouts/00/stages/05-compile/a00", "failed", "64Gi", "32Gi"), # other model
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/06-hbm/a00", "failed", "64Gi", "32Gi"), # other stage
("r2/flow--m--nash-p--ptq--fp16int8/rollouts/01/stages/05-compile/a00", "failed", "32Gi", "64Gi"), # other run/march
("r1/flow--m--nash-m--ptq--int8/rollouts/01/stages/05-compile/a01", "failed", "64Gi", "128Gi"),
("r1/flow--m--nash-m--ptq--int8/rollouts/02/stages/05-compile/a00", "failed", "128Gi", "128Gi"))
for path, job_state, memory, expected in cases:
d = Path(tmp) / path
d.mkdir(parents=True)
(d / "meta.json").write_text(json.dumps({"error_class": "REMOTE_WORKER_FAILED", "salty_job_state": job_state,
"salty_memory": memory}))
assert salty_remote._memory(policy, stage, ctx) == expected, path
empty = Path(tmp) / "r1/flow--m--nash-m--ptq--int8/rollouts/03/stages/05-compile/a00"
empty.mkdir(parents=True)
(empty / "meta.json").write_text("") # cut by a power loss
assert salty_remote._memory(policy, stage, ctx) == "128Gi"
def dev_memory_check():
"""bench dev: a failed dev job of this rollout doubles what it had (the agent cannot raise it); the pi checkpoint's
temp file is never uploaded (it vanishes mid-bundle)."""
from harness import salty_dev
with tempfile.TemporaryDirectory() as tmp:
ctx = SimpleNamespace(job_dir=Path(tmp) / "r1/cls--m--nash-p--ptq--fp16int8")
rd = ctx.job_dir / "rollouts/02"
policy = {"memory": "8Gi", "memory_by_stage": {"terminal": "16Gi"}}
assert salty_dev.memory_gi(policy, ctx, rd) == 16
for name, state, expected in (("terminal-a", {"memory": "16Gi", "job_state": "stopped"}, 16),
("terminal-b", {"memory": "16Gi"}, 16), # succeeded: nothing recorded
("terminal-c", {"memory": "16Gi", "job_state": "failed"}, 32),
("terminal-d", {"memory": "128Gi", "job_state": "failed"}, 128)):
d = rd / "stages/_queue" / name / "salty"
d.mkdir(parents=True)
(d / "remote.json").write_text(json.dumps(state))
assert salty_dev.memory_gi(policy, ctx, rd) == expected, name
assert salty_dev.memory_gi(policy, ctx, ctx.job_dir / "rollouts/03") == 16 # other rollout
ws = Path(tmp) / "ws"
ws.mkdir()
for f in (".bench-session.json", ".bench-session.json.tmp", "a.py"):
(ws / f).write_text("x")
assert list(salty_dev.local_files(ws)) == ["a.py"]
def dev_bundle_check():
"""bench dev carries only the sandbox's own sitecustomize: no adapters, export tools, evaluator or gates."""
from harness import salty_dev
with tempfile.TemporaryDirectory() as tmp:
ws = Path(tmp) / "ws"
ws.mkdir()
(ws / "a.py").write_text("x")
salty_dev.bundle(Path(tmp) / "b.tar", ws, ["a.py"], {"dev": {}})
with tarfile.open(Path(tmp) / "b.tar") as tar:
names = sorted(tar.getnames())
assert names == ["code/harness/container_site/sitecustomize.py", "request.json", "ws/a.py"], names
def usage_log_check():
import threading, time
with tempfile.TemporaryDirectory() as tmp:
log = Path(tmp) / "usage.log"
threading.Thread(target=worker.usage_log, args=(log, 3600), daemon=True).start() # one tick, then sleeps past the test
time.sleep(0.5)
assert "mem=" in log.read_text() and "tmp_free=" in log.read_text() and "top:" in log.read_text()
def failure_meta_check():
"""A container that never wrote meta: the controller's meta must carry the operation identity
(stageworker compares it with the receipt) and mirror the stage-level copy."""
from harness.stages.base import StageOutcome
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
ident = {"operation_id": "05-compile-x", "payload_sha256": "p", "execution_id": "e", "epoch": 1}
ctx = SimpleNamespace(stage_root=root, attempt=0, rollout=0, extra={"operation_identity": ident})
stage = SimpleNamespace(name="05-compile", stage_dir=lambda c: (root / "05-compile/a00").mkdir(parents=True) or root / "05-compile/a00")
real = salty_remote._run
salty_remote._run = lambda *a, **k: StageOutcome("FAIL", 1, "REMOTE_WORKER_FAILED", 1.0, {})
try:
salty_remote.run(stage, ctx)
finally:
salty_remote._run = real
meta = json.loads((root / "05-compile/a00/meta.json").read_text())
assert {k: meta[k] for k in ident} == ident and meta["error_class"] == "REMOTE_WORKER_FAILED"
assert (root / "05-compile/meta.json").read_text() == (root / "05-compile/a00/meta.json").read_text()
def poll_retry_check():
"""Transient CLI failures are retried; a lost `run` reply is recovered by name, never resubmitted."""
import stat, tempfile
from harness.salty_client import TransportError
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
fake = tmp / "salty"
fake.write_text(f"""#!/bin/sh
echo "$@" >> {tmp}/calls
n=$(grep -c . {tmp}/calls)
case "$1" in
describe)
[ -f {tmp}/created ] && {{ echo "ID: salty-00000000-0000-0000-0000-000000000001"; echo "状态: succeeded"; exit 0; }}
echo "错误:server 返回错误 not_found(HTTP 404)" >&2; exit 1;;
run) touch {tmp}/created; echo "网络错误" >&2; exit 1;; # created, but the reply is lost
storage) [ "$n" -le 2 ] && {{ echo "网络错误" >&2; exit 1; }}; exit 0;; # first two uploads fail
esac
""")
fake.chmod(fake.stat().st_mode | stat.S_IEXEC)
clock = iter(range(0, 10**6, 5)).__next__
cli = salty_remote._Cli(str(fake), 1800, lambda s: None, clock, lambda m: None)
adapter = tmp / "a"; adapter.mkdir()
for f in ("bundle.tar", "worker.py"):
(adapter / f).write_text("x")
state = {"name": "bench-stage-t", "remote": "personal:/bench-stage-t"}
policy = {"image": "r.example/i@sha256:" + "0" * 64}
from unittest.mock import patch
phase = SimpleNamespace(policy=lambda name: {"execution_deadline_s": 3600.0} if name == "01-fetch" else None)
with patch.object(salty_remote.execution, "current", return_value=phase):
job = salty_remote._submit(cli, state, policy, type("S", (), {"name": "01-fetch"}), adapter, "8Gi")
assert job == "salty-00000000-0000-0000-0000-000000000001"
calls = (tmp / "calls").read_text().splitlines()
assert sum(c.startswith("run") for c in calls) == 1, calls # never a second submit
run, = [c for c in calls if c.startswith("run")]
assert "-e OMP_NUM_THREADS=4 -e MKL_NUM_THREADS=4 -e OPENBLAS_NUM_THREADS=4 --cpu 4" in run, run # thread pools sized to the quota
assert "&& exec timeout 3600 /home/" in run, run # the container stops with the local phase deadline
assert salty_remote._wait(cli, job, {}, 0, clock, lambda s: None) == "succeeded"
down = salty_remote._Cli("/nonexistent/salty-down", 60, lambda s: None, iter(range(0, 10**6, 30)).__next__, lambda m: None)
try:
down.call("describe", job)
except (TransportError, OSError):
pass
else:
raise AssertionError("a lasting outage must fail")
if __name__ == "__main__":
main()