data-gen-bpu/scripts/test_salty_offload.py
bench ad51e86f5f 导出进程每轮停掉本地已结束的 Salty 作业,停好后清理
阶段时限只管超时;轨迹被取消、进程崩溃时,远端容器照样跑完,没人取结果。
10-13 晚有 12 个这样的作业占着约 64 核,同时我们自己的作业在排队。
现在每轮先查 salty ps:本地这一步已经写了 meta.json、远端还在 running/pending 的,
就停掉;已经 stopped 的,按 salty_remote 处理自己停掉的作业那样删除作业和临时目录。
failed 的照旧保留,供排查。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 22:24:47 +08:00

138 lines
7.8 KiB
Python

"""Archive (review gate, mirrored products, overlays, freeing) → container unpack. Run: PYTHONPATH=. python3 scripts/test_salty_offload.py"""
import importlib.util
import json
import os
from pathlib import Path
import shutil
import sqlite3
import tarfile
import tempfile
import time
from types import SimpleNamespace
from harness import salty_remote_worker as worker
spec = importlib.util.spec_from_file_location("salty_offload", Path(__file__).with_name("salty_offload.py"))
offload = importlib.util.module_from_spec(spec)
spec.loader.exec_module(offload)
TERMINAL = ("PASS", "FAIL", "UNVERIFIED")
class Cli:
"""personal:/ is a local directory."""
def __init__(self, root):
self.root = root
def local(self, remote):
return self.root / remote.removeprefix("personal:/")
def once(self, args):
assert args[:2] == ["storage", "ls"], args
d = self.local(args[2])
if not d.is_dir():
return 1, "", "not_found"
return 0, "名称\t类型\t大小\n" + "".join(f"{p.name}\t文件\t{p.stat().st_size}\n" for p in d.iterdir() if p.is_file()), ""
def call(self, *args, timeout=120):
if args[:2] == ("storage", "mkdir"):
self.local(args[3]).mkdir(parents=True, exist_ok=True)
else:
assert args[:2] == ("storage", "cp"), args
shutil.copyfile(args[2], self.local(args[3]))
return 0, "", ""
def main():
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
personal = tmp / "personal"
paths = SimpleNamespace(RUNS=tmp / "runs", DB_PATH=tmp / "jobs.sqlite", DATASETS=tmp / "datasets")
cli = Cli(personal)
rd = paths.RUNS / "r1/j1/rollouts/00"
for rel, size in (("agent/events.jsonl", 2 << 20), ("artifacts/j1.hbm", 2 << 20), ("artifacts/j1.bc", 2 << 20),
("rollout.json", 10), ("stages/08-eval/a00/predictions.jsonl", 10),
("stages/06-hbm/a00/spot_in.npz", 2 << 20)):
(rd / rel).parent.mkdir(parents=True, exist_ok=True)
(rd / rel).write_bytes(os.urandom(size))
(rd / "execution.json").write_text('{"execution_state": "TERMINAL"}')
mirror = personal / "bench-root/runs/r1/j1/rollouts/00/artifacts"
mirror.mkdir(parents=True)
shutil.copyfile(rd / "artifacts/j1.hbm", mirror / "j1.hbm") # the container wrote it there
with sqlite3.connect(paths.DB_PATH) as con:
con.execute("CREATE TABLE rollouts (run_id, job_id, kk, state, finished_at)")
con.execute("INSERT INTO rollouts VALUES ('r1', 'j1', 0, 'PASS', ?)", (time.time() - 7200,))
archive = personal / "bench-root/archive/r1/j1"
def run():
return offload.archive_all(cli, paths, ["r1"], TERMINAL)
assert run() == 0 and not archive.exists() # PASS, export has not judged it yet
(paths.DATASETS / "r1-training").mkdir(parents=True)
(paths.DATASETS / "r1-training/disposition.jsonl").write_text(
json.dumps({"job_id": "j1", "rollout": 0, "reasons": ["sft:content-review-unreviewed"]}) + "\n")
assert run() == 0 and not archive.exists() # still waiting for its review
(rd / "content-reviews").mkdir()
(rd / "content-reviews/a.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
events = (rd / "agent/events.jsonl").read_bytes()
assert run() == 1
assert sorted(tarfile.open(archive / "00.tgz").getnames()) == [
f"r1/j1/rollouts/00/{n}" for n in ("agent/events.jsonl", "artifacts/j1.bc", "content-reviews/a.json",
"execution.json", "rollout.json", "stages/06-hbm/a00/spot_in.npz",
"stages/08-eval/a00/predictions.jsonl")] # mirrored HBM left out
assert not (rd / "agent/events.jsonl").exists() and not (rd / "artifacts/j1.hbm").exists()
assert (rd / "rollout.json").exists() and run() == 0 # freed and unchanged: nothing to do
info = json.loads((rd / "ARCHIVED.json").read_text())
(rd / "content-reviews/b.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
os.utime(rd / "content-reviews/b.json", (info["at"] + 10, info["at"] + 10))
assert run() == 1
overlay, = [p for p in archive.iterdir() if p.name != "00.tgz"]
assert overlay.name.startswith("00.u") and tarfile.open(overlay).getnames() == ["r1/j1/rollouts/00/content-reviews/b.json"]
assert json.loads((rd / "ARCHIVED.json").read_text())["overlays"][0]["archive"].endswith(overlay.name)
worker.PERSONAL, worker.BENCH = personal, tmp / "bench"
worker.subprocess = SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0), DEVNULL=None)
out = personal / "bench-root/runs/r1/j1/rollouts/00"
(out / "stages/06-hbm/a01").mkdir(parents=True)
(out / "stages/06-hbm/a01/stdout.log").write_text("an attempt the controller had abandoned")
(tmp / "root").mkdir() # holds bundle.tar in a real job
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
assert (out / "agent/events.jsonl").read_bytes() == events and (out / "content-reviews/b.json").exists()
assert (out / "artifacts/j1.hbm").exists() and json.loads((tmp / "root/result/result.json").read_text())["exit_codes"] == {"r1": 0}
local = worker.BENCH / "runs/r1/j1/rollouts/00" # what the exporter reads: real dirs, bulk linked
assert not (worker.BENCH / "runs").is_symlink() and not (local / "stages").is_symlink()
assert (local / "stages/08-eval/a00/predictions.jsonl").is_file() and not (local / "stages/08-eval/a00/predictions.jsonl").is_symlink()
assert (local / "agent/events.jsonl").is_symlink() and (local / "agent/events.jsonl").read_bytes() == events
assert (local / "artifacts/j1.hbm").is_symlink() and (local / "stages/06-hbm/a00/spot_in.npz").exists()
assert not (local / "stages/06-hbm/a01/stdout.log").exists() # written to personal by a container only
(out / "rollout.json").write_text("changed in personal")
(archive / "01.tgz").write_bytes((archive / "00.tgz").read_bytes()[:200]) # caught mid-upload
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
assert (out / "rollout.json").read_text() == "changed in personal" # unpacked archives are not unpacked again
assert not list((out.parent / "01").glob(".unpacked-*")) # the partial one is retried next time
reap_check()
print("ok")
def reap_check():
"""Ended attempt: its Salty job is stopped, then removed once stopped; live and failed jobs are left alone."""
from harness import salty_remote as sr
with tempfile.TemporaryDirectory() as tmp:
runs = Path(tmp) / "runs"
rows = (("live", "running", False), ("orphan", "pending", True), ("stopped", "stopped", True), ("failed", "failed", True))
for name, _, ended in rows:
a = runs / f"r1/{name}/rollouts/00/stages/05-compile/a00"
(a / "salty").mkdir(parents=True)
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/s-" + name}))
if ended:
(a / "meta.json").write_text("{}")
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-stage\t{s}\t0\n" for name, s, _ in rows)
calls = []
cli = SimpleNamespace(once=lambda args: (0, ps, ""), call=lambda *a, **k: calls.append(a) or (0, "", ""), log=print)
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
assert sorted(calls) == [("rm", "j-stopped"), ("stop", "j-orphan"), ("storage", "rm", "-r", "personal:/s-stopped")], calls
if __name__ == "__main__":
main()