data-gen-bpu/scripts/test_salty_offload.py
bench 89c4801f53 Salty cleanup: remove the job before its storage dir
Jobs were kept until their dir delete succeeded; every storage delete
has failed server side since 10-19, so ~8000 finished jobs piled up and
from 10-20 17:00 /jobs grew past the CLI's response limit: salty ps and
describe by name failed and every Salty step ended REMOTE_UNAVAILABLE.
The job record is only an index (logs/describe are in stderr.log), so it
now goes first; the dir delete keeps its own retry.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 19:47:14 +08:00

229 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Archive (review gate, mirrored products, overlays, freeing) → container unpack. Run: PYTHONPATH=. python3 scripts/test_salty_offload.py"""
import importlib.util
import json
import os
from pathlib import Path
import shutil
import sqlite3
import tarfile
import tempfile
import time
from types import SimpleNamespace
from harness import salty_remote_worker as worker
spec = importlib.util.spec_from_file_location("salty_offload", Path(__file__).with_name("salty_offload.py"))
offload = importlib.util.module_from_spec(spec)
spec.loader.exec_module(offload)
TERMINAL = ("PASS", "FAIL", "UNVERIFIED")
class Cli:
"""personal:/ is a local directory."""
def __init__(self, root):
self.root = root
def local(self, remote):
return self.root / remote.removeprefix("personal:/")
def once(self, args):
assert args[:2] == ["storage", "ls"], args
d = self.local(args[2])
if not d.is_dir():
return 1, "", "not_found"
return 0, "名称\t类型\t大小\n" + "".join(f"{p.name}\t文件\t{p.stat().st_size}\n" for p in d.iterdir() if p.is_file()), ""
def call(self, *args, timeout=120):
if args[:2] == ("storage", "mkdir"):
self.local(args[3]).mkdir(parents=True, exist_ok=True)
else:
assert args[:2] == ("storage", "cp"), args
shutil.copyfile(args[2], self.local(args[3]))
return 0, "", ""
def main():
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
personal = tmp / "personal"
paths = SimpleNamespace(RUNS=tmp / "runs", DB_PATH=tmp / "jobs.sqlite", DATASETS=tmp / "datasets",
expand=Path, config=lambda: {"paths": {"work_root": str(tmp / "work")}})
scratch = tmp / "work/r1/j1/r00/05-compile-work"
scratch.mkdir(parents=True); (scratch / "model.bc").write_bytes(b"x")
cli = Cli(personal)
rd = paths.RUNS / "r1/j1/rollouts/00"
for rel, size in (("agent/events.jsonl", 2 << 20), ("artifacts/j1.hbm", 2 << 20), ("artifacts/j1.bc", 2 << 20),
("rollout.json", 10), ("stages/08-eval/a00/predictions.jsonl", 10),
("stages/06-hbm/a00/spot_in.npz", 2 << 20)):
(rd / rel).parent.mkdir(parents=True, exist_ok=True)
(rd / rel).write_bytes(os.urandom(size))
(rd / "execution.json").write_text('{"execution_state": "TERMINAL"}')
mirror = personal / "bench-root/runs/r1/j1/rollouts/00/artifacts"
mirror.mkdir(parents=True)
shutil.copyfile(rd / "artifacts/j1.hbm", mirror / "j1.hbm") # the container wrote it there
with sqlite3.connect(paths.DB_PATH) as con:
con.execute("CREATE TABLE rollouts (run_id, job_id, kk, state, finished_at)")
con.execute("INSERT INTO rollouts VALUES ('r1', 'j1', 0, 'PASS', ?)", (time.time() - 7200,))
archive = personal / "bench-root/archive/r1/j1"
def run():
return offload.archive_all(cli, paths, ["r1"], TERMINAL)
assert run() == 0 and not archive.exists() # PASS, export has not judged it yet
(paths.DATASETS / "r1-training").mkdir(parents=True)
(paths.DATASETS / "r1-training/disposition.jsonl").write_text(
json.dumps({"job_id": "j1", "rollout": 0, "reasons": ["sft:content-review-unreviewed"]}) + "\n")
assert run() == 0 and not archive.exists() # still waiting for its review
(rd / "content-reviews").mkdir()
(rd / "content-reviews/a.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
events = (rd / "agent/events.jsonl").read_bytes()
assert scratch.exists() # not archived yet: scratch untouched
assert run() == 1
assert not scratch.parent.exists() # archived: its compile scratch is gone
assert sorted(tarfile.open(archive / "00.tgz").getnames()) == [
f"r1/j1/rollouts/00/{n}" for n in ("agent/events.jsonl", "artifacts/j1.bc", "content-reviews/a.json",
"execution.json", "rollout.json", "stages/06-hbm/a00/spot_in.npz",
"stages/08-eval/a00/predictions.jsonl")] # mirrored HBM left out
assert not (rd / "agent/events.jsonl").exists() and not (rd / "artifacts/j1.hbm").exists()
assert (rd / "rollout.json").exists() and run() == 0 # freed and unchanged: nothing to do
info = json.loads((rd / "ARCHIVED.json").read_text())
(rd / "content-reviews/b.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
os.utime(rd / "content-reviews/b.json", (info["at"] + 10, info["at"] + 10))
assert run() == 1
overlay, = [p for p in archive.iterdir() if p.name != "00.tgz"]
assert overlay.name.startswith("00.u") and tarfile.open(overlay).getnames() == ["r1/j1/rollouts/00/content-reviews/b.json"]
assert json.loads((rd / "ARCHIVED.json").read_text())["overlays"][0]["archive"].endswith(overlay.name)
worker.PERSONAL, worker.BENCH = personal, tmp / "bench"
worker.subprocess = SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0), DEVNULL=None)
out = personal / "bench-root/runs/r1/j1/rollouts/00"
(out / "stages/06-hbm/a01").mkdir(parents=True)
(out / "stages/06-hbm/a01/stdout.log").write_text("an attempt the controller had abandoned")
(tmp / "root").mkdir() # holds bundle.tar in a real job
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
assert (out / "agent/events.jsonl").read_bytes() == events and (out / "content-reviews/b.json").exists()
assert (out / "artifacts/j1.hbm").exists() and json.loads((tmp / "root/result/result.json").read_text())["exit_codes"] == {"r1": 0}
local = worker.BENCH / "runs/r1/j1/rollouts/00" # what the exporter reads: real dirs, bulk linked
assert not (worker.BENCH / "runs").is_symlink() and not (local / "stages").is_symlink()
assert (local / "stages/08-eval/a00/predictions.jsonl").is_file() and not (local / "stages/08-eval/a00/predictions.jsonl").is_symlink()
assert (local / "agent/events.jsonl").is_symlink() and (local / "agent/events.jsonl").read_bytes() == events
assert (local / "artifacts/j1.hbm").is_symlink() and (local / "stages/06-hbm/a00/spot_in.npz").exists()
assert not (local / "stages/06-hbm/a01/stdout.log").exists() # written to personal by a container only
(out / "rollout.json").write_text("changed in personal")
(archive / "01.tgz").write_bytes((archive / "00.tgz").read_bytes()[:200]) # caught mid-upload
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
assert (out / "rollout.json").read_text() == "changed in personal" # unpacked archives are not unpacked again
assert not list((out.parent / "01").glob(".unpacked-*")) # the partial one is retried next time
reap_check()
cleanup_check()
print("ok")
def reap_check():
"""Ended attempt: its Salty job is stopped, then removed once it has ended (failed too, 10-15); live jobs are left alone."""
from harness import salty_remote as sr
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
def make_cli(ps, calls):
remote = None
def once(args, **kw):
nonlocal remote
if args == ["ps"]:
return 0, ps, ""
calls.append(tuple(args))
if args[:2] == ["storage", "rm"]:
remote = args[3]
return 0, f"删除任务已受理:{task}(状态 queued)\n", ""
if args[:2] == ["storage", "task"]:
return 0, f"ID: {task}\n操作: delete\n状态: succeeded\n源: {remote}\n递归: true\n", ""
return 0, "", ""
return SimpleNamespace(once=once, call=lambda *a, **k: calls.append(a) or (0, "", ""), log=print)
with tempfile.TemporaryDirectory() as tmp:
runs = Path(tmp) / "runs"
rows = (("live", "running", False), ("orphan", "pending", True), ("stopped", "stopped", True), ("failed", "failed", True),
("cleaned", "running", True)) # removed by its own run after ps listed it: no stop retries
for name, _, ended in rows:
a = runs / f"r1/{name}/rollouts/00/stages/05-compile/a00"
(a / "salty").mkdir(parents=True)
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/s-" + name,
**({"cleaned": True} if name == "cleaned" else {})}))
if ended:
(a / "meta.json").write_text("{}")
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-stage\t{s}\t0\n" for name, s, _ in rows)
calls = []
cli = make_cli(ps, calls)
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
assert sorted(calls) == [("rm", "j-failed"), ("rm", "j-stopped"), ("stop", "j-orphan"),
("storage", "rm", "-r", "personal:/s-failed"), ("storage", "rm", "-r", "personal:/s-stopped"),
("storage", "rm-task", task), ("storage", "rm-task", task),
("storage", "task", task), ("storage", "task", task)], calls
assert sum(c[:2] == ("storage", "task") for c in calls) == 2
# No longer listed by ps but its attempt ended: the dir may still be there, so it is cleaned too.
a = runs / "r1/unlisted/rollouts/00/stages/05-compile/a00"
(a / "salty").mkdir(parents=True); (a / "meta.json").write_text("{}")
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-unlisted", "remote": "personal:/s-unlisted"}))
calls = []
offload.reap_orphans(make_cli(ps, calls), SimpleNamespace(RUNS=runs), sr, ["r1"])
assert ("storage", "rm", "-r", "personal:/s-unlisted") in calls and ("rm", "j-unlisted") in calls, calls
# bench dev containers: once the ticket is DONE (e.g. the rollout was cancelled mid-command) stop or remove them.
with tempfile.TemporaryDirectory() as tmp:
runs = Path(tmp) / "runs"
rows = (("devlive", "running", False), ("devcut", "running", True), ("devdone", "succeeded", True))
for name, _, ended in rows:
q = runs / f"r1/j/rollouts/00/stages/_queue/terminal-{name}"
(q / "salty").mkdir(parents=True)
(q / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/d-" + name}))
(q / "status.json").write_text(json.dumps({"state": "DONE" if ended else "RUNNING"}))
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-dev\t{s}\t0\n" for name, s, _ in rows)
calls = []
cli = make_cli(ps, calls)
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
assert sorted(calls) == [("rm", "j-devdone"), ("stop", "j-devcut"), ("storage", "rm", "-r", "personal:/d-devdone"),
("storage", "rm-task", task), ("storage", "task", task)], calls
def cleanup_check():
"""Refused or failed deletes go again only from the background, after a backoff; an unanswered one never does."""
from harness import salty_remote as sr
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
def cli(*replies):
calls, todo = [], list(replies)
return calls, SimpleNamespace(once=lambda args, **kw: calls.append(args[:2]) or todo.pop(0), log=lambda m: None)
refused = (1, "", "错误:server 返回异常状态 405")
accepted = (0, f"删除任务已受理:{task}(状态 queued)\n", "")
def task_reply(status, error="-"):
return 0, f"ID: {task}\n操作: delete\n状态: {status}\n源: personal:/x\n递归: true\n错误: {error}\n", ""
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "remote.json"
state = {"remote": "personal:/x", "job_id": "j"}
calls, c = cli((0, "", ""), refused)
sr._cleanup(c, state, path) # result path: one try; the job goes first
sr._cleanup(c, state, path)
assert state["cleanup"]["status"] == "rejected" and calls == [["rm", "j"], ["storage", "rm"]], calls
sr._cleanup(c, state, path, retry_after=3600) # background, too soon: nothing sent
assert calls == [["rm", "j"], ["storage", "rm"]]
calls, c = cli(accepted, task_reply("failed", "internal:对象存储操作失败"))
sr._cleanup(c, state, path, retry_after=0) # due: submitted again, its task queried
assert state["cleanup"]["status"] == "failed" and state["cleanup"]["tries"] == 2 and not state.get("cleaned")
calls, c = cli(accepted, task_reply("failed", "not_found:源不存在"), (0, "", ""))
state["cleanup"]["at"] = 0
sr._cleanup(c, state, path, retry_after=1) # the earlier try did remove it
assert state["cleaned"] and calls == [["storage", "rm"], ["storage", "task"], ["storage", "rm-task"]], calls
assert json.loads(path.read_text())["cleaned"]
state = {"remote": "personal:/x", "job_id": "j"}
calls, c = cli((0, "", ""), (None, "", "timeout")) # no reply: unknown, never sent blindly again
sr._cleanup(c, state, path)
sr._cleanup(c, state, path, retry_after=0)
assert state["cleanup"]["status"] == "submission_unknown" and calls == [["rm", "j"], ["storage", "rm"]], calls
state = {"remote": "personal:/x", "job_id": "j"}
calls, c = cli((0, "", ""), accepted, task_reply("queued"), task_reply("succeeded"), (0, "", ""))
sr._cleanup(c, state, path) # result path leaves it queued
assert state["cleanup"]["status"] == "queued"
sr._cleanup(c, state, path, retry_after=sr.CLEANUP_RETRY_S) # background pass checks it again
assert state["cleaned"] and calls[-2:] == [["storage", "task"], ["storage", "rm-task"]], calls
if __name__ == "__main__":
main()