Jobs were kept until their dir delete succeeded; every storage delete has failed server side since 10-19, so ~8000 finished jobs piled up and from 10-20 17:00 /jobs grew past the CLI's response limit: salty ps and describe by name failed and every Salty step ended REMOTE_UNAVAILABLE. The job record is only an index (logs/describe are in stderr.log), so it now goes first; the dir delete keeps its own retry. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
229 lines
14 KiB
Python
229 lines
14 KiB
Python
"""Archive (review gate, mirrored products, overlays, freeing) → container unpack. Run: PYTHONPATH=. python3 scripts/test_salty_offload.py"""
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import shutil
|
||
import sqlite3
|
||
import tarfile
|
||
import tempfile
|
||
import time
|
||
from types import SimpleNamespace
|
||
|
||
from harness import salty_remote_worker as worker
|
||
|
||
spec = importlib.util.spec_from_file_location("salty_offload", Path(__file__).with_name("salty_offload.py"))
|
||
offload = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(offload)
|
||
TERMINAL = ("PASS", "FAIL", "UNVERIFIED")
|
||
|
||
|
||
class Cli:
|
||
"""personal:/ is a local directory."""
|
||
def __init__(self, root):
|
||
self.root = root
|
||
|
||
def local(self, remote):
|
||
return self.root / remote.removeprefix("personal:/")
|
||
|
||
def once(self, args):
|
||
assert args[:2] == ["storage", "ls"], args
|
||
d = self.local(args[2])
|
||
if not d.is_dir():
|
||
return 1, "", "not_found"
|
||
return 0, "名称\t类型\t大小\n" + "".join(f"{p.name}\t文件\t{p.stat().st_size}\n" for p in d.iterdir() if p.is_file()), ""
|
||
|
||
def call(self, *args, timeout=120):
|
||
if args[:2] == ("storage", "mkdir"):
|
||
self.local(args[3]).mkdir(parents=True, exist_ok=True)
|
||
else:
|
||
assert args[:2] == ("storage", "cp"), args
|
||
shutil.copyfile(args[2], self.local(args[3]))
|
||
return 0, "", ""
|
||
|
||
|
||
def main():
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
tmp = Path(tmp)
|
||
personal = tmp / "personal"
|
||
paths = SimpleNamespace(RUNS=tmp / "runs", DB_PATH=tmp / "jobs.sqlite", DATASETS=tmp / "datasets",
|
||
expand=Path, config=lambda: {"paths": {"work_root": str(tmp / "work")}})
|
||
scratch = tmp / "work/r1/j1/r00/05-compile-work"
|
||
scratch.mkdir(parents=True); (scratch / "model.bc").write_bytes(b"x")
|
||
cli = Cli(personal)
|
||
rd = paths.RUNS / "r1/j1/rollouts/00"
|
||
for rel, size in (("agent/events.jsonl", 2 << 20), ("artifacts/j1.hbm", 2 << 20), ("artifacts/j1.bc", 2 << 20),
|
||
("rollout.json", 10), ("stages/08-eval/a00/predictions.jsonl", 10),
|
||
("stages/06-hbm/a00/spot_in.npz", 2 << 20)):
|
||
(rd / rel).parent.mkdir(parents=True, exist_ok=True)
|
||
(rd / rel).write_bytes(os.urandom(size))
|
||
(rd / "execution.json").write_text('{"execution_state": "TERMINAL"}')
|
||
mirror = personal / "bench-root/runs/r1/j1/rollouts/00/artifacts"
|
||
mirror.mkdir(parents=True)
|
||
shutil.copyfile(rd / "artifacts/j1.hbm", mirror / "j1.hbm") # the container wrote it there
|
||
with sqlite3.connect(paths.DB_PATH) as con:
|
||
con.execute("CREATE TABLE rollouts (run_id, job_id, kk, state, finished_at)")
|
||
con.execute("INSERT INTO rollouts VALUES ('r1', 'j1', 0, 'PASS', ?)", (time.time() - 7200,))
|
||
archive = personal / "bench-root/archive/r1/j1"
|
||
|
||
def run():
|
||
return offload.archive_all(cli, paths, ["r1"], TERMINAL)
|
||
|
||
assert run() == 0 and not archive.exists() # PASS, export has not judged it yet
|
||
(paths.DATASETS / "r1-training").mkdir(parents=True)
|
||
(paths.DATASETS / "r1-training/disposition.jsonl").write_text(
|
||
json.dumps({"job_id": "j1", "rollout": 0, "reasons": ["sft:content-review-unreviewed"]}) + "\n")
|
||
assert run() == 0 and not archive.exists() # still waiting for its review
|
||
(rd / "content-reviews").mkdir()
|
||
(rd / "content-reviews/a.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
|
||
events = (rd / "agent/events.jsonl").read_bytes()
|
||
assert scratch.exists() # not archived yet: scratch untouched
|
||
assert run() == 1
|
||
assert not scratch.parent.exists() # archived: its compile scratch is gone
|
||
assert sorted(tarfile.open(archive / "00.tgz").getnames()) == [
|
||
f"r1/j1/rollouts/00/{n}" for n in ("agent/events.jsonl", "artifacts/j1.bc", "content-reviews/a.json",
|
||
"execution.json", "rollout.json", "stages/06-hbm/a00/spot_in.npz",
|
||
"stages/08-eval/a00/predictions.jsonl")] # mirrored HBM left out
|
||
assert not (rd / "agent/events.jsonl").exists() and not (rd / "artifacts/j1.hbm").exists()
|
||
assert (rd / "rollout.json").exists() and run() == 0 # freed and unchanged: nothing to do
|
||
|
||
info = json.loads((rd / "ARCHIVED.json").read_text())
|
||
(rd / "content-reviews/b.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
|
||
os.utime(rd / "content-reviews/b.json", (info["at"] + 10, info["at"] + 10))
|
||
assert run() == 1
|
||
overlay, = [p for p in archive.iterdir() if p.name != "00.tgz"]
|
||
assert overlay.name.startswith("00.u") and tarfile.open(overlay).getnames() == ["r1/j1/rollouts/00/content-reviews/b.json"]
|
||
assert json.loads((rd / "ARCHIVED.json").read_text())["overlays"][0]["archive"].endswith(overlay.name)
|
||
|
||
worker.PERSONAL, worker.BENCH = personal, tmp / "bench"
|
||
worker.subprocess = SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0), DEVNULL=None)
|
||
out = personal / "bench-root/runs/r1/j1/rollouts/00"
|
||
(out / "stages/06-hbm/a01").mkdir(parents=True)
|
||
(out / "stages/06-hbm/a01/stdout.log").write_text("an attempt the controller had abandoned")
|
||
(tmp / "root").mkdir() # holds bundle.tar in a real job
|
||
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
|
||
assert (out / "agent/events.jsonl").read_bytes() == events and (out / "content-reviews/b.json").exists()
|
||
assert (out / "artifacts/j1.hbm").exists() and json.loads((tmp / "root/result/result.json").read_text())["exit_codes"] == {"r1": 0}
|
||
local = worker.BENCH / "runs/r1/j1/rollouts/00" # what the exporter reads: real dirs, bulk linked
|
||
assert not (worker.BENCH / "runs").is_symlink() and not (local / "stages").is_symlink()
|
||
assert (local / "stages/08-eval/a00/predictions.jsonl").is_file() and not (local / "stages/08-eval/a00/predictions.jsonl").is_symlink()
|
||
assert (local / "agent/events.jsonl").is_symlink() and (local / "agent/events.jsonl").read_bytes() == events
|
||
assert (local / "artifacts/j1.hbm").is_symlink() and (local / "stages/06-hbm/a00/spot_in.npz").exists()
|
||
assert not (local / "stages/06-hbm/a01/stdout.log").exists() # written to personal by a container only
|
||
(out / "rollout.json").write_text("changed in personal")
|
||
(archive / "01.tgz").write_bytes((archive / "00.tgz").read_bytes()[:200]) # caught mid-upload
|
||
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
|
||
assert (out / "rollout.json").read_text() == "changed in personal" # unpacked archives are not unpacked again
|
||
assert not list((out.parent / "01").glob(".unpacked-*")) # the partial one is retried next time
|
||
reap_check()
|
||
cleanup_check()
|
||
print("ok")
|
||
|
||
|
||
def reap_check():
|
||
"""Ended attempt: its Salty job is stopped, then removed once it has ended (failed too, 10-15); live jobs are left alone."""
|
||
from harness import salty_remote as sr
|
||
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
|
||
def make_cli(ps, calls):
|
||
remote = None
|
||
def once(args, **kw):
|
||
nonlocal remote
|
||
if args == ["ps"]:
|
||
return 0, ps, ""
|
||
calls.append(tuple(args))
|
||
if args[:2] == ["storage", "rm"]:
|
||
remote = args[3]
|
||
return 0, f"删除任务已受理:{task}(状态 queued)\n", ""
|
||
if args[:2] == ["storage", "task"]:
|
||
return 0, f"ID: {task}\n操作: delete\n状态: succeeded\n源: {remote}\n递归: true\n", ""
|
||
return 0, "", ""
|
||
return SimpleNamespace(once=once, call=lambda *a, **k: calls.append(a) or (0, "", ""), log=print)
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
runs = Path(tmp) / "runs"
|
||
rows = (("live", "running", False), ("orphan", "pending", True), ("stopped", "stopped", True), ("failed", "failed", True),
|
||
("cleaned", "running", True)) # removed by its own run after ps listed it: no stop retries
|
||
for name, _, ended in rows:
|
||
a = runs / f"r1/{name}/rollouts/00/stages/05-compile/a00"
|
||
(a / "salty").mkdir(parents=True)
|
||
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/s-" + name,
|
||
**({"cleaned": True} if name == "cleaned" else {})}))
|
||
if ended:
|
||
(a / "meta.json").write_text("{}")
|
||
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-stage\t{s}\t0\n" for name, s, _ in rows)
|
||
calls = []
|
||
cli = make_cli(ps, calls)
|
||
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
|
||
assert sorted(calls) == [("rm", "j-failed"), ("rm", "j-stopped"), ("stop", "j-orphan"),
|
||
("storage", "rm", "-r", "personal:/s-failed"), ("storage", "rm", "-r", "personal:/s-stopped"),
|
||
("storage", "rm-task", task), ("storage", "rm-task", task),
|
||
("storage", "task", task), ("storage", "task", task)], calls
|
||
assert sum(c[:2] == ("storage", "task") for c in calls) == 2
|
||
# No longer listed by ps but its attempt ended: the dir may still be there, so it is cleaned too.
|
||
a = runs / "r1/unlisted/rollouts/00/stages/05-compile/a00"
|
||
(a / "salty").mkdir(parents=True); (a / "meta.json").write_text("{}")
|
||
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-unlisted", "remote": "personal:/s-unlisted"}))
|
||
calls = []
|
||
offload.reap_orphans(make_cli(ps, calls), SimpleNamespace(RUNS=runs), sr, ["r1"])
|
||
assert ("storage", "rm", "-r", "personal:/s-unlisted") in calls and ("rm", "j-unlisted") in calls, calls
|
||
# bench dev containers: once the ticket is DONE (e.g. the rollout was cancelled mid-command) stop or remove them.
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
runs = Path(tmp) / "runs"
|
||
rows = (("devlive", "running", False), ("devcut", "running", True), ("devdone", "succeeded", True))
|
||
for name, _, ended in rows:
|
||
q = runs / f"r1/j/rollouts/00/stages/_queue/terminal-{name}"
|
||
(q / "salty").mkdir(parents=True)
|
||
(q / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/d-" + name}))
|
||
(q / "status.json").write_text(json.dumps({"state": "DONE" if ended else "RUNNING"}))
|
||
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-dev\t{s}\t0\n" for name, s, _ in rows)
|
||
calls = []
|
||
cli = make_cli(ps, calls)
|
||
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
|
||
assert sorted(calls) == [("rm", "j-devdone"), ("stop", "j-devcut"), ("storage", "rm", "-r", "personal:/d-devdone"),
|
||
("storage", "rm-task", task), ("storage", "task", task)], calls
|
||
|
||
|
||
|
||
def cleanup_check():
|
||
"""Refused or failed deletes go again only from the background, after a backoff; an unanswered one never does."""
|
||
from harness import salty_remote as sr
|
||
task = "4adc8ce7-6f08-477a-a3d5-aed4fb42d611"
|
||
def cli(*replies):
|
||
calls, todo = [], list(replies)
|
||
return calls, SimpleNamespace(once=lambda args, **kw: calls.append(args[:2]) or todo.pop(0), log=lambda m: None)
|
||
refused = (1, "", "错误:server 返回异常状态 405")
|
||
accepted = (0, f"删除任务已受理:{task}(状态 queued)\n", "")
|
||
def task_reply(status, error="-"):
|
||
return 0, f"ID: {task}\n操作: delete\n状态: {status}\n源: personal:/x\n递归: true\n错误: {error}\n", ""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
path = Path(tmp) / "remote.json"
|
||
state = {"remote": "personal:/x", "job_id": "j"}
|
||
calls, c = cli((0, "", ""), refused)
|
||
sr._cleanup(c, state, path) # result path: one try; the job goes first
|
||
sr._cleanup(c, state, path)
|
||
assert state["cleanup"]["status"] == "rejected" and calls == [["rm", "j"], ["storage", "rm"]], calls
|
||
sr._cleanup(c, state, path, retry_after=3600) # background, too soon: nothing sent
|
||
assert calls == [["rm", "j"], ["storage", "rm"]]
|
||
calls, c = cli(accepted, task_reply("failed", "internal:对象存储操作失败"))
|
||
sr._cleanup(c, state, path, retry_after=0) # due: submitted again, its task queried
|
||
assert state["cleanup"]["status"] == "failed" and state["cleanup"]["tries"] == 2 and not state.get("cleaned")
|
||
calls, c = cli(accepted, task_reply("failed", "not_found:源不存在"), (0, "", ""))
|
||
state["cleanup"]["at"] = 0
|
||
sr._cleanup(c, state, path, retry_after=1) # the earlier try did remove it
|
||
assert state["cleaned"] and calls == [["storage", "rm"], ["storage", "task"], ["storage", "rm-task"]], calls
|
||
assert json.loads(path.read_text())["cleaned"]
|
||
state = {"remote": "personal:/x", "job_id": "j"}
|
||
calls, c = cli((0, "", ""), (None, "", "timeout")) # no reply: unknown, never sent blindly again
|
||
sr._cleanup(c, state, path)
|
||
sr._cleanup(c, state, path, retry_after=0)
|
||
assert state["cleanup"]["status"] == "submission_unknown" and calls == [["rm", "j"], ["storage", "rm"]], calls
|
||
state = {"remote": "personal:/x", "job_id": "j"}
|
||
calls, c = cli((0, "", ""), accepted, task_reply("queued"), task_reply("succeeded"), (0, "", ""))
|
||
sr._cleanup(c, state, path) # result path leaves it queued
|
||
assert state["cleanup"]["status"] == "queued"
|
||
sr._cleanup(c, state, path, retry_after=sr.CLEANUP_RETRY_S) # background pass checks it again
|
||
assert state["cleaned"] and calls[-2:] == [["storage", "task"], ["storage", "rm-task"]], calls
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|