10-14 重启后续跑的轨迹自己删掉了作业,ps 仍短暂列出,清理去 stop 得到 404,按网络故障重试 30 分钟,期间归档和导出都停着。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
154 lines
9.1 KiB
Python
154 lines
9.1 KiB
Python
"""Archive (review gate, mirrored products, overlays, freeing) → container unpack. Run: PYTHONPATH=. python3 scripts/test_salty_offload.py"""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sqlite3
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
from types import SimpleNamespace
|
|
|
|
from harness import salty_remote_worker as worker
|
|
|
|
spec = importlib.util.spec_from_file_location("salty_offload", Path(__file__).with_name("salty_offload.py"))
|
|
offload = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(offload)
|
|
TERMINAL = ("PASS", "FAIL", "UNVERIFIED")
|
|
|
|
|
|
class Cli:
|
|
"""personal:/ is a local directory."""
|
|
def __init__(self, root):
|
|
self.root = root
|
|
|
|
def local(self, remote):
|
|
return self.root / remote.removeprefix("personal:/")
|
|
|
|
def once(self, args):
|
|
assert args[:2] == ["storage", "ls"], args
|
|
d = self.local(args[2])
|
|
if not d.is_dir():
|
|
return 1, "", "not_found"
|
|
return 0, "名称\t类型\t大小\n" + "".join(f"{p.name}\t文件\t{p.stat().st_size}\n" for p in d.iterdir() if p.is_file()), ""
|
|
|
|
def call(self, *args, timeout=120):
|
|
if args[:2] == ("storage", "mkdir"):
|
|
self.local(args[3]).mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
assert args[:2] == ("storage", "cp"), args
|
|
shutil.copyfile(args[2], self.local(args[3]))
|
|
return 0, "", ""
|
|
|
|
|
|
def main():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp = Path(tmp)
|
|
personal = tmp / "personal"
|
|
paths = SimpleNamespace(RUNS=tmp / "runs", DB_PATH=tmp / "jobs.sqlite", DATASETS=tmp / "datasets")
|
|
cli = Cli(personal)
|
|
rd = paths.RUNS / "r1/j1/rollouts/00"
|
|
for rel, size in (("agent/events.jsonl", 2 << 20), ("artifacts/j1.hbm", 2 << 20), ("artifacts/j1.bc", 2 << 20),
|
|
("rollout.json", 10), ("stages/08-eval/a00/predictions.jsonl", 10),
|
|
("stages/06-hbm/a00/spot_in.npz", 2 << 20)):
|
|
(rd / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
(rd / rel).write_bytes(os.urandom(size))
|
|
(rd / "execution.json").write_text('{"execution_state": "TERMINAL"}')
|
|
mirror = personal / "bench-root/runs/r1/j1/rollouts/00/artifacts"
|
|
mirror.mkdir(parents=True)
|
|
shutil.copyfile(rd / "artifacts/j1.hbm", mirror / "j1.hbm") # the container wrote it there
|
|
with sqlite3.connect(paths.DB_PATH) as con:
|
|
con.execute("CREATE TABLE rollouts (run_id, job_id, kk, state, finished_at)")
|
|
con.execute("INSERT INTO rollouts VALUES ('r1', 'j1', 0, 'PASS', ?)", (time.time() - 7200,))
|
|
archive = personal / "bench-root/archive/r1/j1"
|
|
|
|
def run():
|
|
return offload.archive_all(cli, paths, ["r1"], TERMINAL)
|
|
|
|
assert run() == 0 and not archive.exists() # PASS, export has not judged it yet
|
|
(paths.DATASETS / "r1-training").mkdir(parents=True)
|
|
(paths.DATASETS / "r1-training/disposition.jsonl").write_text(
|
|
json.dumps({"job_id": "j1", "rollout": 0, "reasons": ["sft:content-review-unreviewed"]}) + "\n")
|
|
assert run() == 0 and not archive.exists() # still waiting for its review
|
|
(rd / "content-reviews").mkdir()
|
|
(rd / "content-reviews/a.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
|
|
events = (rd / "agent/events.jsonl").read_bytes()
|
|
assert run() == 1
|
|
assert sorted(tarfile.open(archive / "00.tgz").getnames()) == [
|
|
f"r1/j1/rollouts/00/{n}" for n in ("agent/events.jsonl", "artifacts/j1.bc", "content-reviews/a.json",
|
|
"execution.json", "rollout.json", "stages/06-hbm/a00/spot_in.npz",
|
|
"stages/08-eval/a00/predictions.jsonl")] # mirrored HBM left out
|
|
assert not (rd / "agent/events.jsonl").exists() and not (rd / "artifacts/j1.hbm").exists()
|
|
assert (rd / "rollout.json").exists() and run() == 0 # freed and unchanged: nothing to do
|
|
|
|
info = json.loads((rd / "ARCHIVED.json").read_text())
|
|
(rd / "content-reviews/b.json").write_text('{"state": "COMPLETE", "decision": "PASS"}')
|
|
os.utime(rd / "content-reviews/b.json", (info["at"] + 10, info["at"] + 10))
|
|
assert run() == 1
|
|
overlay, = [p for p in archive.iterdir() if p.name != "00.tgz"]
|
|
assert overlay.name.startswith("00.u") and tarfile.open(overlay).getnames() == ["r1/j1/rollouts/00/content-reviews/b.json"]
|
|
assert json.loads((rd / "ARCHIVED.json").read_text())["overlays"][0]["archive"].endswith(overlay.name)
|
|
|
|
worker.PERSONAL, worker.BENCH = personal, tmp / "bench"
|
|
worker.subprocess = SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=0), DEVNULL=None)
|
|
out = personal / "bench-root/runs/r1/j1/rollouts/00"
|
|
(out / "stages/06-hbm/a01").mkdir(parents=True)
|
|
(out / "stages/06-hbm/a01/stdout.log").write_text("an attempt the controller had abandoned")
|
|
(tmp / "root").mkdir() # holds bundle.tar in a real job
|
|
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
|
|
assert (out / "agent/events.jsonl").read_bytes() == events and (out / "content-reviews/b.json").exists()
|
|
assert (out / "artifacts/j1.hbm").exists() and json.loads((tmp / "root/result/result.json").read_text())["exit_codes"] == {"r1": 0}
|
|
local = worker.BENCH / "runs/r1/j1/rollouts/00" # what the exporter reads: real dirs, bulk linked
|
|
assert not (worker.BENCH / "runs").is_symlink() and not (local / "stages").is_symlink()
|
|
assert (local / "stages/08-eval/a00/predictions.jsonl").is_file() and not (local / "stages/08-eval/a00/predictions.jsonl").is_symlink()
|
|
assert (local / "agent/events.jsonl").is_symlink() and (local / "agent/events.jsonl").read_bytes() == events
|
|
assert (local / "artifacts/j1.hbm").is_symlink() and (local / "stages/06-hbm/a00/spot_in.npz").exists()
|
|
assert not (local / "stages/06-hbm/a01/stdout.log").exists() # written to personal by a container only
|
|
(out / "rollout.json").write_text("changed in personal")
|
|
(archive / "01.tgz").write_bytes((archive / "00.tgz").read_bytes()[:200]) # caught mid-upload
|
|
worker.export({"identity": {"export": "e"}, "export": {"r1": "cfg.yaml"}}, tmp / "root")
|
|
assert (out / "rollout.json").read_text() == "changed in personal" # unpacked archives are not unpacked again
|
|
assert not list((out.parent / "01").glob(".unpacked-*")) # the partial one is retried next time
|
|
reap_check()
|
|
print("ok")
|
|
|
|
|
|
def reap_check():
|
|
"""Ended attempt: its Salty job is stopped, then removed once stopped; live and failed jobs are left alone."""
|
|
from harness import salty_remote as sr
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
runs = Path(tmp) / "runs"
|
|
rows = (("live", "running", False), ("orphan", "pending", True), ("stopped", "stopped", True), ("failed", "failed", True),
|
|
("cleaned", "running", True)) # removed by its own run after ps listed it: no stop retries
|
|
for name, _, ended in rows:
|
|
a = runs / f"r1/{name}/rollouts/00/stages/05-compile/a00"
|
|
(a / "salty").mkdir(parents=True)
|
|
(a / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/s-" + name,
|
|
**({"cleaned": True} if name == "cleaned" else {})}))
|
|
if ended:
|
|
(a / "meta.json").write_text("{}")
|
|
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-stage\t{s}\t0\n" for name, s, _ in rows)
|
|
calls = []
|
|
cli = SimpleNamespace(once=lambda args: (0, ps, ""), call=lambda *a, **k: calls.append(a) or (0, "", ""), log=print)
|
|
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
|
|
assert sorted(calls) == [("rm", "j-stopped"), ("stop", "j-orphan"), ("storage", "rm", "-r", "personal:/s-stopped")], calls
|
|
# bench dev containers: once the ticket is DONE (e.g. the rollout was cancelled mid-command) stop or remove them.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
runs = Path(tmp) / "runs"
|
|
rows = (("devlive", "running", False), ("devcut", "running", True), ("devdone", "succeeded", True))
|
|
for name, _, ended in rows:
|
|
q = runs / f"r1/j/rollouts/00/stages/_queue/terminal-{name}"
|
|
(q / "salty").mkdir(parents=True)
|
|
(q / "salty/remote.json").write_text(json.dumps({"job_id": "j-" + name, "remote": "personal:/d-" + name}))
|
|
(q / "status.json").write_text(json.dumps({"state": "DONE" if ended else "RUNNING"}))
|
|
ps = "ID\t名称\t状态\n" + "".join(f"j-{name}\tbench-dev\t{s}\t0\n" for name, s, _ in rows)
|
|
calls = []
|
|
cli = SimpleNamespace(once=lambda args: (0, ps, ""), call=lambda *a, **k: calls.append(a) or (0, "", ""), log=print)
|
|
offload.reap_orphans(cli, SimpleNamespace(RUNS=runs), sr, ["r1"])
|
|
assert sorted(calls) == [("rm", "j-devdone"), ("stop", "j-devcut"), ("storage", "rm", "-r", "personal:/d-devdone")], calls
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|