data-gen-bpu/scripts/salty_offload.py
bench 67d306de0f salty_offload 孤儿清理:跳过已自行清理的作业,停止时 404 视为已不在
10-14 重启后续跑的轨迹自己删掉了作业,ps 仍短暂列出,清理去 stop 得到 404,按网络故障重试 30 分钟,期间归档和导出都停着。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 19:16:43 +08:00

273 lines
14 KiB
Python

#!/usr/bin/env python3
"""Move finished rollouts to Salty personal space and build the datasets there; the controller keeps
only small files and the final datasets.
Every 10 minutes:
1. archive: a rollout that ended over 30 min ago (UNVERIFIED: 24 h, it can still be re-verified), whose
execution is TERMINAL and, if PASS, whose content review is settled, goes to
personal:/bench-root/archive/<run>/<job>/<kk>.tgz. Stage products already in personal:/bench-root/runs
with the same name and size are left out. Later changes go to <kk>.u<time>.tgz overlays; an archive is
never rewritten. Then every local file over 1 MB is deleted.
2. export: at most hourly and only after new archives, one Salty job unpacks the archives into personal
runs and runs export/to_agentrl.py there (salty_remote_worker.export); the datasets stay in
personal:/bench-root/datasets/ and are copied to bench/datasets-final/.
The collectors keep their own per-rollout export into bench/datasets/: it finds and reviews new PASS
rollouts while their files are still here, which is what step 1 waits for.
Usage: salty_offload.py RUN=CONFIG [RUN=CONFIG ...] [--once] [--export-now]
"""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import os
import shutil
import sqlite3
import sys
import tarfile
import tempfile
import time
import uuid
from pathlib import Path
from types import SimpleNamespace
PRODUCTS = (".hbm", ".bc", ".npz", ".onnx")
BIG = 1 << 20
def log(message):
print(f"{time.strftime('%F %T')} {message}", flush=True)
def remote_sizes(cli, remote_dir):
rc, out, _ = cli.once(["storage", "ls", remote_dir])
rows = (line.split("\t") for line in out.splitlines()[1:]) if rc == 0 else ()
return {f[0]: int(f[2]) for f in rows if len(f) > 2 and f[1] == "文件"}
def settled(rd, reasons):
"""PASS leaves once the collector's export has judged it: a finished review, or excluded for another reason."""
for receipt in (rd / "content-reviews").glob("*.json"):
with contextlib.suppress(ValueError, OSError):
if json.loads(receipt.read_text()).get("state") == "COMPLETE":
return True
return reasons is not None and not any(r.endswith("content-review-unreviewed") for r in reasons)
def archive(cli, paths, rd):
"""Upload what changed since the last archive of this rollout, then free its local bulk."""
marker = rd / "ARCHIVED.json"
info = json.loads(marker.read_text()) if marker.exists() else {}
started = time.time()
files = [p for p in rd.rglob("*") if p.is_file() and not p.is_symlink() and p != marker]
changed = [p for p in files if p.stat().st_mtime > info.get("at", 0)]
if changed:
mirrored, listings = set(), {}
for p in changed:
if p.suffix in PRODUCTS and p.stat().st_size > BIG:
d = str(p.parent.relative_to(paths.RUNS))
listings.setdefault(d, remote_sizes(cli, f"personal:/bench-root/runs/{d}"))
if listings[d].get(p.name) == p.stat().st_size:
mirrored.add(p)
run, job, kk = rd.parts[-4], rd.parts[-3], rd.name
name = f"{kk}.tgz" if not info else f"{kk}.u{int(started)}.tgz"
remote = f"personal:/bench-root/archive/{run}/{job}/{name}"
with tempfile.TemporaryDirectory(prefix="salty-archive-") as tmp:
tgz = Path(tmp) / name
with tarfile.open(tgz, "w:gz") as tar:
for p in changed:
if p not in mirrored:
tar.add(p, str(p.relative_to(paths.RUNS)))
cli.call("storage", "mkdir", "-p", remote.rsplit("/", 1)[0])
cli.call("storage", "cp", str(tgz), remote, timeout=3600)
size = tgz.stat().st_size
if remote_sizes(cli, remote.rsplit("/", 1)[0]).get(name) != size:
raise RuntimeError(f"uploaded {name} size differs")
entry = {"archive": remote, "bytes": size, "sha256": hashlib.sha256(tgz.read_bytes()).hexdigest(),
"mirrored": sorted(str(p.relative_to(paths.RUNS)) for p in mirrored)}
info = {**entry, **info, "at": started}
if name != f"{kk}.tgz":
info["overlays"] = [*info.get("overlays", []), entry]
marker.write_text(json.dumps(info, ensure_ascii=False, indent=1))
freed = 0
for p in files:
if p.exists() and p.stat().st_size > BIG:
freed += p.stat().st_size
p.unlink()
return len(changed), freed
def archive_all(cli, paths, runs, terminal):
con = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True)
rows = con.execute(f"SELECT run_id, job_id, kk, state, finished_at FROM rollouts WHERE run_id IN "
f"({','.join('?' * len(runs))})", tuple(runs)).fetchall()
con.close()
reasons = {}
for run in runs:
with contextlib.suppress(OSError):
for line in (paths.DATASETS / f"{run}-training/disposition.jsonl").read_text().splitlines():
item = json.loads(line)
reasons[(run, item["job_id"], item["rollout"])] = item["reasons"]
uploaded = 0
for run, job, kk, state, finished in rows:
rd = paths.RUNS / run / job / "rollouts" / f"{kk:02d}"
delay = 86400 if state == "UNVERIFIED" else 1800
if state not in terminal or not finished or finished > time.time() - delay or not rd.is_dir():
continue
marker = rd / "ARCHIVED.json"
at = json.loads(marker.read_text())["at"] if marker.exists() else None
if at is not None and all(p.stat().st_size <= BIG and p.stat().st_mtime <= at for p in rd.rglob("*")
if p.is_file() and not p.is_symlink() and p != marker):
continue # archived, freed and unchanged
execution = rd / "execution.json"
if execution.exists() and json.loads(execution.read_text()).get("execution_state") != "TERMINAL":
continue
if state == "PASS" and not settled(rd, reasons.get((run, job, kk))):
continue
try:
changed, freed = archive(cli, paths, rd)
uploaded += bool(changed)
log(f"{run} {job} {kk}: archived {changed} files, freed {freed >> 20}MB")
except Exception as exc: # next pass retries; nothing local was deleted before the upload was checked
log(f"{run} {job} {kk}: SKIP {type(exc).__name__}: {exc}")
return uploaded
def reap_orphans(cli, paths, sr, runs):
"""A Salty job whose controller attempt already ended (phase deadline, cancel, crash) has no reader:
stop it, and once stopped remove it like salty_remote removes its own stopped jobs. Failed jobs stay
for forensics. 10-13: 12 such jobs held ~64 cores while ours queued."""
rc, out, _ = cli.once(["ps"])
if rc:
return
status = {f[0]: f[2] for f in (line.split("\t") for line in out.splitlines()[1:]) if len(f) > 2}
for run in runs:
attempts = (paths.RUNS / run).glob("*/rollouts/*/stages/*/a*/salty/remote.json")
# bench dev commands (harness/salty_dev.py): ended when their ticket is DONE; nothing to keep for forensics.
dev = (paths.RUNS / run).glob("*/rollouts/*/stages/_queue/terminal-*/salty/remote.json")
for p in [*attempts, *dev]:
state = json.loads(p.read_text())
if state.get("cleaned"): # its own run already removed it; ps can still list it for a moment
continue
s = status.get(state.get("job_id"))
is_dev = p.parent.parent.name.startswith("terminal-")
if is_dev:
try:
ended = json.loads((p.parent.parent / "status.json").read_text()).get("state") == "DONE"
except (OSError, ValueError):
ended = False
else:
ended = (p.parent.parent / "meta.json").exists()
if s not in ("running", "pending", "stopped", *(("succeeded", "failed") if is_dev else ())) or not ended:
continue
if s in ("stopped", "succeeded", "failed"):
sr._cleanup(cli, state)
else:
cli.call("stop", state["job_id"], ok_404=True) # 404: gone meanwhile, nothing to stop (10-14: 30 min of retries)
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: "
+ ("removed" if s in ("stopped", "succeeded", "failed") else "stop requested"))
def export(cli, paths, sr, policy, runs):
name = "bench-export-" + str(uuid.uuid4())
bench = paths.BENCH_ROOT
with tempfile.TemporaryDirectory(prefix="salty-export-") as tmp:
work = Path(tmp)
with contextlib.closing(sqlite3.connect(paths.DB_PATH)) as src, \
contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as dst:
src.backup(dst)
files, configs = {str(bench / "state/jobs.sqlite"): work / "jobs.sqlite"}, {}
with contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as con:
for run, config in runs.items():
configs[run] = str(bench / "export-configs" / f"{run}.yaml")
files[configs[run]] = Path(config)
seed = paths.DATASETS / f"{run}-training/manifest.json"
if seed.exists():
files[str(bench / "export-seed" / f"{run}.json")] = seed
for (job,) in con.execute("SELECT job_id FROM jobs WHERE run_id=?", (run,)):
if (paths.RUNS / run / job / "job.json").exists():
files[str(paths.RUNS / run / job / "job.json")] = paths.RUNS / run / job / "job.json"
request = {"identity": {"export": name}, "stage": "export", "export": configs, "env": {}, "links": {},
"files": {destination: f"files/{i}" for i, destination in enumerate(files)}}
data = json.dumps(request).encode()
with tarfile.open(work / "bundle.tar", "w:gz") as tar: # the worker opens it with tarfile's autodetect
info = tarfile.TarInfo("request.json")
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
for i, source in enumerate(files.values()):
tar.add(source, f"files/{i}")
for d in (*sr.CODE_DIRS, "agent"):
tar.add(bench / d, f"code/{d}", filter=lambda m: None if "__pycache__" in m.name else m)
shutil.copyfile(sr.WORKER, work / "worker.py")
state = {"name": name, "remote": "personal:/" + name}
state["job_id"] = sr._submit(cli, state, policy, SimpleNamespace(name="export"), work,
policy.get("export_memory", "32Gi"))
log(f"export job {state['job_id']} submitted")
job_state = sr._wait(cli, state["job_id"], policy, time.time(), time.time, time.sleep)
if job_state != "succeeded":
sr._remote_logs(policy, state["job_id"], work / "remote.log")
log(f"export job {job_state}, kept for forensics:\n{(work / 'remote.log').read_text()[-4000:]}")
return False
sr._fetch(cli, state["remote"] + "/result/result.json", work / "result.json")
sr._fetch(cli, state["remote"] + "/result/datasets.tar.gz", work / "datasets.tar.gz")
codes = json.loads((work / "result.json").read_text())["exit_codes"]
with tarfile.open(work / "datasets.tar.gz") as tar:
tar.extractall(work / "out", members=[m for m in tar.getmembers() if m.isfile() and ".." not in m.name
and not m.name.startswith("/")])
final = bench / "datasets-final"
final.mkdir(exist_ok=True)
for run, rc in codes.items():
if rc != 0:
log(f"export of {run} failed (exit {rc}); its last good dataset stays")
continue
target, old = final / f"{run}-training", final / f".{run}-training.old"
shutil.rmtree(old, ignore_errors=True)
if target.exists():
target.rename(old)
(work / "out" / f"{run}-training").rename(target)
shutil.rmtree(old, ignore_errors=True)
manifest = json.loads((target / "manifest.json").read_text())
log(f"{run}: sft {manifest['stats']['sft_eligible']} of {manifest['stats']['scanned']} scanned")
sr._cleanup(cli, state)
return all(rc == 0 for rc in codes.values())
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("runs", nargs="+", metavar="RUN=CONFIG")
ap.add_argument("--once", action="store_true", help="one pass, then exit")
ap.add_argument("--export-now", action="store_true", help="export on the first pass even if nothing new was archived")
a = ap.parse_args()
runs = dict(item.split("=", 1) for item in a.runs)
os.environ["BENCH_CONFIG"] = next(iter(runs.values())) # salty CLI and image come from the first config
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_remote as sr
from export.to_agentrl import DB_TERMINAL_STATES
policy = dict(paths.config()["execution"]["salty_remote"])
if "@sha256:" not in policy.get("image", ""):
raise SystemExit("execution.salty_remote.image must be pinned by digest")
cli = sr._Cli(policy.get("salty", "salty"), policy.get("outage_seconds", 1800), time.sleep, time.time, log)
pending, last_export = a.export_now, 0.0
while True:
try:
reap_orphans(cli, paths, sr, runs)
except Exception as exc: # retried next pass
log(f"reap failed: {type(exc).__name__}: {exc}")
pending = archive_all(cli, paths, runs, DB_TERMINAL_STATES) > 0 or pending
if pending and time.time() - last_export >= 3600:
last_export = time.time()
try:
pending = not export(cli, paths, sr, policy, runs)
except Exception as exc: # retried next hour; the last good datasets stay in place
log(f"export failed: {type(exc).__name__}: {exc}")
if a.once:
return 0
time.sleep(600)
if __name__ == "__main__":
sys.exit(main())