data-gen-bpu/scripts/salty_offload.py
bench 77031cd05b Salty cleanup: refused or failed deletes are retried from the background
With Salty 0.2.4 a delete is a background task; since 10-19 every one fails
server side, and a failure was recorded once and never retried, so garbage
would pile up for good. Now a submission the server refused (an HTTP error
such as 405) or a task that failed/was interrupted is submitted again by the
reaper after 30 min, doubling to 6 h; a task failing with not_found means an
earlier try removed it. A submission with no reply stays unknown and is never
resent blindly. The result path still makes one try. The reaper also covers
ended jobs ps no longer lists, logs only removals, and salty_offload gains
--reap-only so staging gets a background cleaner without archive/export.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 14:02:07 +08:00

281 lines
15 KiB
Python

#!/usr/bin/env python3
"""Move finished rollouts to Salty personal space and build the datasets there; the controller keeps
only small files and the final datasets.
Every 10 minutes:
1. archive: a rollout that ended over 30 min ago (UNVERIFIED: 24 h, it can still be re-verified), whose
execution is TERMINAL and, if PASS, whose content review is settled, goes to
personal:/bench-root/archive/<run>/<job>/<kk>.tgz. Stage products already in personal:/bench-root/runs
with the same name and size are left out. Later changes go to <kk>.u<time>.tgz overlays; an archive is
never rewritten. Then every local file over 1 MB is deleted, and so is its compile scratch in work_root.
2. export: at most hourly and only after new archives, one Salty job unpacks the archives into personal
runs and runs export/to_agentrl.py there (salty_remote_worker.export); the datasets stay in
personal:/bench-root/datasets/ and are copied to bench/datasets-final/.
The collectors keep their own per-rollout export into bench/datasets/: it finds and reviews new PASS
rollouts while their files are still here, which is what step 1 waits for.
Usage: salty_offload.py RUN=CONFIG [RUN=CONFIG ...] [--once] [--export-now]
"""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import os
import shutil
import sqlite3
import sys
import tarfile
import tempfile
import time
import uuid
from pathlib import Path
from types import SimpleNamespace
PRODUCTS = (".hbm", ".bc", ".npz", ".onnx")
BIG = 1 << 20
def log(message):
print(f"{time.strftime('%F %T')} {message}", flush=True)
def remote_sizes(cli, remote_dir):
rc, out, _ = cli.once(["storage", "ls", remote_dir])
rows = (line.split("\t") for line in out.splitlines()[1:]) if rc == 0 else ()
return {f[0]: int(f[2]) for f in rows if len(f) > 2 and f[1] == "文件"}
def settled(rd, reasons):
"""PASS leaves once the collector's export has judged it: a finished review, or excluded for another reason."""
for receipt in (rd / "content-reviews").glob("*.json"):
with contextlib.suppress(ValueError, OSError):
if json.loads(receipt.read_text()).get("state") == "COMPLETE":
return True
return reasons is not None and not any(r.endswith("content-review-unreviewed") for r in reasons)
def archive(cli, paths, rd):
"""Upload what changed since the last archive of this rollout, then free its local bulk."""
marker = rd / "ARCHIVED.json"
info = json.loads(marker.read_text()) if marker.exists() else {}
started = time.time()
files = [p for p in rd.rglob("*") if p.is_file() and not p.is_symlink() and p != marker]
changed = [p for p in files if p.stat().st_mtime > info.get("at", 0)]
if changed:
mirrored, listings = set(), {}
for p in changed:
if p.suffix in PRODUCTS and p.stat().st_size > BIG:
d = str(p.parent.relative_to(paths.RUNS))
listings.setdefault(d, remote_sizes(cli, f"personal:/bench-root/runs/{d}"))
if listings[d].get(p.name) == p.stat().st_size:
mirrored.add(p)
run, job, kk = rd.parts[-4], rd.parts[-3], rd.name
name = f"{kk}.tgz" if not info else f"{kk}.u{int(started)}.tgz"
remote = f"personal:/bench-root/archive/{run}/{job}/{name}"
with tempfile.TemporaryDirectory(prefix="salty-archive-") as tmp:
tgz = Path(tmp) / name
with tarfile.open(tgz, "w:gz") as tar:
for p in changed:
if p not in mirrored:
tar.add(p, str(p.relative_to(paths.RUNS)))
cli.call("storage", "mkdir", "-p", remote.rsplit("/", 1)[0])
cli.call("storage", "cp", str(tgz), remote, timeout=3600)
size = tgz.stat().st_size
if remote_sizes(cli, remote.rsplit("/", 1)[0]).get(name) != size:
raise RuntimeError(f"uploaded {name} size differs")
entry = {"archive": remote, "bytes": size, "sha256": hashlib.sha256(tgz.read_bytes()).hexdigest(),
"mirrored": sorted(str(p.relative_to(paths.RUNS)) for p in mirrored)}
info = {**entry, **info, "at": started}
if name != f"{kk}.tgz":
info["overlays"] = [*info.get("overlays", []), entry]
marker.write_text(json.dumps(info, ensure_ascii=False, indent=1))
freed = 0
for p in files:
if p.exists() and p.stat().st_size > BIG:
freed += p.stat().st_size
p.unlink()
# Its compile scratch (hb_compile working_dir, only 05-compile/06-hbm read it) on the VM's own disk:
# 10-18 742 finished rollouts left 36 GB there and the Mac mini disk filled.
shutil.rmtree(paths.expand(paths.config()["paths"]["work_root"]) / rd.parts[-4] / rd.parts[-3] / f"r{int(rd.name):02d}",
ignore_errors=True)
return len(changed), freed
def archive_all(cli, paths, runs, terminal):
con = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True)
rows = con.execute(f"SELECT run_id, job_id, kk, state, finished_at FROM rollouts WHERE run_id IN "
f"({','.join('?' * len(runs))})", tuple(runs)).fetchall()
con.close()
reasons = {}
for run in runs:
with contextlib.suppress(OSError):
for line in (paths.DATASETS / f"{run}-training/disposition.jsonl").read_text().splitlines():
item = json.loads(line)
reasons[(run, item["job_id"], item["rollout"])] = item["reasons"]
uploaded = 0
for run, job, kk, state, finished in rows:
rd = paths.RUNS / run / job / "rollouts" / f"{kk:02d}"
delay = 86400 if state == "UNVERIFIED" else 1800
if state not in terminal or not finished or finished > time.time() - delay or not rd.is_dir():
continue
marker = rd / "ARCHIVED.json"
at = json.loads(marker.read_text())["at"] if marker.exists() else None
if at is not None and all(p.stat().st_size <= BIG and p.stat().st_mtime <= at for p in rd.rglob("*")
if p.is_file() and not p.is_symlink() and p != marker):
continue # archived, freed and unchanged
execution = rd / "execution.json"
if execution.exists() and json.loads(execution.read_text()).get("execution_state") != "TERMINAL":
continue
if state == "PASS" and not settled(rd, reasons.get((run, job, kk))):
continue
try:
changed, freed = archive(cli, paths, rd)
uploaded += bool(changed)
log(f"{run} {job} {kk}: archived {changed} files, freed {freed >> 20}MB")
except Exception as exc: # next pass retries; nothing local was deleted before the upload was checked
log(f"{run} {job} {kk}: SKIP {type(exc).__name__}: {exc}")
return uploaded
def reap_orphans(cli, paths, sr, runs):
"""A Salty job whose controller attempt already ended (phase deadline, cancel, crash) has no reader:
stop it, and once it has ended remove it and its dir like salty_remote removes its own jobs (their logs and
describe are already in the attempt's stderr.log). 10-13: 12 such jobs held ~64 cores while ours queued."""
rc, out, _ = cli.once(["ps"])
if rc:
return
status = {f[0]: f[2] for f in (line.split("\t") for line in out.splitlines()[1:]) if len(f) > 2}
quiet = SimpleNamespace(once=cli.once, log=lambda message: None)
for run in runs:
attempts = (paths.RUNS / run).glob("*/rollouts/*/stages/*/a*/salty/remote.json")
# bench dev commands (harness/salty_dev.py): ended when their ticket is DONE; nothing to keep for forensics.
dev = (paths.RUNS / run).glob("*/rollouts/*/stages/_queue/terminal-*/salty/remote.json")
for p in [*attempts, *dev]:
state = json.loads(p.read_text())
if state.get("cleaned") or not state.get("job_id"): # removed by its own run / never submitted
continue
s = status.get(state.get("job_id"))
is_dev = p.parent.parent.name.startswith("terminal-")
if is_dev:
try:
ended = json.loads((p.parent.parent / "status.json").read_text()).get("state") == "DONE"
except (OSError, ValueError):
ended = False
else:
ended = (p.parent.parent / "meta.json").exists()
if s not in ("running", "pending", "stopped", "succeeded", "failed", None) or not ended:
continue
if s in ("running", "pending"):
cli.call("stop", state["job_id"], ok_404=True) # 404: gone meanwhile, nothing to stop (10-14: 30 min of retries)
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: stop requested")
continue
# Ended, or no longer listed: its dir may still be there. Refused/failed deletes are submitted again later.
sr._cleanup(quiet, state, p, retry_after=sr.CLEANUP_RETRY_S) # its receipt records why, every pass
if state.get("cleaned"):
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: removed")
def export(cli, paths, sr, policy, runs):
name = "bench-export-" + str(uuid.uuid4())
bench = paths.BENCH_ROOT
with tempfile.TemporaryDirectory(prefix="salty-export-") as tmp:
work = Path(tmp)
with contextlib.closing(sqlite3.connect(paths.DB_PATH)) as src, \
contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as dst:
src.backup(dst)
files, configs = {str(bench / "state/jobs.sqlite"): work / "jobs.sqlite"}, {}
with contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as con:
for run, config in runs.items():
configs[run] = str(bench / "export-configs" / f"{run}.yaml")
files[configs[run]] = Path(config)
seed = paths.DATASETS / f"{run}-training/manifest.json"
if seed.exists():
files[str(bench / "export-seed" / f"{run}.json")] = seed
for (job,) in con.execute("SELECT job_id FROM jobs WHERE run_id=?", (run,)):
if (paths.RUNS / run / job / "job.json").exists():
files[str(paths.RUNS / run / job / "job.json")] = paths.RUNS / run / job / "job.json"
request = {"identity": {"export": name}, "stage": "export", "export": configs, "env": {}, "links": {},
"files": {destination: f"files/{i}" for i, destination in enumerate(files)}}
data = json.dumps(request).encode()
with tarfile.open(work / "bundle.tar", "w:gz") as tar: # the worker opens it with tarfile's autodetect
info = tarfile.TarInfo("request.json")
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
for i, source in enumerate(files.values()):
tar.add(source, f"files/{i}")
for d in (*sr.CODE_DIRS, "agent"):
tar.add(bench / d, f"code/{d}", filter=lambda m: None if "__pycache__" in m.name else m)
shutil.copyfile(sr.WORKER, work / "worker.py")
state = {"name": name, "remote": "personal:/" + name}
state["job_id"] = sr._submit(cli, state, policy, SimpleNamespace(name="export"), work,
policy.get("export_memory", "32Gi"))
log(f"export job {state['job_id']} submitted")
job_state = sr._wait(cli, state["job_id"], policy, time.time(), time.time, time.sleep)
if job_state != "succeeded":
sr._remote_logs(policy, state["job_id"], work / "remote.log")
log(f"export job {job_state}, kept for forensics:\n{(work / 'remote.log').read_text()[-4000:]}")
return False
sr._fetch(cli, state["remote"] + "/result/result.json", work / "result.json")
sr._fetch(cli, state["remote"] + "/result/datasets.tar.gz", work / "datasets.tar.gz")
codes = json.loads((work / "result.json").read_text())["exit_codes"]
with tarfile.open(work / "datasets.tar.gz") as tar:
tar.extractall(work / "out", members=[m for m in tar.getmembers() if m.isfile() and ".." not in m.name
and not m.name.startswith("/")])
final = bench / "datasets-final"
final.mkdir(exist_ok=True)
for run, rc in codes.items():
if rc != 0:
log(f"export of {run} failed (exit {rc}); its last good dataset stays")
continue
target, old = final / f"{run}-training", final / f".{run}-training.old"
shutil.rmtree(old, ignore_errors=True)
if target.exists():
target.rename(old)
(work / "out" / f"{run}-training").rename(target)
shutil.rmtree(old, ignore_errors=True)
manifest = json.loads((target / "manifest.json").read_text())
log(f"{run}: sft {manifest['stats']['sft_eligible']} of {manifest['stats']['scanned']} scanned")
sr._cleanup(cli, state, paths.STATE / "salty-export" / f"{name}.json")
return all(rc == 0 for rc in codes.values())
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("runs", nargs="+", metavar="RUN=CONFIG")
ap.add_argument("--once", action="store_true", help="one pass, then exit")
ap.add_argument("--export-now", action="store_true", help="export on the first pass even if nothing new was archived")
ap.add_argument("--reap-only", action="store_true", help="only stop/remove ended attempts' Salty jobs and dirs (staging)")
a = ap.parse_args()
runs = dict(item.split("=", 1) for item in a.runs)
os.environ["BENCH_CONFIG"] = next(iter(runs.values())) # salty CLI and image come from the first config
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_remote as sr
from export.to_agentrl import DB_TERMINAL_STATES
policy = dict(paths.config()["execution"]["salty_remote"])
if "@sha256:" not in policy.get("image", ""):
raise SystemExit("execution.salty_remote.image must be pinned by digest")
cli = sr._Cli(policy.get("salty", "salty"), policy.get("outage_seconds", 1800), time.sleep, time.time, log)
pending, last_export = a.export_now, 0.0
while True:
try:
reap_orphans(cli, paths, sr, runs)
except Exception as exc: # retried next pass
log(f"reap failed: {type(exc).__name__}: {exc}")
pending = not a.reap_only and archive_all(cli, paths, runs, DB_TERMINAL_STATES) > 0 or pending
if pending and not a.reap_only and time.time() - last_export >= 3600:
last_export = time.time()
try:
pending = not export(cli, paths, sr, policy, runs)
except Exception as exc: # retried next hour; the last good datasets stay in place
log(f"export failed: {type(exc).__name__}: {exc}")
if a.once:
return 0
time.sleep(600)
if __name__ == "__main__":
sys.exit(main())