With Salty 0.2.4 a delete is a background task; since 10-19 every one fails server side, and a failure was recorded once and never retried, so garbage would pile up for good. Now a submission the server refused (an HTTP error such as 405) or a task that failed/was interrupted is submitted again by the reaper after 30 min, doubling to 6 h; a task failing with not_found means an earlier try removed it. A submission with no reply stays unknown and is never resent blindly. The result path still makes one try. The reaper also covers ended jobs ps no longer lists, logs only removals, and salty_offload gains --reap-only so staging gets a background cleaner without archive/export. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
281 lines
15 KiB
Python
281 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Move finished rollouts to Salty personal space and build the datasets there; the controller keeps
|
|
only small files and the final datasets.
|
|
|
|
Every 10 minutes:
|
|
1. archive: a rollout that ended over 30 min ago (UNVERIFIED: 24 h, it can still be re-verified), whose
|
|
execution is TERMINAL and, if PASS, whose content review is settled, goes to
|
|
personal:/bench-root/archive/<run>/<job>/<kk>.tgz. Stage products already in personal:/bench-root/runs
|
|
with the same name and size are left out. Later changes go to <kk>.u<time>.tgz overlays; an archive is
|
|
never rewritten. Then every local file over 1 MB is deleted, and so is its compile scratch in work_root.
|
|
2. export: at most hourly and only after new archives, one Salty job unpacks the archives into personal
|
|
runs and runs export/to_agentrl.py there (salty_remote_worker.export); the datasets stay in
|
|
personal:/bench-root/datasets/ and are copied to bench/datasets-final/.
|
|
The collectors keep their own per-rollout export into bench/datasets/: it finds and reviews new PASS
|
|
rollouts while their files are still here, which is what step 1 waits for.
|
|
|
|
Usage: salty_offload.py RUN=CONFIG [RUN=CONFIG ...] [--once] [--export-now]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
PRODUCTS = (".hbm", ".bc", ".npz", ".onnx")
|
|
BIG = 1 << 20
|
|
|
|
|
|
def log(message):
|
|
print(f"{time.strftime('%F %T')} {message}", flush=True)
|
|
|
|
|
|
def remote_sizes(cli, remote_dir):
|
|
rc, out, _ = cli.once(["storage", "ls", remote_dir])
|
|
rows = (line.split("\t") for line in out.splitlines()[1:]) if rc == 0 else ()
|
|
return {f[0]: int(f[2]) for f in rows if len(f) > 2 and f[1] == "文件"}
|
|
|
|
|
|
def settled(rd, reasons):
|
|
"""PASS leaves once the collector's export has judged it: a finished review, or excluded for another reason."""
|
|
for receipt in (rd / "content-reviews").glob("*.json"):
|
|
with contextlib.suppress(ValueError, OSError):
|
|
if json.loads(receipt.read_text()).get("state") == "COMPLETE":
|
|
return True
|
|
return reasons is not None and not any(r.endswith("content-review-unreviewed") for r in reasons)
|
|
|
|
|
|
def archive(cli, paths, rd):
|
|
"""Upload what changed since the last archive of this rollout, then free its local bulk."""
|
|
marker = rd / "ARCHIVED.json"
|
|
info = json.loads(marker.read_text()) if marker.exists() else {}
|
|
started = time.time()
|
|
files = [p for p in rd.rglob("*") if p.is_file() and not p.is_symlink() and p != marker]
|
|
changed = [p for p in files if p.stat().st_mtime > info.get("at", 0)]
|
|
if changed:
|
|
mirrored, listings = set(), {}
|
|
for p in changed:
|
|
if p.suffix in PRODUCTS and p.stat().st_size > BIG:
|
|
d = str(p.parent.relative_to(paths.RUNS))
|
|
listings.setdefault(d, remote_sizes(cli, f"personal:/bench-root/runs/{d}"))
|
|
if listings[d].get(p.name) == p.stat().st_size:
|
|
mirrored.add(p)
|
|
run, job, kk = rd.parts[-4], rd.parts[-3], rd.name
|
|
name = f"{kk}.tgz" if not info else f"{kk}.u{int(started)}.tgz"
|
|
remote = f"personal:/bench-root/archive/{run}/{job}/{name}"
|
|
with tempfile.TemporaryDirectory(prefix="salty-archive-") as tmp:
|
|
tgz = Path(tmp) / name
|
|
with tarfile.open(tgz, "w:gz") as tar:
|
|
for p in changed:
|
|
if p not in mirrored:
|
|
tar.add(p, str(p.relative_to(paths.RUNS)))
|
|
cli.call("storage", "mkdir", "-p", remote.rsplit("/", 1)[0])
|
|
cli.call("storage", "cp", str(tgz), remote, timeout=3600)
|
|
size = tgz.stat().st_size
|
|
if remote_sizes(cli, remote.rsplit("/", 1)[0]).get(name) != size:
|
|
raise RuntimeError(f"uploaded {name} size differs")
|
|
entry = {"archive": remote, "bytes": size, "sha256": hashlib.sha256(tgz.read_bytes()).hexdigest(),
|
|
"mirrored": sorted(str(p.relative_to(paths.RUNS)) for p in mirrored)}
|
|
info = {**entry, **info, "at": started}
|
|
if name != f"{kk}.tgz":
|
|
info["overlays"] = [*info.get("overlays", []), entry]
|
|
marker.write_text(json.dumps(info, ensure_ascii=False, indent=1))
|
|
freed = 0
|
|
for p in files:
|
|
if p.exists() and p.stat().st_size > BIG:
|
|
freed += p.stat().st_size
|
|
p.unlink()
|
|
# Its compile scratch (hb_compile working_dir, only 05-compile/06-hbm read it) on the VM's own disk:
|
|
# 10-18 742 finished rollouts left 36 GB there and the Mac mini disk filled.
|
|
shutil.rmtree(paths.expand(paths.config()["paths"]["work_root"]) / rd.parts[-4] / rd.parts[-3] / f"r{int(rd.name):02d}",
|
|
ignore_errors=True)
|
|
return len(changed), freed
|
|
|
|
|
|
def archive_all(cli, paths, runs, terminal):
|
|
con = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True)
|
|
rows = con.execute(f"SELECT run_id, job_id, kk, state, finished_at FROM rollouts WHERE run_id IN "
|
|
f"({','.join('?' * len(runs))})", tuple(runs)).fetchall()
|
|
con.close()
|
|
reasons = {}
|
|
for run in runs:
|
|
with contextlib.suppress(OSError):
|
|
for line in (paths.DATASETS / f"{run}-training/disposition.jsonl").read_text().splitlines():
|
|
item = json.loads(line)
|
|
reasons[(run, item["job_id"], item["rollout"])] = item["reasons"]
|
|
uploaded = 0
|
|
for run, job, kk, state, finished in rows:
|
|
rd = paths.RUNS / run / job / "rollouts" / f"{kk:02d}"
|
|
delay = 86400 if state == "UNVERIFIED" else 1800
|
|
if state not in terminal or not finished or finished > time.time() - delay or not rd.is_dir():
|
|
continue
|
|
marker = rd / "ARCHIVED.json"
|
|
at = json.loads(marker.read_text())["at"] if marker.exists() else None
|
|
if at is not None and all(p.stat().st_size <= BIG and p.stat().st_mtime <= at for p in rd.rglob("*")
|
|
if p.is_file() and not p.is_symlink() and p != marker):
|
|
continue # archived, freed and unchanged
|
|
execution = rd / "execution.json"
|
|
if execution.exists() and json.loads(execution.read_text()).get("execution_state") != "TERMINAL":
|
|
continue
|
|
if state == "PASS" and not settled(rd, reasons.get((run, job, kk))):
|
|
continue
|
|
try:
|
|
changed, freed = archive(cli, paths, rd)
|
|
uploaded += bool(changed)
|
|
log(f"{run} {job} {kk}: archived {changed} files, freed {freed >> 20}MB")
|
|
except Exception as exc: # next pass retries; nothing local was deleted before the upload was checked
|
|
log(f"{run} {job} {kk}: SKIP {type(exc).__name__}: {exc}")
|
|
return uploaded
|
|
|
|
|
|
def reap_orphans(cli, paths, sr, runs):
|
|
"""A Salty job whose controller attempt already ended (phase deadline, cancel, crash) has no reader:
|
|
stop it, and once it has ended remove it and its dir like salty_remote removes its own jobs (their logs and
|
|
describe are already in the attempt's stderr.log). 10-13: 12 such jobs held ~64 cores while ours queued."""
|
|
rc, out, _ = cli.once(["ps"])
|
|
if rc:
|
|
return
|
|
status = {f[0]: f[2] for f in (line.split("\t") for line in out.splitlines()[1:]) if len(f) > 2}
|
|
quiet = SimpleNamespace(once=cli.once, log=lambda message: None)
|
|
for run in runs:
|
|
attempts = (paths.RUNS / run).glob("*/rollouts/*/stages/*/a*/salty/remote.json")
|
|
# bench dev commands (harness/salty_dev.py): ended when their ticket is DONE; nothing to keep for forensics.
|
|
dev = (paths.RUNS / run).glob("*/rollouts/*/stages/_queue/terminal-*/salty/remote.json")
|
|
for p in [*attempts, *dev]:
|
|
state = json.loads(p.read_text())
|
|
if state.get("cleaned") or not state.get("job_id"): # removed by its own run / never submitted
|
|
continue
|
|
s = status.get(state.get("job_id"))
|
|
is_dev = p.parent.parent.name.startswith("terminal-")
|
|
if is_dev:
|
|
try:
|
|
ended = json.loads((p.parent.parent / "status.json").read_text()).get("state") == "DONE"
|
|
except (OSError, ValueError):
|
|
ended = False
|
|
else:
|
|
ended = (p.parent.parent / "meta.json").exists()
|
|
if s not in ("running", "pending", "stopped", "succeeded", "failed", None) or not ended:
|
|
continue
|
|
if s in ("running", "pending"):
|
|
cli.call("stop", state["job_id"], ok_404=True) # 404: gone meanwhile, nothing to stop (10-14: 30 min of retries)
|
|
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: stop requested")
|
|
continue
|
|
# Ended, or no longer listed: its dir may still be there. Refused/failed deletes are submitted again later.
|
|
sr._cleanup(quiet, state, p, retry_after=sr.CLEANUP_RETRY_S) # its receipt records why, every pass
|
|
if state.get("cleaned"):
|
|
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: removed")
|
|
|
|
|
|
def export(cli, paths, sr, policy, runs):
|
|
name = "bench-export-" + str(uuid.uuid4())
|
|
bench = paths.BENCH_ROOT
|
|
with tempfile.TemporaryDirectory(prefix="salty-export-") as tmp:
|
|
work = Path(tmp)
|
|
with contextlib.closing(sqlite3.connect(paths.DB_PATH)) as src, \
|
|
contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as dst:
|
|
src.backup(dst)
|
|
files, configs = {str(bench / "state/jobs.sqlite"): work / "jobs.sqlite"}, {}
|
|
with contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as con:
|
|
for run, config in runs.items():
|
|
configs[run] = str(bench / "export-configs" / f"{run}.yaml")
|
|
files[configs[run]] = Path(config)
|
|
seed = paths.DATASETS / f"{run}-training/manifest.json"
|
|
if seed.exists():
|
|
files[str(bench / "export-seed" / f"{run}.json")] = seed
|
|
for (job,) in con.execute("SELECT job_id FROM jobs WHERE run_id=?", (run,)):
|
|
if (paths.RUNS / run / job / "job.json").exists():
|
|
files[str(paths.RUNS / run / job / "job.json")] = paths.RUNS / run / job / "job.json"
|
|
request = {"identity": {"export": name}, "stage": "export", "export": configs, "env": {}, "links": {},
|
|
"files": {destination: f"files/{i}" for i, destination in enumerate(files)}}
|
|
data = json.dumps(request).encode()
|
|
with tarfile.open(work / "bundle.tar", "w:gz") as tar: # the worker opens it with tarfile's autodetect
|
|
info = tarfile.TarInfo("request.json")
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
for i, source in enumerate(files.values()):
|
|
tar.add(source, f"files/{i}")
|
|
for d in (*sr.CODE_DIRS, "agent"):
|
|
tar.add(bench / d, f"code/{d}", filter=lambda m: None if "__pycache__" in m.name else m)
|
|
shutil.copyfile(sr.WORKER, work / "worker.py")
|
|
state = {"name": name, "remote": "personal:/" + name}
|
|
state["job_id"] = sr._submit(cli, state, policy, SimpleNamespace(name="export"), work,
|
|
policy.get("export_memory", "32Gi"))
|
|
log(f"export job {state['job_id']} submitted")
|
|
job_state = sr._wait(cli, state["job_id"], policy, time.time(), time.time, time.sleep)
|
|
if job_state != "succeeded":
|
|
sr._remote_logs(policy, state["job_id"], work / "remote.log")
|
|
log(f"export job {job_state}, kept for forensics:\n{(work / 'remote.log').read_text()[-4000:]}")
|
|
return False
|
|
sr._fetch(cli, state["remote"] + "/result/result.json", work / "result.json")
|
|
sr._fetch(cli, state["remote"] + "/result/datasets.tar.gz", work / "datasets.tar.gz")
|
|
codes = json.loads((work / "result.json").read_text())["exit_codes"]
|
|
with tarfile.open(work / "datasets.tar.gz") as tar:
|
|
tar.extractall(work / "out", members=[m for m in tar.getmembers() if m.isfile() and ".." not in m.name
|
|
and not m.name.startswith("/")])
|
|
final = bench / "datasets-final"
|
|
final.mkdir(exist_ok=True)
|
|
for run, rc in codes.items():
|
|
if rc != 0:
|
|
log(f"export of {run} failed (exit {rc}); its last good dataset stays")
|
|
continue
|
|
target, old = final / f"{run}-training", final / f".{run}-training.old"
|
|
shutil.rmtree(old, ignore_errors=True)
|
|
if target.exists():
|
|
target.rename(old)
|
|
(work / "out" / f"{run}-training").rename(target)
|
|
shutil.rmtree(old, ignore_errors=True)
|
|
manifest = json.loads((target / "manifest.json").read_text())
|
|
log(f"{run}: sft {manifest['stats']['sft_eligible']} of {manifest['stats']['scanned']} scanned")
|
|
sr._cleanup(cli, state, paths.STATE / "salty-export" / f"{name}.json")
|
|
return all(rc == 0 for rc in codes.values())
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("runs", nargs="+", metavar="RUN=CONFIG")
|
|
ap.add_argument("--once", action="store_true", help="one pass, then exit")
|
|
ap.add_argument("--export-now", action="store_true", help="export on the first pass even if nothing new was archived")
|
|
ap.add_argument("--reap-only", action="store_true", help="only stop/remove ended attempts' Salty jobs and dirs (staging)")
|
|
a = ap.parse_args()
|
|
runs = dict(item.split("=", 1) for item in a.runs)
|
|
os.environ["BENCH_CONFIG"] = next(iter(runs.values())) # salty CLI and image come from the first config
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from harness import paths, salty_remote as sr
|
|
from export.to_agentrl import DB_TERMINAL_STATES
|
|
policy = dict(paths.config()["execution"]["salty_remote"])
|
|
if "@sha256:" not in policy.get("image", ""):
|
|
raise SystemExit("execution.salty_remote.image must be pinned by digest")
|
|
cli = sr._Cli(policy.get("salty", "salty"), policy.get("outage_seconds", 1800), time.sleep, time.time, log)
|
|
pending, last_export = a.export_now, 0.0
|
|
while True:
|
|
try:
|
|
reap_orphans(cli, paths, sr, runs)
|
|
except Exception as exc: # retried next pass
|
|
log(f"reap failed: {type(exc).__name__}: {exc}")
|
|
pending = not a.reap_only and archive_all(cli, paths, runs, DB_TERMINAL_STATES) > 0 or pending
|
|
if pending and not a.reap_only and time.time() - last_export >= 3600:
|
|
last_export = time.time()
|
|
try:
|
|
pending = not export(cli, paths, sr, policy, runs)
|
|
except Exception as exc: # retried next hour; the last good datasets stay in place
|
|
log(f"export failed: {type(exc).__name__}: {exc}")
|
|
if a.once:
|
|
return 0
|
|
time.sleep(600)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|