data-gen-bpu/scripts/salty_offload.py
bench ad51e86f5f 导出进程每轮停掉本地已结束的 Salty 作业,停好后清理
阶段时限只管超时;轨迹被取消、进程崩溃时,远端容器照样跑完,没人取结果。
10-13 晚有 12 个这样的作业占着约 64 核,同时我们自己的作业在排队。
现在每轮先查 salty ps:本地这一步已经写了 meta.json、远端还在 running/pending 的,
就停掉;已经 stopped 的,按 salty_remote 处理自己停掉的作业那样删除作业和临时目录。
failed 的照旧保留,供排查。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 22:24:47 +08:00

260 lines
13 KiB
Python

#!/usr/bin/env python3
"""Move finished rollouts to Salty personal space and build the datasets there; the controller keeps
only small files and the final datasets.
Every 10 minutes:
1. archive: a rollout that ended over 30 min ago (UNVERIFIED: 24 h, it can still be re-verified), whose
execution is TERMINAL and, if PASS, whose content review is settled, goes to
personal:/bench-root/archive/<run>/<job>/<kk>.tgz. Stage products already in personal:/bench-root/runs
with the same name and size are left out. Later changes go to <kk>.u<time>.tgz overlays; an archive is
never rewritten. Then every local file over 1 MB is deleted.
2. export: at most hourly and only after new archives, one Salty job unpacks the archives into personal
runs and runs export/to_agentrl.py there (salty_remote_worker.export); the datasets stay in
personal:/bench-root/datasets/ and are copied to bench/datasets-final/.
The collectors keep their own per-rollout export into bench/datasets/: it finds and reviews new PASS
rollouts while their files are still here, which is what step 1 waits for.
Usage: salty_offload.py RUN=CONFIG [RUN=CONFIG ...] [--once] [--export-now]
"""
from __future__ import annotations
import argparse
import contextlib
import hashlib
import io
import json
import os
import shutil
import sqlite3
import sys
import tarfile
import tempfile
import time
import uuid
from pathlib import Path
from types import SimpleNamespace
PRODUCTS = (".hbm", ".bc", ".npz", ".onnx")
BIG = 1 << 20
def log(message):
print(f"{time.strftime('%F %T')} {message}", flush=True)
def remote_sizes(cli, remote_dir):
rc, out, _ = cli.once(["storage", "ls", remote_dir])
rows = (line.split("\t") for line in out.splitlines()[1:]) if rc == 0 else ()
return {f[0]: int(f[2]) for f in rows if len(f) > 2 and f[1] == "文件"}
def settled(rd, reasons):
"""PASS leaves once the collector's export has judged it: a finished review, or excluded for another reason."""
for receipt in (rd / "content-reviews").glob("*.json"):
with contextlib.suppress(ValueError, OSError):
if json.loads(receipt.read_text()).get("state") == "COMPLETE":
return True
return reasons is not None and not any(r.endswith("content-review-unreviewed") for r in reasons)
def archive(cli, paths, rd):
"""Upload what changed since the last archive of this rollout, then free its local bulk."""
marker = rd / "ARCHIVED.json"
info = json.loads(marker.read_text()) if marker.exists() else {}
started = time.time()
files = [p for p in rd.rglob("*") if p.is_file() and not p.is_symlink() and p != marker]
changed = [p for p in files if p.stat().st_mtime > info.get("at", 0)]
if changed:
mirrored, listings = set(), {}
for p in changed:
if p.suffix in PRODUCTS and p.stat().st_size > BIG:
d = str(p.parent.relative_to(paths.RUNS))
listings.setdefault(d, remote_sizes(cli, f"personal:/bench-root/runs/{d}"))
if listings[d].get(p.name) == p.stat().st_size:
mirrored.add(p)
run, job, kk = rd.parts[-4], rd.parts[-3], rd.name
name = f"{kk}.tgz" if not info else f"{kk}.u{int(started)}.tgz"
remote = f"personal:/bench-root/archive/{run}/{job}/{name}"
with tempfile.TemporaryDirectory(prefix="salty-archive-") as tmp:
tgz = Path(tmp) / name
with tarfile.open(tgz, "w:gz") as tar:
for p in changed:
if p not in mirrored:
tar.add(p, str(p.relative_to(paths.RUNS)))
cli.call("storage", "mkdir", "-p", remote.rsplit("/", 1)[0])
cli.call("storage", "cp", str(tgz), remote, timeout=3600)
size = tgz.stat().st_size
if remote_sizes(cli, remote.rsplit("/", 1)[0]).get(name) != size:
raise RuntimeError(f"uploaded {name} size differs")
entry = {"archive": remote, "bytes": size, "sha256": hashlib.sha256(tgz.read_bytes()).hexdigest(),
"mirrored": sorted(str(p.relative_to(paths.RUNS)) for p in mirrored)}
info = {**entry, **info, "at": started}
if name != f"{kk}.tgz":
info["overlays"] = [*info.get("overlays", []), entry]
marker.write_text(json.dumps(info, ensure_ascii=False, indent=1))
freed = 0
for p in files:
if p.exists() and p.stat().st_size > BIG:
freed += p.stat().st_size
p.unlink()
return len(changed), freed
def archive_all(cli, paths, runs, terminal):
con = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True)
rows = con.execute(f"SELECT run_id, job_id, kk, state, finished_at FROM rollouts WHERE run_id IN "
f"({','.join('?' * len(runs))})", tuple(runs)).fetchall()
con.close()
reasons = {}
for run in runs:
with contextlib.suppress(OSError):
for line in (paths.DATASETS / f"{run}-training/disposition.jsonl").read_text().splitlines():
item = json.loads(line)
reasons[(run, item["job_id"], item["rollout"])] = item["reasons"]
uploaded = 0
for run, job, kk, state, finished in rows:
rd = paths.RUNS / run / job / "rollouts" / f"{kk:02d}"
delay = 86400 if state == "UNVERIFIED" else 1800
if state not in terminal or not finished or finished > time.time() - delay or not rd.is_dir():
continue
marker = rd / "ARCHIVED.json"
at = json.loads(marker.read_text())["at"] if marker.exists() else None
if at is not None and all(p.stat().st_size <= BIG and p.stat().st_mtime <= at for p in rd.rglob("*")
if p.is_file() and not p.is_symlink() and p != marker):
continue # archived, freed and unchanged
execution = rd / "execution.json"
if execution.exists() and json.loads(execution.read_text()).get("execution_state") != "TERMINAL":
continue
if state == "PASS" and not settled(rd, reasons.get((run, job, kk))):
continue
try:
changed, freed = archive(cli, paths, rd)
uploaded += bool(changed)
log(f"{run} {job} {kk}: archived {changed} files, freed {freed >> 20}MB")
except Exception as exc: # next pass retries; nothing local was deleted before the upload was checked
log(f"{run} {job} {kk}: SKIP {type(exc).__name__}: {exc}")
return uploaded
def reap_orphans(cli, paths, sr, runs):
"""A Salty job whose controller attempt already ended (phase deadline, cancel, crash) has no reader:
stop it, and once stopped remove it like salty_remote removes its own stopped jobs. Failed jobs stay
for forensics. 10-13: 12 such jobs held ~64 cores while ours queued."""
rc, out, _ = cli.once(["ps"])
if rc:
return
status = {f[0]: f[2] for f in (line.split("\t") for line in out.splitlines()[1:]) if len(f) > 2}
for run in runs:
for p in (paths.RUNS / run).glob("*/rollouts/*/stages/*/a*/salty/remote.json"):
state = json.loads(p.read_text())
s = status.get(state.get("job_id"))
if s not in ("running", "pending", "stopped") or not (p.parent.parent / "meta.json").exists():
continue
if s == "stopped":
sr._cleanup(cli, state)
else:
cli.call("stop", state["job_id"])
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: "
+ ("removed" if s == "stopped" else "stop requested"))
def export(cli, paths, sr, policy, runs):
name = "bench-export-" + str(uuid.uuid4())
bench = paths.BENCH_ROOT
with tempfile.TemporaryDirectory(prefix="salty-export-") as tmp:
work = Path(tmp)
with contextlib.closing(sqlite3.connect(paths.DB_PATH)) as src, \
contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as dst:
src.backup(dst)
files, configs = {str(bench / "state/jobs.sqlite"): work / "jobs.sqlite"}, {}
with contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as con:
for run, config in runs.items():
configs[run] = str(bench / "export-configs" / f"{run}.yaml")
files[configs[run]] = Path(config)
seed = paths.DATASETS / f"{run}-training/manifest.json"
if seed.exists():
files[str(bench / "export-seed" / f"{run}.json")] = seed
for (job,) in con.execute("SELECT job_id FROM jobs WHERE run_id=?", (run,)):
if (paths.RUNS / run / job / "job.json").exists():
files[str(paths.RUNS / run / job / "job.json")] = paths.RUNS / run / job / "job.json"
request = {"identity": {"export": name}, "stage": "export", "export": configs, "env": {}, "links": {},
"files": {destination: f"files/{i}" for i, destination in enumerate(files)}}
data = json.dumps(request).encode()
with tarfile.open(work / "bundle.tar", "w:gz") as tar: # the worker opens it with tarfile's autodetect
info = tarfile.TarInfo("request.json")
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
for i, source in enumerate(files.values()):
tar.add(source, f"files/{i}")
for d in (*sr.CODE_DIRS, "agent"):
tar.add(bench / d, f"code/{d}", filter=lambda m: None if "__pycache__" in m.name else m)
shutil.copyfile(sr.WORKER, work / "worker.py")
state = {"name": name, "remote": "personal:/" + name}
state["job_id"] = sr._submit(cli, state, policy, SimpleNamespace(name="export"), work,
policy.get("export_memory", "32Gi"))
log(f"export job {state['job_id']} submitted")
job_state = sr._wait(cli, state["job_id"], policy, time.time(), time.time, time.sleep)
if job_state != "succeeded":
sr._remote_logs(policy, state["job_id"], work / "remote.log")
log(f"export job {job_state}, kept for forensics:\n{(work / 'remote.log').read_text()[-4000:]}")
return False
sr._fetch(cli, state["remote"] + "/result/result.json", work / "result.json")
sr._fetch(cli, state["remote"] + "/result/datasets.tar.gz", work / "datasets.tar.gz")
codes = json.loads((work / "result.json").read_text())["exit_codes"]
with tarfile.open(work / "datasets.tar.gz") as tar:
tar.extractall(work / "out", members=[m for m in tar.getmembers() if m.isfile() and ".." not in m.name
and not m.name.startswith("/")])
final = bench / "datasets-final"
final.mkdir(exist_ok=True)
for run, rc in codes.items():
if rc != 0:
log(f"export of {run} failed (exit {rc}); its last good dataset stays")
continue
target, old = final / f"{run}-training", final / f".{run}-training.old"
shutil.rmtree(old, ignore_errors=True)
if target.exists():
target.rename(old)
(work / "out" / f"{run}-training").rename(target)
shutil.rmtree(old, ignore_errors=True)
manifest = json.loads((target / "manifest.json").read_text())
log(f"{run}: sft {manifest['stats']['sft_eligible']} of {manifest['stats']['scanned']} scanned")
sr._cleanup(cli, state)
return all(rc == 0 for rc in codes.values())
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("runs", nargs="+", metavar="RUN=CONFIG")
ap.add_argument("--once", action="store_true", help="one pass, then exit")
ap.add_argument("--export-now", action="store_true", help="export on the first pass even if nothing new was archived")
a = ap.parse_args()
runs = dict(item.split("=", 1) for item in a.runs)
os.environ["BENCH_CONFIG"] = next(iter(runs.values())) # salty CLI and image come from the first config
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from harness import paths, salty_remote as sr
from export.to_agentrl import DB_TERMINAL_STATES
policy = dict(paths.config()["execution"]["salty_remote"])
if "@sha256:" not in policy.get("image", ""):
raise SystemExit("execution.salty_remote.image must be pinned by digest")
cli = sr._Cli(policy.get("salty", "salty"), policy.get("outage_seconds", 1800), time.sleep, time.time, log)
pending, last_export = a.export_now, 0.0
while True:
try:
reap_orphans(cli, paths, sr, runs)
except Exception as exc: # retried next pass
log(f"reap failed: {type(exc).__name__}: {exc}")
pending = archive_all(cli, paths, runs, DB_TERMINAL_STATES) > 0 or pending
if pending and time.time() - last_export >= 3600:
last_export = time.time()
try:
pending = not export(cli, paths, sr, policy, runs)
except Exception as exc: # retried next hour; the last good datasets stay in place
log(f"export failed: {type(exc).__name__}: {exc}")
if a.once:
return 0
time.sleep(600)
if __name__ == "__main__":
sys.exit(main())