10-14 重启后续跑的轨迹自己删掉了作业,ps 仍短暂列出,清理去 stop 得到 404,按网络故障重试 30 分钟,期间归档和导出都停着。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
273 lines
14 KiB
Python
273 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Move finished rollouts to Salty personal space and build the datasets there; the controller keeps
|
|
only small files and the final datasets.
|
|
|
|
Every 10 minutes:
|
|
1. archive: a rollout that ended over 30 min ago (UNVERIFIED: 24 h, it can still be re-verified), whose
|
|
execution is TERMINAL and, if PASS, whose content review is settled, goes to
|
|
personal:/bench-root/archive/<run>/<job>/<kk>.tgz. Stage products already in personal:/bench-root/runs
|
|
with the same name and size are left out. Later changes go to <kk>.u<time>.tgz overlays; an archive is
|
|
never rewritten. Then every local file over 1 MB is deleted.
|
|
2. export: at most hourly and only after new archives, one Salty job unpacks the archives into personal
|
|
runs and runs export/to_agentrl.py there (salty_remote_worker.export); the datasets stay in
|
|
personal:/bench-root/datasets/ and are copied to bench/datasets-final/.
|
|
The collectors keep their own per-rollout export into bench/datasets/: it finds and reviews new PASS
|
|
rollouts while their files are still here, which is what step 1 waits for.
|
|
|
|
Usage: salty_offload.py RUN=CONFIG [RUN=CONFIG ...] [--once] [--export-now]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
PRODUCTS = (".hbm", ".bc", ".npz", ".onnx")
|
|
BIG = 1 << 20
|
|
|
|
|
|
def log(message):
|
|
print(f"{time.strftime('%F %T')} {message}", flush=True)
|
|
|
|
|
|
def remote_sizes(cli, remote_dir):
|
|
rc, out, _ = cli.once(["storage", "ls", remote_dir])
|
|
rows = (line.split("\t") for line in out.splitlines()[1:]) if rc == 0 else ()
|
|
return {f[0]: int(f[2]) for f in rows if len(f) > 2 and f[1] == "文件"}
|
|
|
|
|
|
def settled(rd, reasons):
|
|
"""PASS leaves once the collector's export has judged it: a finished review, or excluded for another reason."""
|
|
for receipt in (rd / "content-reviews").glob("*.json"):
|
|
with contextlib.suppress(ValueError, OSError):
|
|
if json.loads(receipt.read_text()).get("state") == "COMPLETE":
|
|
return True
|
|
return reasons is not None and not any(r.endswith("content-review-unreviewed") for r in reasons)
|
|
|
|
|
|
def archive(cli, paths, rd):
|
|
"""Upload what changed since the last archive of this rollout, then free its local bulk."""
|
|
marker = rd / "ARCHIVED.json"
|
|
info = json.loads(marker.read_text()) if marker.exists() else {}
|
|
started = time.time()
|
|
files = [p for p in rd.rglob("*") if p.is_file() and not p.is_symlink() and p != marker]
|
|
changed = [p for p in files if p.stat().st_mtime > info.get("at", 0)]
|
|
if changed:
|
|
mirrored, listings = set(), {}
|
|
for p in changed:
|
|
if p.suffix in PRODUCTS and p.stat().st_size > BIG:
|
|
d = str(p.parent.relative_to(paths.RUNS))
|
|
listings.setdefault(d, remote_sizes(cli, f"personal:/bench-root/runs/{d}"))
|
|
if listings[d].get(p.name) == p.stat().st_size:
|
|
mirrored.add(p)
|
|
run, job, kk = rd.parts[-4], rd.parts[-3], rd.name
|
|
name = f"{kk}.tgz" if not info else f"{kk}.u{int(started)}.tgz"
|
|
remote = f"personal:/bench-root/archive/{run}/{job}/{name}"
|
|
with tempfile.TemporaryDirectory(prefix="salty-archive-") as tmp:
|
|
tgz = Path(tmp) / name
|
|
with tarfile.open(tgz, "w:gz") as tar:
|
|
for p in changed:
|
|
if p not in mirrored:
|
|
tar.add(p, str(p.relative_to(paths.RUNS)))
|
|
cli.call("storage", "mkdir", "-p", remote.rsplit("/", 1)[0])
|
|
cli.call("storage", "cp", str(tgz), remote, timeout=3600)
|
|
size = tgz.stat().st_size
|
|
if remote_sizes(cli, remote.rsplit("/", 1)[0]).get(name) != size:
|
|
raise RuntimeError(f"uploaded {name} size differs")
|
|
entry = {"archive": remote, "bytes": size, "sha256": hashlib.sha256(tgz.read_bytes()).hexdigest(),
|
|
"mirrored": sorted(str(p.relative_to(paths.RUNS)) for p in mirrored)}
|
|
info = {**entry, **info, "at": started}
|
|
if name != f"{kk}.tgz":
|
|
info["overlays"] = [*info.get("overlays", []), entry]
|
|
marker.write_text(json.dumps(info, ensure_ascii=False, indent=1))
|
|
freed = 0
|
|
for p in files:
|
|
if p.exists() and p.stat().st_size > BIG:
|
|
freed += p.stat().st_size
|
|
p.unlink()
|
|
return len(changed), freed
|
|
|
|
|
|
def archive_all(cli, paths, runs, terminal):
|
|
con = sqlite3.connect(f"file:{paths.DB_PATH}?mode=ro", uri=True)
|
|
rows = con.execute(f"SELECT run_id, job_id, kk, state, finished_at FROM rollouts WHERE run_id IN "
|
|
f"({','.join('?' * len(runs))})", tuple(runs)).fetchall()
|
|
con.close()
|
|
reasons = {}
|
|
for run in runs:
|
|
with contextlib.suppress(OSError):
|
|
for line in (paths.DATASETS / f"{run}-training/disposition.jsonl").read_text().splitlines():
|
|
item = json.loads(line)
|
|
reasons[(run, item["job_id"], item["rollout"])] = item["reasons"]
|
|
uploaded = 0
|
|
for run, job, kk, state, finished in rows:
|
|
rd = paths.RUNS / run / job / "rollouts" / f"{kk:02d}"
|
|
delay = 86400 if state == "UNVERIFIED" else 1800
|
|
if state not in terminal or not finished or finished > time.time() - delay or not rd.is_dir():
|
|
continue
|
|
marker = rd / "ARCHIVED.json"
|
|
at = json.loads(marker.read_text())["at"] if marker.exists() else None
|
|
if at is not None and all(p.stat().st_size <= BIG and p.stat().st_mtime <= at for p in rd.rglob("*")
|
|
if p.is_file() and not p.is_symlink() and p != marker):
|
|
continue # archived, freed and unchanged
|
|
execution = rd / "execution.json"
|
|
if execution.exists() and json.loads(execution.read_text()).get("execution_state") != "TERMINAL":
|
|
continue
|
|
if state == "PASS" and not settled(rd, reasons.get((run, job, kk))):
|
|
continue
|
|
try:
|
|
changed, freed = archive(cli, paths, rd)
|
|
uploaded += bool(changed)
|
|
log(f"{run} {job} {kk}: archived {changed} files, freed {freed >> 20}MB")
|
|
except Exception as exc: # next pass retries; nothing local was deleted before the upload was checked
|
|
log(f"{run} {job} {kk}: SKIP {type(exc).__name__}: {exc}")
|
|
return uploaded
|
|
|
|
|
|
def reap_orphans(cli, paths, sr, runs):
|
|
"""A Salty job whose controller attempt already ended (phase deadline, cancel, crash) has no reader:
|
|
stop it, and once stopped remove it like salty_remote removes its own stopped jobs. Failed jobs stay
|
|
for forensics. 10-13: 12 such jobs held ~64 cores while ours queued."""
|
|
rc, out, _ = cli.once(["ps"])
|
|
if rc:
|
|
return
|
|
status = {f[0]: f[2] for f in (line.split("\t") for line in out.splitlines()[1:]) if len(f) > 2}
|
|
for run in runs:
|
|
attempts = (paths.RUNS / run).glob("*/rollouts/*/stages/*/a*/salty/remote.json")
|
|
# bench dev commands (harness/salty_dev.py): ended when their ticket is DONE; nothing to keep for forensics.
|
|
dev = (paths.RUNS / run).glob("*/rollouts/*/stages/_queue/terminal-*/salty/remote.json")
|
|
for p in [*attempts, *dev]:
|
|
state = json.loads(p.read_text())
|
|
if state.get("cleaned"): # its own run already removed it; ps can still list it for a moment
|
|
continue
|
|
s = status.get(state.get("job_id"))
|
|
is_dev = p.parent.parent.name.startswith("terminal-")
|
|
if is_dev:
|
|
try:
|
|
ended = json.loads((p.parent.parent / "status.json").read_text()).get("state") == "DONE"
|
|
except (OSError, ValueError):
|
|
ended = False
|
|
else:
|
|
ended = (p.parent.parent / "meta.json").exists()
|
|
if s not in ("running", "pending", "stopped", *(("succeeded", "failed") if is_dev else ())) or not ended:
|
|
continue
|
|
if s in ("stopped", "succeeded", "failed"):
|
|
sr._cleanup(cli, state)
|
|
else:
|
|
cli.call("stop", state["job_id"], ok_404=True) # 404: gone meanwhile, nothing to stop (10-14: 30 min of retries)
|
|
log(f"{state['job_id']} {s}, its attempt {p.parent.parent.relative_to(paths.RUNS)} already ended: "
|
|
+ ("removed" if s in ("stopped", "succeeded", "failed") else "stop requested"))
|
|
|
|
|
|
def export(cli, paths, sr, policy, runs):
|
|
name = "bench-export-" + str(uuid.uuid4())
|
|
bench = paths.BENCH_ROOT
|
|
with tempfile.TemporaryDirectory(prefix="salty-export-") as tmp:
|
|
work = Path(tmp)
|
|
with contextlib.closing(sqlite3.connect(paths.DB_PATH)) as src, \
|
|
contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as dst:
|
|
src.backup(dst)
|
|
files, configs = {str(bench / "state/jobs.sqlite"): work / "jobs.sqlite"}, {}
|
|
with contextlib.closing(sqlite3.connect(work / "jobs.sqlite")) as con:
|
|
for run, config in runs.items():
|
|
configs[run] = str(bench / "export-configs" / f"{run}.yaml")
|
|
files[configs[run]] = Path(config)
|
|
seed = paths.DATASETS / f"{run}-training/manifest.json"
|
|
if seed.exists():
|
|
files[str(bench / "export-seed" / f"{run}.json")] = seed
|
|
for (job,) in con.execute("SELECT job_id FROM jobs WHERE run_id=?", (run,)):
|
|
if (paths.RUNS / run / job / "job.json").exists():
|
|
files[str(paths.RUNS / run / job / "job.json")] = paths.RUNS / run / job / "job.json"
|
|
request = {"identity": {"export": name}, "stage": "export", "export": configs, "env": {}, "links": {},
|
|
"files": {destination: f"files/{i}" for i, destination in enumerate(files)}}
|
|
data = json.dumps(request).encode()
|
|
with tarfile.open(work / "bundle.tar", "w:gz") as tar: # the worker opens it with tarfile's autodetect
|
|
info = tarfile.TarInfo("request.json")
|
|
info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
for i, source in enumerate(files.values()):
|
|
tar.add(source, f"files/{i}")
|
|
for d in (*sr.CODE_DIRS, "agent"):
|
|
tar.add(bench / d, f"code/{d}", filter=lambda m: None if "__pycache__" in m.name else m)
|
|
shutil.copyfile(sr.WORKER, work / "worker.py")
|
|
state = {"name": name, "remote": "personal:/" + name}
|
|
state["job_id"] = sr._submit(cli, state, policy, SimpleNamespace(name="export"), work,
|
|
policy.get("export_memory", "32Gi"))
|
|
log(f"export job {state['job_id']} submitted")
|
|
job_state = sr._wait(cli, state["job_id"], policy, time.time(), time.time, time.sleep)
|
|
if job_state != "succeeded":
|
|
sr._remote_logs(policy, state["job_id"], work / "remote.log")
|
|
log(f"export job {job_state}, kept for forensics:\n{(work / 'remote.log').read_text()[-4000:]}")
|
|
return False
|
|
sr._fetch(cli, state["remote"] + "/result/result.json", work / "result.json")
|
|
sr._fetch(cli, state["remote"] + "/result/datasets.tar.gz", work / "datasets.tar.gz")
|
|
codes = json.loads((work / "result.json").read_text())["exit_codes"]
|
|
with tarfile.open(work / "datasets.tar.gz") as tar:
|
|
tar.extractall(work / "out", members=[m for m in tar.getmembers() if m.isfile() and ".." not in m.name
|
|
and not m.name.startswith("/")])
|
|
final = bench / "datasets-final"
|
|
final.mkdir(exist_ok=True)
|
|
for run, rc in codes.items():
|
|
if rc != 0:
|
|
log(f"export of {run} failed (exit {rc}); its last good dataset stays")
|
|
continue
|
|
target, old = final / f"{run}-training", final / f".{run}-training.old"
|
|
shutil.rmtree(old, ignore_errors=True)
|
|
if target.exists():
|
|
target.rename(old)
|
|
(work / "out" / f"{run}-training").rename(target)
|
|
shutil.rmtree(old, ignore_errors=True)
|
|
manifest = json.loads((target / "manifest.json").read_text())
|
|
log(f"{run}: sft {manifest['stats']['sft_eligible']} of {manifest['stats']['scanned']} scanned")
|
|
sr._cleanup(cli, state)
|
|
return all(rc == 0 for rc in codes.values())
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("runs", nargs="+", metavar="RUN=CONFIG")
|
|
ap.add_argument("--once", action="store_true", help="one pass, then exit")
|
|
ap.add_argument("--export-now", action="store_true", help="export on the first pass even if nothing new was archived")
|
|
a = ap.parse_args()
|
|
runs = dict(item.split("=", 1) for item in a.runs)
|
|
os.environ["BENCH_CONFIG"] = next(iter(runs.values())) # salty CLI and image come from the first config
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from harness import paths, salty_remote as sr
|
|
from export.to_agentrl import DB_TERMINAL_STATES
|
|
policy = dict(paths.config()["execution"]["salty_remote"])
|
|
if "@sha256:" not in policy.get("image", ""):
|
|
raise SystemExit("execution.salty_remote.image must be pinned by digest")
|
|
cli = sr._Cli(policy.get("salty", "salty"), policy.get("outage_seconds", 1800), time.sleep, time.time, log)
|
|
pending, last_export = a.export_now, 0.0
|
|
while True:
|
|
try:
|
|
reap_orphans(cli, paths, sr, runs)
|
|
except Exception as exc: # retried next pass
|
|
log(f"reap failed: {type(exc).__name__}: {exc}")
|
|
pending = archive_all(cli, paths, runs, DB_TERMINAL_STATES) > 0 or pending
|
|
if pending and time.time() - last_export >= 3600:
|
|
last_export = time.time()
|
|
try:
|
|
pending = not export(cli, paths, sr, policy, runs)
|
|
except Exception as exc: # retried next hour; the last good datasets stay in place
|
|
log(f"export failed: {type(exc).__name__}: {exc}")
|
|
if a.once:
|
|
return 0
|
|
time.sleep(600)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|