The 10-21 03:16 Mac mini reboot left 7 GLM rollouts (05: densenet201,
efficientnet.b2, rtmo.s; 06: rtmpose.x, rtmo.m, hypersim.base, yolo11n.seg)
quarantined OPERATION_STATE_UNKNOWN while still counted RUNNING: each had a
05-compile in flight, its reaper died with the VM before writing a receipt, and
_validate_recovered_receipt demanded one ('scope cleanup unconfirmed') although
scope_cleaned already accepts an earlier boot. Without a receipt an earlier-boot
scope now proves cleanup for a failed result and for the inline 04 whose meta is
hashed; any other success still needs its receipt.
speed_ref: stages/ref of an interrupted run kept bundle.tar, which salty_dev
creates exclusively, so the 3 in-flight references failed with FileExistsError
every 2 minutes and took all 3 workers. The leftover dir is renamed aside.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
161 lines
8.8 KiB
Python
161 lines
8.8 KiB
Python
"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
|
||
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
|
||
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
|
||
|
||
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
|
||
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
|
||
|
||
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3]
|
||
|
||
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
|
||
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import shlex
|
||
import sys
|
||
import time
|
||
import traceback
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
import yaml
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||
from harness import paths, salty_dev # noqa: E402
|
||
from harness.backends import ptq # noqa: E402
|
||
from harness.stages.s01_fetch import weights_dir # noqa: E402
|
||
from harness.stages.s03_export import onnx_path # noqa: E402
|
||
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
|
||
from harness.stagerun import own_inputs_required # noqa: E402
|
||
|
||
DEADLINE_S = 10 * 3600
|
||
|
||
|
||
def command(job: dict, ws: Path) -> str:
|
||
onnx = onnx_path(job)
|
||
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
|
||
if onnx is None or weights is None:
|
||
raise ValueError("weights not fetched yet")
|
||
mine = ws / "model.onnx"
|
||
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
|
||
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
|
||
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
|
||
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
|
||
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
|
||
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
|
||
f"else {export} > export.log 2>&1; fi; "
|
||
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
|
||
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
|
||
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
|
||
|
||
|
||
OUT = paths.AGENT / "speed_ref"
|
||
|
||
|
||
def one(job_dir: Path) -> str:
|
||
job = json.loads((job_dir / "job.json").read_text())
|
||
jid = job["job_id"]
|
||
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
|
||
ws = rd / "workspace"
|
||
ws.mkdir(parents=True, exist_ok=True)
|
||
(rd / "stages").mkdir(exist_ok=True)
|
||
(rd.parent.parent / "job.json").write_text(json.dumps(job))
|
||
hd = rd / "stages" / "ref"
|
||
if hd.exists():
|
||
# Left by a process that died mid-compile (salty_dev creates bundle.tar exclusively): every retry failed with
|
||
# FileExistsError and the 3 workers spun on those jobs (10-21 reboot). Kept aside: salty/remote.json names its job.
|
||
hd.rename(hd.with_name(f"ref.interrupted-{int(time.time())}"))
|
||
hd.mkdir()
|
||
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
|
||
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
|
||
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log")
|
||
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
|
||
perf = _parse_static_perf(info)
|
||
if result.state != "OK" or "latency_ms_est" not in perf:
|
||
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
|
||
unavailable(job, f"{DEADLINE_S // 3600} 小时内没编完" if result.exit_code == 124 else f"编译失败,退出码 {result.exit_code}")
|
||
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
|
||
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
|
||
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
|
||
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
|
||
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
|
||
publish(jid, ref)
|
||
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
|
||
|
||
|
||
def publish(jid: str, ref: dict) -> None:
|
||
OUT.mkdir(exist_ok=True)
|
||
tmp = OUT / f".{jid}.json.tmp"
|
||
tmp.write_text(json.dumps(ref, indent=2, ensure_ascii=False))
|
||
tmp.replace(OUT / f"{jid}.json")
|
||
|
||
|
||
def unavailable(job: dict, why: str) -> None:
|
||
"""The default configuration did not compile here (10-16: big S600 depth models, >10 h on 6 cores): say so,
|
||
so agents stop waiting for the file or compiling the default themselves."""
|
||
publish(job["job_id"], {"schema": "bench.speed_ref/1", "job_id": job["job_id"], "march": job["march"], "qcfg": job["qcfg"],
|
||
"latency_ms_est": None, "status": "unavailable",
|
||
"note": "平台默认配置在这道题上没编出来(" + why + "),本题没有速度参考值:不用再等,也不要自己编默认配置;"
|
||
"只要别整模型提到 int16/fp16、别让大段算子落到 CPU,并用 board 阶段实测确认。",
|
||
"computed_at": time.time()})
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("runs", nargs="+")
|
||
ap.add_argument("--workers", type=int, default=3)
|
||
ap.add_argument("--cpu", default="6")
|
||
ap.add_argument("--only", action="append", help="job ids")
|
||
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
|
||
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
|
||
ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time")
|
||
a = ap.parse_args()
|
||
cfg = paths.config()
|
||
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
|
||
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
|
||
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
|
||
|
||
def pending() -> tuple[list[str], dict[str, Path]]:
|
||
jobs: dict[str, Path] = {}
|
||
started: dict[str, int] = {}
|
||
for run in a.runs:
|
||
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
|
||
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
|
||
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
|
||
jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run
|
||
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
|
||
for j in jobs: # failures recorded before the unavailable file existed (or by an older process)
|
||
if (failed / j / "FAILED").exists() and not (OUT / f"{j}.json").exists():
|
||
unavailable(json.loads((jobs[j] / "job.json").read_text()), "编译失败或超时")
|
||
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip),
|
||
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
|
||
return todo, jobs
|
||
|
||
def safe(jid, jd):
|
||
try:
|
||
line = one(jd)
|
||
except Exception as exc:
|
||
line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
|
||
print(time.strftime("%F %T"), line, flush=True)
|
||
|
||
running: dict[str, object] = {}
|
||
with ThreadPoolExecutor(a.workers) as pool:
|
||
while True:
|
||
running = {j: f for j, f in running.items() if not f.done()}
|
||
todo, jobs = pending()
|
||
for jid in [j for j in todo if j not in running][:a.workers - len(running)]:
|
||
print(time.strftime("%F %T"), "start", jid, flush=True)
|
||
running[jid] = pool.submit(safe, jid, jobs[jid])
|
||
if not a.follow and not running:
|
||
break
|
||
time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|