10-15 夜:提示词只说把 /data/calib/ 换成 /data/calib-raw/,agent 去 /data/calib-raw 找,每条花 5-8 次 dev 命令。 speed_ref 按轮跑时一轮要等最慢的那道编完,新题排不上。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
140 lines
7.3 KiB
Python
140 lines
7.3 KiB
Python
"""Speed reference per job: compile the platform's default configuration once (in a bench dev container,
|
|
outside any rollout) and write the compiler's latency estimate to agent/speed_ref/<job_id>.json: the agent's jail shows bench/agent
|
|
read-only and live (not the job directory), so a reference computed after a rollout started still reaches it.
|
|
|
|
The agent compares its own model against it (SYSTEM.md 速度要求) instead of compiling the default itself
|
|
(10-15: 9 of 32 v17 rollouts spent ~4 h each doing that). Jobs that already have the file are skipped.
|
|
|
|
BENCH_CONFIG=<a run config> python scripts/speed_ref.py r20260929-04 r20260929-05 r20260929-06 [--active --follow] [--workers 3]
|
|
|
|
--active: only jobs a v18+ rollout is on. Salty is shared with the live rollouts (10-15 20:40: their own dev
|
|
jobs queued once 8 of these were added), so references follow demand instead of compiling all 418 up front.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import shlex
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import yaml
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
from harness import paths, salty_dev # noqa: E402
|
|
from harness.backends import ptq # noqa: E402
|
|
from harness.stages.s01_fetch import weights_dir # noqa: E402
|
|
from harness.stages.s03_export import onnx_path # noqa: E402
|
|
from harness.stages.s06_hbm import _parse_static_perf # noqa: E402
|
|
from harness.stagerun import own_inputs_required # noqa: E402
|
|
|
|
DEADLINE_S = 10 * 3600
|
|
|
|
|
|
def command(job: dict, ws: Path) -> str:
|
|
onnx = onnx_path(job)
|
|
weights = weights_dir(job["model"]) # its files are on personal storage, listed in the container
|
|
if onnx is None or weights is None:
|
|
raise ValueError("weights not fetched yet")
|
|
mine = ws / "model.onnx"
|
|
calib = Path(job["data"]["calib"]["path"]) / "npy_f32"
|
|
cfg = ptq.compile_config(job, Path("model.onnx"), calib, Path("ref"), None)
|
|
(ws / "ref.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False))
|
|
export = shlex.join(ptq.export_cmd(job, mine, Path("WEIGHTS"))).replace("WEIGHTS", '"$W"')
|
|
export = f"W=$(ls -d {shlex.quote(str(weights))}/* | grep -v /meta.json | head -1); {export}"
|
|
return (f"set -e; if [ -f {shlex.quote(str(onnx))} ]; then ln -sfn {shlex.quote(str(onnx))} model.onnx; "
|
|
f"else {export} > export.log 2>&1; fi; "
|
|
"hb_compile -c ref.yaml > ref_compile.log 2>&1 || { tail -40 ref_compile.log; exit 1; }; "
|
|
"hb_model_info ref/*.hbm > ref_info.log 2>&1; " # the dev PATH's OE toolchain, as the agent uses it
|
|
"sha256sum ref/*.hbm > ref_hbm.sha256; tail -20 ref_info.log; rm -rf ref model.onnx") # keep personal storage small
|
|
|
|
|
|
OUT = paths.AGENT / "speed_ref"
|
|
|
|
|
|
def one(job_dir: Path) -> str:
|
|
job = json.loads((job_dir / "job.json").read_text())
|
|
jid = job["job_id"]
|
|
rd = paths.BENCH_ROOT / "runs" / "zz-speedref" / jid / "rollouts" / "00"
|
|
ws = rd / "workspace"
|
|
ws.mkdir(parents=True, exist_ok=True)
|
|
(rd / "stages").mkdir(exist_ok=True)
|
|
(rd.parent.parent / "job.json").write_text(json.dumps(job))
|
|
hd = rd / "stages" / "ref"
|
|
hd.mkdir(exist_ok=True)
|
|
ctx = SimpleNamespace(stage_root=rd / "stages", job=job, job_dir=rd.parent.parent, extra={})
|
|
result = salty_dev.execute(ctx, hd, {"command": command(job, ws), "cwd": "."},
|
|
time.monotonic() + DEADLINE_S, hd / "out.log", hd / "err.log")
|
|
info = (ws / "ref_info.log").read_text(errors="ignore") if (ws / "ref_info.log").exists() else ""
|
|
perf = _parse_static_perf(info)
|
|
if result.state != "OK" or "latency_ms_est" not in perf:
|
|
(rd.parent.parent / "FAILED").write_text((hd / "out.log").read_text(errors="ignore")[-4000:]) # not retried
|
|
return f"{jid}\tFAIL {result.state} {result.exit_code}\t{(hd / 'out.log').read_text(errors='ignore')[-300:]!r}"
|
|
sha = (ws / "ref_hbm.sha256").read_text().split()[0] if (ws / "ref_hbm.sha256").exists() else None
|
|
ref = {"schema": "bench.speed_ref/1", "job_id": jid, "march": job["march"], "qcfg": job["qcfg"],
|
|
"config": "platform default (compile.yaml with no overrides)", **perf, "hbm_sha256": sha,
|
|
"measured": "hb_model_info static estimate, not a board run", "computed_at": time.time()}
|
|
OUT.mkdir(exist_ok=True)
|
|
tmp = OUT / f".{jid}.json.tmp"
|
|
tmp.write_text(json.dumps(ref, indent=2))
|
|
tmp.replace(OUT / f"{jid}.json")
|
|
return f"{jid}\tOK\t{perf['latency_ms_est']} ms"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("runs", nargs="+")
|
|
ap.add_argument("--workers", type=int, default=3)
|
|
ap.add_argument("--cpu", default="6")
|
|
ap.add_argument("--only", action="append", help="job ids")
|
|
ap.add_argument("--active", action="store_true", help="only jobs with a rollout under agent v18+")
|
|
ap.add_argument("--skip", action="append", default=[], help="job ids (e.g. one another run is doing)")
|
|
ap.add_argument("--follow", action="store_true", help="keep scanning every 2 min; at most --workers at a time")
|
|
a = ap.parse_args()
|
|
cfg = paths.config()
|
|
cfg["execution"]["salty_remote"].setdefault("cpu_by_stage", {})["terminal"] = int(a.cpu)
|
|
cfg["execution"]["salty_remote"].setdefault("memory_by_stage", {})["terminal"] = "32Gi"
|
|
failed = paths.BENCH_ROOT / "runs" / "zz-speedref"
|
|
|
|
def pending() -> tuple[list[str], dict[str, Path]]:
|
|
jobs: dict[str, Path] = {}
|
|
started: dict[str, int] = {}
|
|
for run in a.runs:
|
|
for jd in sorted((paths.BENCH_ROOT / "runs" / run).iterdir()):
|
|
if (jd / "job.json").exists() and (not a.only or jd.name in a.only) and (
|
|
not a.active or any(own_inputs_required(r) for r in (jd / "rollouts").glob("*"))):
|
|
jobs.setdefault(jd.name, jd) # same job id = same model, march and qcfg in every run
|
|
started[jd.name] = started.get(jd.name, 0) + len(list((jd / "rollouts").glob("*")))
|
|
todo = sorted((j for j in jobs if not (OUT / f"{j}.json").exists() and j not in a.skip
|
|
and not (failed / j / "FAILED").exists()),
|
|
key=lambda j: started.get(j, 0)) # untouched jobs first: new rollouts go there
|
|
return todo, jobs
|
|
|
|
def safe(jid, jd):
|
|
try:
|
|
line = one(jd)
|
|
except Exception as exc:
|
|
line = f"{jid}\tERROR {type(exc).__name__}: {exc}\t{traceback.format_exc(limit=2)[-300:]!r}"
|
|
print(time.strftime("%F %T"), line, flush=True)
|
|
|
|
running: dict[str, object] = {}
|
|
with ThreadPoolExecutor(a.workers) as pool:
|
|
while True:
|
|
running = {j: f for j, f in running.items() if not f.done()}
|
|
todo, jobs = pending()
|
|
for jid in [j for j in todo if j not in running][:a.workers - len(running)]:
|
|
print(time.strftime("%F %T"), "start", jid, flush=True)
|
|
running[jid] = pool.submit(safe, jid, jobs[jid])
|
|
if not a.follow and not running:
|
|
break
|
|
time.sleep(120 if a.follow else 5) # --follow: v18 rollouts keep starting on new jobs
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|