From 29ef645b58a91f2163e9d77125c68f563c3c1cb2 Mon Sep 17 00:00:00 2001 From: bench Date: Tue, 15 Sep 2026 08:10:26 +0800 Subject: [PATCH] =?UTF-8?q?baseline=202026-09-15:=20P1-P3=20landed;=20firs?= =?UTF-8?q?t=20PASS=20trajectory;=20collaboration=20rules=20(CONTRACT=20?= =?UTF-8?q?=C2=A712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.local.example | 9 + .gitignore | 12 + CONTRACT.md | 144 ++++++++ README.md | 100 ++++++ agent/SYSTEM.md | 37 +++ agent/extensions/budget-kill.ts | 97 ++++++ agent/extensions/claim-card.ts | 52 +++ agent/extensions/oe-doc.ts | 33 ++ agent/extensions/path-guard.ts | 125 +++++++ agent/models.json.tmpl | 37 +++ agent/oe-doc.sh | 14 + agent/prompts/e2e-occ.md | 18 + agent/prompts/e2e.md | 21 ++ agent/prompts/repair.md | 15 + agent/settings.json.tmpl | 17 + bin/bench | 8 + config.yaml | 81 +++++ data/.gitkeep | 0 export/redact.py | 115 +++++++ export/reward.py | 79 +++++ export/to_agentrl.py | 259 +++++++++++++++ harness/__init__.py | 17 + harness/adapters/__init__.py | 2 + harness/backends/__init__.py | 12 + harness/backends/ptq.py | 111 +++++++ harness/backends/qat.py | 52 +++ harness/cli.py | 258 +++++++++++++++ harness/db.py | 238 +++++++++++++ harness/errors.py | 46 +++ harness/eval/__init__.py | 11 + harness/eval/cls.py | 57 ++++ harness/eval/common.py | 153 +++++++++ harness/eval/det.py | 101 ++++++ harness/eval/run.py | 104 ++++++ harness/gates.py | 175 ++++++++++ harness/paths.py | 85 +++++ harness/pipeline.py | 516 +++++++++++++++++++++++++++++ harness/registry.py | 303 +++++++++++++++++ harness/report.py | 59 ++++ harness/rollout.py | 165 +++++++++ harness/scheduler.py | 150 +++++++++ harness/stagerun.py | 161 +++++++++ harness/stages/__init__.py | 25 ++ harness/stages/base.py | 149 +++++++++ harness/stages/s01_fetch.py | 65 ++++ harness/stages/s02_baseline.py | 48 +++ harness/stages/s03_export.py | 52 +++ harness/stages/s04_calibrate.py | 109 ++++++ harness/stages/s05_compile.py | 27 ++ harness/stages/s06_hbm.py | 212 ++++++++++++ harness/stages/s07_board.py | 54 +++ harness/stages/s08_eval.py | 42 +++ harness/tools/__init__.py | 1 + harness/tools/board_bench.py | 120 +++++++ harness/tools/board_perf.py | 90 +++++ harness/tools/export_onnx.py | 81 +++++ harness/tools/fetch_weights.py | 85 +++++ harness/tools/infer_hbm.py | 65 ++++ harness/tools/qat_calib_compile.py | 16 + harness/verify.py | 152 +++++++++ harness/watch.py | 117 +++++++ harness/watchdog.py | 139 ++++++++ registry/boards.yaml | 26 ++ registry/budgets.yaml | 28 ++ registry/datasets.yaml | 153 +++++++++ registry/gates.yaml | 100 ++++++ registry/matrix.yaml | 54 +++ registry/models.csv | 339 +++++++++++++++++++ reports/.gitkeep | 0 requirements.txt | 2 + scripts/_probe.py | 24 ++ scripts/board_probe.sh | 23 ++ scripts/excel_to_models.py | 240 ++++++++++++++ scripts/fix_env.sh | 23 ++ scripts/gate_summary.py | 49 +++ scripts/ingest_pool.py | 223 +++++++++++++ scripts/make_calib.py | 149 +++++++++ scripts/make_t1.py | 120 +++++++ scripts/setup_agent.sh | 41 +++ scripts/start_run.sh | 28 ++ scripts/sync.sh | 50 +++ scripts/test_path_guard.py | 119 +++++++ scripts/trace.py | 77 +++++ 83 files changed, 7566 insertions(+) create mode 100644 .env.local.example create mode 100644 .gitignore create mode 100644 CONTRACT.md create mode 100644 README.md create mode 100644 agent/SYSTEM.md create mode 100644 agent/extensions/budget-kill.ts create mode 100644 agent/extensions/claim-card.ts create mode 100644 agent/extensions/oe-doc.ts create mode 100644 agent/extensions/path-guard.ts create mode 100644 agent/models.json.tmpl create mode 100755 agent/oe-doc.sh create mode 100644 agent/prompts/e2e-occ.md create mode 100644 agent/prompts/e2e.md create mode 100644 agent/prompts/repair.md create mode 100644 agent/settings.json.tmpl create mode 100755 bin/bench create mode 100644 config.yaml create mode 100644 data/.gitkeep create mode 100644 export/redact.py create mode 100644 export/reward.py create mode 100644 export/to_agentrl.py create mode 100644 harness/__init__.py create mode 100644 harness/adapters/__init__.py create mode 100644 harness/backends/__init__.py create mode 100644 harness/backends/ptq.py create mode 100644 harness/backends/qat.py create mode 100644 harness/cli.py create mode 100644 harness/db.py create mode 100644 harness/errors.py create mode 100644 harness/eval/__init__.py create mode 100644 harness/eval/cls.py create mode 100644 harness/eval/common.py create mode 100644 harness/eval/det.py create mode 100644 harness/eval/run.py create mode 100644 harness/gates.py create mode 100644 harness/paths.py create mode 100644 harness/pipeline.py create mode 100644 harness/registry.py create mode 100644 harness/report.py create mode 100644 harness/rollout.py create mode 100644 harness/scheduler.py create mode 100644 harness/stagerun.py create mode 100644 harness/stages/__init__.py create mode 100644 harness/stages/base.py create mode 100644 harness/stages/s01_fetch.py create mode 100644 harness/stages/s02_baseline.py create mode 100644 harness/stages/s03_export.py create mode 100644 harness/stages/s04_calibrate.py create mode 100644 harness/stages/s05_compile.py create mode 100644 harness/stages/s06_hbm.py create mode 100644 harness/stages/s07_board.py create mode 100644 harness/stages/s08_eval.py create mode 100644 harness/tools/__init__.py create mode 100644 harness/tools/board_bench.py create mode 100644 harness/tools/board_perf.py create mode 100644 harness/tools/export_onnx.py create mode 100644 harness/tools/fetch_weights.py create mode 100644 harness/tools/infer_hbm.py create mode 100644 harness/tools/qat_calib_compile.py create mode 100644 harness/verify.py create mode 100644 harness/watch.py create mode 100644 harness/watchdog.py create mode 100644 registry/boards.yaml create mode 100644 registry/budgets.yaml create mode 100644 registry/datasets.yaml create mode 100644 registry/gates.yaml create mode 100644 registry/matrix.yaml create mode 100644 registry/models.csv create mode 100644 reports/.gitkeep create mode 100644 requirements.txt create mode 100644 scripts/_probe.py create mode 100755 scripts/board_probe.sh create mode 100644 scripts/excel_to_models.py create mode 100755 scripts/fix_env.sh create mode 100644 scripts/gate_summary.py create mode 100644 scripts/ingest_pool.py create mode 100644 scripts/make_calib.py create mode 100644 scripts/make_t1.py create mode 100644 scripts/setup_agent.sh create mode 100755 scripts/start_run.sh create mode 100755 scripts/sync.sh create mode 100644 scripts/test_path_guard.py create mode 100644 scripts/trace.py diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 0000000..d300586 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,9 @@ +# cp .env.local.example .env.local && chmod 600 .env.local +# harness 启动时 source;agent 进程永远拿不到这些变量 +TEACHER_API_KEY= +STUDENT_API_KEY= +# 板卡:只给 harness 的 07-board 用;优先密钥登录 +BOARD_HOST=rdk-sandbox.d-robotics.cc +BOARD_PORT=40010 +BOARD_USER= +BOARD_SSH_KEY=~/.ssh/rdk_sandbox diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5558206 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.sync/ +__pycache__/ +runs/ +state/ +data/pool/ +data/t1/ +data/calib/ +cache/ +datasets/ +reports/*.html +.env.local +agent/models.json diff --git a/CONTRACT.md b/CONTRACT.md new file mode 100644 index 0000000..30e8c2a --- /dev/null +++ b/CONTRACT.md @@ -0,0 +1,144 @@ +# bench/ 契约(CONTRACT v0.4 落地版) + +本文件是《BPU 量化轨迹工厂 设计方案 v0.4》的可执行摘要。代码与本文件冲突时,以本文件为准并修代码;本文件与设计方案冲突时,先改设计方案再改这里。 + +## 0. 三条原则 + +1. **harness 是 oracle。** 每个 job 先由确定性流水线跑出 ground truth(`oracle/result.json`)。agent 的表现永远相对 oracle 衡量。 +2. **agent 只有一个可写区。** `rollouts//workspace/`。`job.json`、`harness/eval/`、`data/`、`cache/` 全部只读。agent 说什么不算数,verdict 才算数。 +3. **Append-only。** `run_id` 冻结;换工具链、改配置、修 bug 都开新 run。`manifest.json` 记两套 venv 的 `pip freeze` sha256,job 启动前比对,不一致拒跑。状态只在 `state/jobs.sqlite`,文件系统只存内容。 + +## 1. 目录 + +``` +bench/ +├── CONTRACT.md README.md config.yaml requirements.txt +├── bin/bench # CLI 入口 +├── registry/ # 输入 · 不可变 · git 管理 +│ ├── models.csv matrix.yaml gates.yaml budgets.yaml datasets.yaml boards.yaml +├── data/ # 只读 · agent 永不下载 +│ ├── pool/// # 全量 + manifest.json +│ ├── t1//.tar # smoke 子集,单包 +│ └── calib/// # npy,与 val 零交集 +├── cache/{weights,onnx}// cache/baseline/--/ +├── harness/ # L1 + L3 +│ ├── pipeline.py scheduler.py watchdog.py registry.py db.py errors.py gates.py +│ ├── stages/ backends/{ptq,qat}.py eval/ +├── agent/ # L2 pi 配置 +│ ├── SYSTEM.md settings.json.tmpl models.json.tmpl prompts/ extensions/ +├── export/{to_agentrl,redact,reward}.py +├── scripts/ # 一次性:入池、Excel 转换、环境修补、板卡探测 +├── state/jobs.sqlite reports/ datasets// +└── runs// + ├── manifest.json + └── / + ├── job.json # 只读 · 输入契约 + ├── oracle/stages/-/{stdout.log,stderr.log,exit_code,meta.json} + ├── oracle/artifacts/.{onnx,bc,hbm} + ├── oracle/result.json + └── rollouts//{workspace/, agent/{session.jsonl,events.jsonl}, claim.json, verdict.json} +``` + +## 2. 命名 + +| 标识 | 格式 | 示例 | +|---|---|---| +| run_id | `r-` | `r20260914-01` | +| job_id | `--------` | `det--yolov8n--nash-e--ptq--int8` | +| task | 10 类固定 | `cls det seg iseg pose mde stereo flow feat occ` | +| march | `nash-e / nash-m / nash-p` | s100→e · s100p→m · s600→p | +| qcfg | 段内 `a-z0-9.` | `int8` · `fp16int8` | +| rollout | `/rollouts/` | `…/rollouts/03` | +| stage | `-` | `01-fetch 02-baseline 03-export 04-calibrate 05-compile 06-hbm 07-board 08-eval` | +| dataset_id | `ds-` | `ds20260914-01` | + +段内字符集 `[a-z0-9.]`,段间 `--`。job_id 既是目录名也是 join key。 + +## 3. 状态机 + +**job(由 oracle 决定)**:`PENDING → ORACLE_RUNNING → PASS | FAIL | UNVERIFIED` +**rollout(由 verdict 决定)**:`QUEUED → RUNNING → VERIFYING → PASS | FAIL | BLOCKED | TIMEOUT` + +- PASS = 必选门全部 pass。 +- 门无法判定(数据缺、板未接)→ 该门 `unverified`,job `UNVERIFIED`。 +- `na`(本轮设计上不评)≠ `unverified`(本该评没评成)。na 不阻塞 PASS。 +- BLOCKED 判定:oracle 同阶段同类 error_class 失败 → 正确;oracle 通过 → 误判。 + +## 4. 四道门 + +| 门 | 必选 | 判定 | +|---|---|---| +| G1 compile | 是 | .hbm 存在;`hb_model_info` 的 march / 输入 dtype / layout 与 job.json 一致;`hb_verifier` 通过;日志无 ERROR | +| G2 numeric | 是 | 校准外样本逐输出余弦 ≥ 阈值(默认 0.99;occ 0.97;feat 逐输出);只报 min | +| G3 task | 是 | 量化实测 − FP32 基线在容差内;kind 必须显式 `abs_pp` 或 `rel_pct`;T1 上记 `pass-screen`,T2 上才 `pass` | +| G4 perf | 否 | 仅 job.json 有 `latency_budget_ms` 才判;mean 与 p95 分开存 | + +## 5. 数据三档 + +| 档 | 用途 | 规模 | +|---|---|---| +| T0 calib | PTQ 校准 | 128–512,train 抽,与 val 零交集 | +| T1 smoke | 默认门 | cls 2000 · det/iseg/pose 1000 · seg 500 · 其余全量 | +| T2 full | 交付数 | 完整 val,开新 run | + +复制入池,不软链;`chmod -R a-w`;`manifest.json` 记 sha256/数量/来源/日期。T1 打成单 tar 避开 JuiceFS 小文件。 + +## 6. 调度与看门狗 + +- 池:`gpu 4 · cpu 8 · board 1 · agent 3`。板卡严格串行带锁。 +- **没有固定超时。** STALLED = 日志 `stall_s`(900 s)不增长 且 进程树 CPU < 5%。硬上限 24 h 只是保险。 +- 重阶段(compile / board / eval)由 harness 执行。agent 走 `bench stage run` / `bench stage wait`,不在 bash 里跑 hb_compile。 +- rollout 预算 = turns + tokens;墙钟 12 h 仅防挂死。 + +## 7. 重试与轨迹边界 + +| 层 | 谁 | 触发 | 轨迹 | +|---|---|---|---| +| oracle 阶段 | harness | 瞬态 error_class | 不是轨迹 | +| rollout 内 | agent | 自己改配置重提 | 同一条轨迹 | +| 新 rollout | harness | matrix k | 全新轨迹,永不串联 | +| repair | harness | oracle FAIL | 全新轨迹,输入是 oracle 现场 | + +## 8. error_class + +`unsupported_op shape_infer export_fail calib_nan verifier_mismatch`(确定性,不重试) +`oom board_offline stalled`(瞬态,重试 ≤ 2 / ≤ 2 / 1) +`unknown`(进人工分类队列) + +## 9. reward(v0,离线算,配方版本化) + +``` +r = 1.0[G1] + 1.0[G2] + 2.0[G3] +r += 1.0·clip((Δ_agent − Δ_oracle)/|tol|, −1, 1) +r += 0.5·clip((lat_oracle − lat_agent)/lat_oracle, −1, 1) +r −= 0.5·turns/max_turns + 0.5·tokens/max_tokens +r −= 2.0[claim ≠ verdict] +BLOCKED: +1.0 若 oracle 同类失败 else −2.0 ;TIMEOUT: −1.0 +G3 = na 时其 2.0 权重按比例分给 G1/G2,精度相对项取 0 +``` + +## 10. 两套 venv + +| backend | venv | 关键版本 | +|---|---|---| +| ptq | `~/toolchains/oe-3.7.0/venv` | hbdk4 4.7.5 · horizon_tc_ui 3.5.3 · hmct 2.6.5 | +| qat | `~/toolchains/oellm2/venv` | hbdk4 4.12.3a2 · horizon_plugin_pytorch 3.3.5 | + +harness 按 backend 选解释器。**禁止把 backend 判断交给 agent。** + +## 11. 下游边界 + +`~/data_collect` 零耦合。只搬清洗规则 R1–R8 与 redact,落在 `export/`,不 import。 + +## 12. 协作规则(2026-09-15 起,两个会话同时改一棵树踩过一次) + +- **只有一个写者能改一层。** 分层:`harness/`(oracle 流水线、阶段、门、调度)· `agent/`(pi 配置、扩展、提示词)· `export/`(清洗、脱敏、reward)· `registry/`(清单与阈值,改了就是新 run)· `scripts/`。谁在改哪层写进下面的表;别的会话对那层只读。 +- **改动只发生在本地副本,机器上不直接改代码。** 流程:改 → `git commit`(`scripts/sync.sh` 会自动提交)→ `scripts/sync.sh` 推送。sync 带守卫:远端自上次推送后被动过就拒绝,先合并再推。 +- `registry/models.csv` 是在机器上由 `excel_to_models.py` 生成的,方向反过来:`scripts/sync.sh --pull`。 +- 机器上的 `runs/ state/ data/ cache/ datasets/` 是数据,不同步、不手改;改判走 `bench rejudge`,补测走 `bench board-backfill`。 +- 正在跑的调度进程(`bench run` / `bench collect`)加载的是启动时的代码;改了 harness 只对新进程生效,子进程工具(`harness.tools.*`、`harness.eval.run`)立刻生效——改评测口径必须同时升 `EVALUATOR_VERSION`。 + +| 层 | 当前写者 | +|---|---| +| harness/ agent/ export/ scripts/ registry/ | 本会话(Cowork · bench 主线) | +| 其他会话 | 只读;发现问题写成 issue 贴过来,不直接改 | diff --git a/README.md b/README.md new file mode 100644 index 0000000..598c043 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# bench/ — BPU 量化轨迹工厂 · P1 骨架 + +设计:`../BPU量化轨迹工厂-设计方案.html`(v0.4)。契约:`CONTRACT.md`。本 README 只讲怎么把它放到机器上跑起来。 + +## 0. 上机 + +```bash +# 本地 → 机器(不带 runs/ state/ data/ cache/,它们在机器上生成) +rsync -av --exclude runs --exclude state --exclude 'data/pool' --exclude 'data/t1' --exclude 'data/calib' \ + --exclude cache --exclude datasets --exclude .env.local \ + bench/ ming01.wang@120.48.151.164:/mnt/data/ming01.wang/model-compile-hbdk/bench/ +# .env.local 单独传(600),不要进 git +scp bench/.env.local ming01.wang@120.48.151.164:/mnt/data/ming01.wang/model-compile-hbdk/bench/.env.local +``` + +机器上: + +```bash +cd /mnt/data/ming01.wang/model-compile-hbdk/bench +export PATH=$PWD/bin:$PATH +# harness 只要 pyyaml;系统 python3 没有就用 PTQ venv 的解释器 +python3 -c 'import yaml' || export BENCH_PYTHON=~/toolchains/oe-3.7.0/venv/bin/python +# JuiceFS 是 FUSE:小文件写入实测 ~0.5 s/个。state/(sqlite+锁)、data/(5 万张图)、cache/ 都放本地盘,软链过去 +for d in state data cache; do mkdir -p /home/$USER/bench-$d; [ -L $d ] || { rm -rf $d; ln -s /home/$USER/bench-$d $d; }; done +# runs/(每 job 几十个文件)留在 JuiceFS 没问题 +``` + +## 1. P1 顺序(每步都能单独重跑) + +| # | 命令 | 产出 | +|---|---|---| +| 1 | `scripts/fix_env.sh` | `.env.oe-package` 的 `PYTHON_BIN` 指向真实 QAT venv(备份 `.bak-日期`) | +| 2 | `python scripts/excel_to_models.py <清单.xlsx> --dry-run` → 改 `COLMAP` → 去掉 `--dry-run` | `registry/models.csv`(338 行) | +| 3 | `source .env.local && scripts/board_probe.sh` | 沙箱板型 → 填 `registry/boards.yaml` 的 `board_type` / `march` | +| 4 | `~/toolchains/oe-3.7.0/venv/bin/python scripts/ingest_pool.py --dataset imagenet1k --dataset coco2017` | `data/pool/…/manifest.json`,只读 | +| 5 | `… scripts/make_t1.py --dataset imagenet1k --dataset coco2017` | `data/t1//20260914.tar` | +| 6 | `… scripts/make_calib.py --task cls --input-spec 1x3x224x224` / `--task det --input-spec 1x3x640x640` | `data/calib/…` | +| 7 | `bench preflight` | 0 errors 才往下 | +| 8 | `bench new-run --note "P2 slice"` → `bench expand` | `runs/r…//job.json` × 6,`state/jobs.sqlite` | +| 9 | `bench run --dry-run` | 状态机空转一遍,不碰工具链 | + +步骤 4–6 用 PTQ venv 的 python(要 pyarrow / pillow / numpy)。ImageNet 首选 HF parquet 14 片;tar 是备用源,入池时会 `tar -tzf | wc -l` 校验。 + +## 2. P2 切片(真跑) + +```bash +bench run --job cls--resnet18--nash-e--ptq--int8 --parallel 1 +bench show cls--resnet18--nash-e--ptq--int8 # 阶段时间线、stderr 尾巴 +bench run --job det--yolov8n--nash-e--ptq--int8 --parallel 1 +bench status; bench status --fail; bench report # reports/.html +``` + +P2 实测已核对(2026-09-15,resnet18 / nash-e / PTQ): +- `hb_compile 3.5.3` 接受我们的 yaml;`norm_type`、`cal_data_type` 已弃用(去掉了);**cal_data_dir 下只能有 .npy**(它把目录里所有文件当样本读,manifest.json 会炸)→ 校准集放 `/npy_f32/`,manifest 在上一级。 +- `hb_model_info`:I/O 表与 march 打在 stderr,静态 perf 估算(FPS / latency / DDR)打在 stdout;两者都解析,静态估算记进 `G1.static_perf`(不是板端实测)。 +- `hb_verifier`:`-m ptq_model.onnx,quantized_model.bc -i `,逐层余弦表,rc=0;取最后一行(模型输出)余弦对照 G2 阈值。 +- x86 跑量化模型:`HB_HBIRRuntime(quantized_model.bc).run()` **0.04 s/张,与 `HB_HBMRuntime(hbm).run_sim()` 逐位一致**(后者 ~20 s/张)。08-eval 全量走 .bc,06-hbm 用 2 张抽查 hbm ≡ bc。`ptq_model.onnx` 走 HB_ONNXRuntime 只是伪量化(cos≈0.9995),不用它出数。oellm2 里的 `hbm_infer` 是板端 RPC,不是模拟器。 +- **HB_HBIRRuntime.run 返回的数组引用内部缓冲区**,下一次 run 原地覆盖——必须 `np.array(v, copy=True)`。踩过:一版 08-eval 的 top-1 算成 2.6%。 +- resnet18 FP32 在 T1(2000 张分层)top-1 = 68.15(官方全量 69.76,2k 子集 SE≈1 pp,合理);hb_compile 校准余弦 0.999。 + +仍待核对:`tools/board_perf.py` 解析 `hrt_model_exec perf`(要板);`eval/det.py` 的 YOLOv8 后处理口径(P2 第二个切片)。核对完把 `VERIFIED = True`。 + +## 3. P3 闭环(agent) + +```bash +bench agent setup # ~/.pi/agent/models.json(600),key 来自 .env.local +bench rollout cls--resnet18--nash-e--ptq--int8 --kk 0 --mode e2e --policy teacher --dry-run # 只准备目录、渲染 prompt/settings +bench rollout cls--resnet18--nash-e--ptq--int8 --kk 0 --mode e2e --policy teacher # 真起 pi +python export/to_agentrl.py --runs r20260914-01 # datasets/ds…/teacher.jsonl +``` + +P3 要核对(代码里标了 `P3`):pi 的 `--append-system-prompt` / `--tools` 参数;`agent/extensions/*.ts` 用到的 ExtensionAPI 形状(`pi.on("tool_call")` 返回 `{block, reason}`、`registerTool`、`turn_end`、`ctx.abort`);`models.json` 的 provider 字段名;`bench stage wait` 会不会被 pi 的 bash 超时掐断(不行就 `--poll 60` 改成 agent 反复调 `stage status`)。 + +## 4. agent 在 rollout 里能用什么 + +``` +bench stage run calibrate|compile|hbm|board|eval [--config workspace/x.yaml] → handle +bench stage wait [--poll 60] +bench stage status +``` +march / 输出前缀 / 工作目录被 harness 钉死;`--config` 必须在 `workspace/` 内;`fetch/baseline/export` 不允许 agent 提交。写文件只能在 `workspace/`(path-guard + 文件权限双保险)。凭证:LLM key 在 `~/.pi/agent/models.json`,板卡凭证只进 `07-board` 子进程。 + +## 5. 实现相对 v0.4 的补充(下次改文档时并入) + +- **机器是共享的**(2026-09-14 实测):244 核 / 1.4 TB / **8 张 32 GB 卡**(不是 4 张),4/5/7 被别人跑满、6 占 29 GB;12 个登录用户。`config.yaml pools.gpu.devices: [0,1,2,3]` 钉死只用空闲卡,槽位 → `CUDA_VISIBLE_DEVICES`。 +- 资源池是跨进程的(`state/locks/-.lock` flock):oracle 调度进程、agent 的 `bench stage run`、多个 `bench run` 共用同一组槽位;board 池 1 槽 = 严格串行。model-level 阶段按模型加锁,三个 march 并行时只有一个真跑 01–03。job 认领是原子的(UPDATE … WHERE state='PENDING'),两个调度进程不会重跑同一个 job。 +- 新 run 的 manifest 记 `harness_code_sha256`;harness 代码变了只告警,venv / registry / EVALUATOR_VERSION 变了拒跑(实测拦下过一次:改 config.yaml 的 venv 列表后 `bench run` 拒绝,开了新 run)。 +- 沙箱板只有一块、一种 march:其他 march 的 job 07-board 直接 SKIPPED → G4 unverified,不阻塞。`board_perf.py` 开测前探测板上有无别人的推理进程 / BPU ratio,最多等 10 分钟,仍忙则照测但标 `contended: true`,reward 的延时相对项忽略这类样本。 + +- `job.json` 与 mode 无关:一个 job_id 一份,`rollout_plans` 列出 e2e/repair 计划;oracle 与所有 rollout 共用。 +- sqlite 默认 TRUNCATE 日志(WAL 在 FUSE 上会 disk I/O error);`BENCH_STATE_DIR` 可把 state 放本地盘。 +- 校准集来源:本机 ImageNet / COCO 只有 val,`make_calib.py` 退到 val-holdout(排除 T1)并在 manifest 记 `source`;出 T2 交付数前必须换 train。 +- 07-board 失败重试后仍失败 → G4 unverified、继续 08-eval;08-eval 失败 → job UNVERIFIED 带 error_class;01–06 失败 → FAIL。 +- claim 缺席(agent 没调 `claim` 工具)→ claim.json 记 `ABSENT`,reward −0.25;claim=PASS 但没有可核验产物 → verdict FAIL 并计 claim≠verdict。 +- BLOCKED 正确性:claim.error_class 与 oracle 同类(`unknown` 不算同类)且阶段一致(claim 未填阶段则不比阶段)。 +- reward 里相对精度项对 lower-is-better 指标已在 gates 层翻转成"越大越好",配方不用区分。 + +## 6. 目录里什么进 git + +`registry/ harness/ agent/ export/ scripts/ bin/ config.yaml CONTRACT.md README.md`。`runs/ datasets/ state/ data/{pool,t1,calib} cache/ .env.local reports/*.html` 不进(见 `.gitignore`)。 diff --git a/agent/SYSTEM.md b/agent/SYSTEM.md new file mode 100644 index 0000000..6be71dc --- /dev/null +++ b/agent/SYSTEM.md @@ -0,0 +1,37 @@ +# bench agent · 系统提示(追加在 pi 默认系统提示之后) + +你在一个受控的量化上板环境里工作。规则不是建议,违反会被工具层拒绝或被判负。 + +## 你能写的地方 +只有当前目录(`workspace/`)。其他一切只读:`../job.json`、`../../oracle/`(如果存在)、`data/`、`cache/`、`harness/`。越界写入会被 path-guard 拒绝并记录。 + +## 重阶段不在 bash 里跑 +编译、上板、评测由 harness 执行。你提交、你等待、你看日志: + +``` +bench stage run calibrate # 生成编译配置(也可先看 job.json 自己写一份) +bench stage run compile --config workspace/compile.yaml +bench stage wait --poll 60 +bench stage run hbm # 收产物 + hb_model_info + hb_verifier +bench stage run board # 板端 perf(可选,有板才跑) +bench stage run eval # G2 / G3 +``` + +`--config` 必须在 `workspace/` 内。march、输出前缀、工作目录由 harness 钉死,改了也不生效。不要自己调用 `hb_compile`、`hb_mapper`、`hrt_model_exec`、`ssh`——它们要么不在你的环境里,要么会被拒。 + +## 同一方法最多重试 1 次 +编译报错后,改配置再提交是允许的、被期待的。但同一套配置原样重提第三次没有意义。换方法:改校准参数、改输入类型、改算子、拆模型。三次不同方法仍失败 → 报 BLOCKED。 + +## 结束前必须调用 `claim` 工具 +``` +claim(terminal="PASS"|"FAIL"|"BLOCKED", stage=..., error_class=..., summary="…", artifacts=[…]) +``` +**预算规则**:剩 5 轮时你会收到一条预算提醒——收到后无论进展如何,先调用 `claim` 汇报当前状态(不确定就报 FAIL 并写清到了哪一步),再决定要不要继续。预算耗尽后除 `claim` 以外的工具都会被拒绝;没有 claim 就结束的轨迹被记为 claim 缺席,另扣分。 + +你的 claim 会与 harness 独立核验的 verdict 比对。claim=PASS 而 verdict=FAIL 是最重的扣分项,比老实报 FAIL 差得多。不确定就报 FAIL 并说明。BLOCKED 只在你确认是工具链/算子层面无法完成时使用,并给出 error_class(unsupported_op / shape_infer / calib_nan / verifier_mismatch 之一)。 + +## 不要做的事 +不下载数据集或权重(都在只读区);不改 `job.json`;不猜指标——指标只来自 `bench stage run eval` 的输出;不在 bash 里 `sleep` 等编译,用 `bench stage wait`。 + +## 工具链参考 +`.horizon/skills/` 下的 skill 是这条工具链的文档。`oe-doc ` 可以查 OE 文档。两套 venv 的选择由 harness 按 job.json 的 backend 决定,你不需要也不能选。 diff --git a/agent/extensions/budget-kill.ts b/agent/extensions/budget-kill.ts new file mode 100644 index 0000000..2665917 --- /dev/null +++ b/agent/extensions/budget-kill.ts @@ -0,0 +1,97 @@ +/** + * budget-kill — turns 与 tokens 是真正的预算(agent 自己能控制的量)。三段式,先软后硬: + * + * 1. 预警(剩 WARN_AHEAD 轮):塞一条 steer 消息告诉 agent"预算快没了,先 claim"。 + * 2. 耗尽(turns ≥ MAX 或 tokens ≥ MAX):写 rollout.json state=TIMEOUT;此后 **除 claim 外的所有工具调用都被 block**, + * 模型只剩 claim 可用 → 调了 claim 或干脆停手 → 会话自然走到 agent_end → JSON 流完整 → 轨迹可导出。 + * 3. 兜底(turns ≥ MAX + GRACE_TURNS):模型还在原地打转 → process.exit(75),harness 按 TIMEOUT 处理。 + * + * 第一版在 turn_end 直接 ctx.abort():实测 pi 0.73.1 只掐断在途请求,进程继续下一轮,第一条轨迹跑到 turn 41 也没停; + * 而立刻 process.exit 会丢 agent_end,导出端拿不到线性化的 messages[]——所以硬退出只能是最后一手。 + * + * 预算从 env 读(BENCH_MAX_TURNS / BENCH_MAX_TOKENS),由 rollout 驱动器按 config.yaml 设置。 + */ +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import * as fs from "node:fs"; +import * as path from "node:path"; + +const ROLLOUT = process.env.BENCH_ROLLOUT_DIR ?? ""; +const MAX_TURNS = Number(process.env.BENCH_MAX_TURNS ?? 40); +const MAX_TOKENS = Number(process.env.BENCH_MAX_TOKENS ?? 200000); +const WARN_AHEAD = Number(process.env.BENCH_WARN_AHEAD_TURNS ?? 5); +const GRACE_TURNS = Number(process.env.BENCH_GRACE_TURNS ?? 3); + +let turns = 0; +let tokens = 0; +let exhausted: string | null = null; +let warned = false; + +function writeRollout(patch: Record) { + if (!ROLLOUT) return; + const p = path.resolve(ROLLOUT, "rollout.json"); + let rec: any = {}; + try { rec = JSON.parse(fs.readFileSync(p, "utf8")); } catch { /* 新建 */ } + Object.assign(rec, patch, { turns, tokens }); + fs.writeFileSync(p, JSON.stringify(rec, null, 2)); +} + +function usageOf(msg: any): number { + const u = msg?.usage; + if (!u) return 0; + return Number(u.input ?? u.input_tokens ?? 0) + Number(u.output ?? u.output_tokens ?? 0) + + Number(u.cacheRead ?? 0) + Number(u.cacheWrite ?? 0); +} + +export default function (pi: ExtensionAPI) { + // 2. 耗尽后只放行 claim + pi.on("tool_call", async (event: any) => { + if (!exhausted) return undefined; + const name: string = event.toolName ?? event.tool ?? ""; + if (name === "claim") return undefined; + return { block: true, reason: `预算已耗尽(${exhausted})。现在只能调用 claim 汇报最终状态;不确定就报 FAIL 并说明。` }; + }); + + pi.on("turn_end", async (event: any, ctx: any) => { + turns += 1; + tokens += usageOf(event?.message); + const left = MAX_TURNS - turns; + + // 1. 预警 + if (!warned && !exhausted && (left <= WARN_AHEAD || tokens >= MAX_TOKENS * 0.85)) { + warned = true; + writeRollout({ budget_warned_at_turn: turns }); + try { + pi.sendMessage( + { customType: "bench-budget", content: `【预算提醒】剩 ${Math.max(left, 0)} 轮 / ${Math.max(MAX_TOKENS - tokens, 0)} tokens。无论进展如何,请现在就调用 claim 汇报当前状态(不确定就报 FAIL 并说明到哪一步),之后再继续。`, display: true }, + { triggerTurn: false, deliverAs: "steer" }, + ); + } catch { /* sendMessage 不可用就只靠 SYSTEM.md 里的规则 */ } + } + + // 2. 耗尽 + if (!exhausted) { + if (turns >= MAX_TURNS) exhausted = `max_turns ${MAX_TURNS}`; + else if (tokens >= MAX_TOKENS) exhausted = `max_tokens ${MAX_TOKENS}`; + if (exhausted) { + writeRollout({ state: "TIMEOUT", timeout_reason: exhausted, timed_out_at: Date.now() / 1000 }); + try { + pi.sendMessage( + { customType: "bench-budget", content: `【预算耗尽:${exhausted}】除 claim 以外的工具已被禁用。请立即调用 claim,然后结束。`, display: true }, + { triggerTurn: false, deliverAs: "steer" }, + ); + } catch { /* ignore */ } + } + } + + // 3. 兜底:耗尽后还赖着不走 + if (exhausted && turns >= MAX_TURNS + GRACE_TURNS) { + writeRollout({ hard_killed_at_turn: turns }); + try { ctx?.abort?.(); } catch { /* ignore */ } + setTimeout(() => process.exit(75), 2000).unref?.(); + } + }); + + pi.on("agent_end", async () => { + writeRollout({ agent_end_turns: turns, agent_end_tokens: tokens }); + }); +} diff --git a/agent/extensions/claim-card.ts b/agent/extensions/claim-card.ts new file mode 100644 index 0000000..0476c6d --- /dev/null +++ b/agent/extensions/claim-card.ts @@ -0,0 +1,52 @@ +/** + * claim-card — 强制结构化汇报。注册 `claim` 工具,写 /claim.json。 + * + * claim 是 agent 自报,不可信;L3 用 verdict 核验并比对。agent_end 时若没调用过 claim, + * 写一份 { terminal: "ABSENT" },reward 小额扣分(见 export/reward.py)。 + * + * P3 核对:pi.registerTool 的签名与 Type(typebox)导入路径。 + */ +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Type } from "@sinclair/typebox"; +import * as fs from "node:fs"; +import * as path from "node:path"; + +const ROLLOUT = process.env.BENCH_ROLLOUT_DIR ?? ""; +const CLAIM = ROLLOUT ? path.resolve(ROLLOUT, "claim.json") : path.resolve(process.cwd(), "claim.json"); + +let claimed = false; + +export default function (pi: ExtensionAPI) { + pi.registerTool({ + name: "claim", + label: "claim", + description: + "结束前必须调用一次。汇报你认为的最终状态。会与 harness 独立核验的 verdict 比对;claim=PASS 而 verdict=FAIL 是最重扣分项。", + parameters: Type.Object({ + terminal: Type.Union([Type.Literal("PASS"), Type.Literal("FAIL"), Type.Literal("BLOCKED")]), + stage: Type.Optional(Type.String({ description: "失败/阻塞发生的阶段,如 05-compile" })), + error_class: Type.Optional( + Type.Union([ + Type.Literal("unsupported_op"), Type.Literal("shape_infer"), Type.Literal("export_fail"), + Type.Literal("calib_nan"), Type.Literal("verifier_mismatch"), Type.Literal("unknown"), + ]), + ), + summary: Type.String({ description: "你做了什么、为什么是这个结论,3–8 句" }), + artifacts: Type.Optional(Type.Array(Type.String(), { description: "workspace 内的关键产物相对路径" })), + metrics_seen: Type.Optional(Type.Record(Type.String(), Type.Number(), { description: "你从 bench stage eval 输出里看到的数字(不是你估计的)" })), + }), + async execute(_id: string, params: any) { + const rec = { ...params, claimed_at: Date.now() / 1000, schema: "bench.claim/1" }; + fs.writeFileSync(CLAIM, JSON.stringify(rec, null, 2)); // 最后一次 claim 为准(预算提醒后可能先 claim 一次再继续) + try { fs.appendFileSync(CLAIM.replace(/\.json$/, "s.jsonl"), JSON.stringify(rec) + "\n"); } catch { /* 历史可选 */ } + claimed = true; + return { content: [{ type: "text", text: `claim 已记录:${params.terminal}` }], details: rec }; + }, + }); + + pi.on("agent_end", async () => { + if (!claimed && !fs.existsSync(CLAIM)) { + fs.writeFileSync(CLAIM, JSON.stringify({ terminal: "ABSENT", schema: "bench.claim/1", claimed_at: Date.now() / 1000 }, null, 2)); + } + }); +} diff --git a/agent/extensions/oe-doc.ts b/agent/extensions/oe-doc.ts new file mode 100644 index 0000000..eb1a0ac --- /dev/null +++ b/agent/extensions/oe-doc.ts @@ -0,0 +1,33 @@ +/** + * oe-doc — 把 oe-mcp 的文档检索包成 pi 工具(pi 不支持 MCP)。 + * + * 实现方式:子进程调用 `bench/agent/oe-doc.sh `,它再去调 oe-mcp 的 CLI 或直接 grep .horizon/docs。 + * 只读;输出截断到 8k 字符。P3 把 oe-doc.sh 接到真实的 oe-mcp 入口。 + */ +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Type } from "@sinclair/typebox"; +import { execFileSync } from "node:child_process"; +import * as path from "node:path"; + +const BENCH_ROOT = process.env.BENCH_ROOT ?? path.resolve(__dirname, "..", ".."); +const SCRIPT = path.resolve(BENCH_ROOT, "agent", "oe-doc.sh"); + +export default function (pi: ExtensionAPI) { + pi.registerTool({ + name: "oe_doc", + label: "oe_doc", + description: "查 OpenExplorer / hbdk / horizon_tc_ui 文档。输入关键词或报错片段,返回相关文档段落。", + parameters: Type.Object({ query: Type.String(), max_chars: Type.Optional(Type.Number({ default: 8000 })) }), + async execute(_id: string, params: any) { + let out = ""; + try { + out = execFileSync(SCRIPT, [params.query], { encoding: "utf8", timeout: 60_000, maxBuffer: 4 << 20 }); + } catch (e: any) { + out = `oe-doc 失败: ${e?.message ?? e}`; + } + const max = Number(params.max_chars ?? 8000); + if (out.length > max) out = out.slice(0, max) + `\n…(截断,共 ${out.length} 字符)`; + return { content: [{ type: "text", text: out }], details: { query: params.query, chars: out.length } }; + }, + }); +} diff --git a/agent/extensions/path-guard.ts b/agent/extensions/path-guard.ts new file mode 100644 index 0000000..ae18a92 --- /dev/null +++ b/agent/extensions/path-guard.ts @@ -0,0 +1,125 @@ +/** + * path-guard — agent 只有一个可写区:/workspace/。 + * + * 拦截 write / edit 的目标路径;拦截 bash 里明显的越界写入(重定向、cp/mv/rm/tee/chmod 指向 workspace 之外)。 + * bash 的静态检查不可能完备,第二道保险是文件系统权限:harness 把 job.json / data / cache / eval 设为只读。 + * 每次拒绝写一行到 /agent/guard.log,L4 导出时可统计"越界尝试次数"。 + * + * API 形状对照 pi-coding-agent 的 ExtensionAPI(P3 按安装版本核对:pi.on("tool_call") 返回 { block, reason })。 + */ +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import * as fs from "node:fs"; +import * as path from "node:path"; + +const ROLLOUT = process.env.BENCH_ROLLOUT_DIR ?? ""; +const WORKSPACE = ROLLOUT ? path.resolve(ROLLOUT, "workspace") : process.cwd(); +const GUARD_LOG = ROLLOUT ? path.resolve(ROLLOUT, "agent", "guard.log") : ""; + +function inside(p: string): boolean { + const abs = path.resolve(WORKSPACE, p); + return abs === WORKSPACE || abs.startsWith(WORKSPACE + path.sep); +} + +function log(kind: string, detail: string) { + if (!GUARD_LOG) return; + try { + fs.mkdirSync(path.dirname(GUARD_LOG), { recursive: true }); + fs.appendFileSync(GUARD_LOG, JSON.stringify({ ts: Date.now() / 1000, kind, detail }) + "\n"); + } catch { /* 日志失败不影响拒绝 */ } +} + +// bash 静态规则:命中即拒。故意保守——误杀一个合法命令的代价远小于放过一次越界写。 +// 第一版把 `2>/dev/null` 也当成了"重定向到绝对路径",第一条真实轨迹里 10 次调用被拒 4 次,全是误杀 → 重定向单独判目标。 +// 被禁的**命令**:只在"作为命令被调用"的位置匹配(行首 / ; | && 之后 / env 赋值之后), +// 否则 `ps aux | grep hb_compile`、`grep -r hb_compile logs/` 这类只读排查也会被拒——第一条轨迹里真的被拒了。 +const FORBIDDEN_CMDS = ["hb_compile", "hb_mapper", "hb_verifier", "hrt_model_exec", "hb_perf", + "ssh", "scp", "rsync", "wget", "curl", "pip", "pip3", "conda", "npm"]; +const CMD_POSITION = new RegExp( + String.raw`(?:^|[;&|]\s*|\$\(|` + "`" + String.raw`)\s*(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)*(?:sudo\s+|env\s+|nohup\s+|time\s+)*(` + + FORBIDDEN_CMDS.join("|") + String.raw`)\b`); +const WRITE_PATTERNS: RegExp[] = [ + /\bsleep\s+[0-9]{3,}/, // 用 bench stage wait,不要 sleep 等编译 + /\bgit\s+push\b/, +]; + +// 写文件的命令:destLast 的最后一个参数才是目的地(cp ../job.json . 是合法的"读进来"); +// 其余命令的每个非选项参数都是目标。/tmp 放行(scratch),/dev/null 放行。 +const DEST_LAST = new Set(["cp", "mv", "ln", "install", "rsync"]); +const ALL_ARGS = new Set(["rm", "rmdir", "tee", "touch", "mkdir", "chmod", "chown", "truncate", "dd", "shred"]); + +function writeTargets(segment: string): string[] { + const toks = segment.trim().split(/\s+/).filter(Boolean); + let i = 0; + while (i < toks.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(toks[i])) i++; // 跳过 env 赋值 + while (i < toks.length && ["sudo", "env", "nohup", "time"].includes(toks[i])) i++; + const cmd = (toks[i] ?? "").replace(/^.*\//, ""); + const args = toks.slice(i + 1).filter((t) => !t.startsWith("-")).map((t) => t.replace(/^['"]|['"]$/g, "")); + if (DEST_LAST.has(cmd)) return args.length ? [args[args.length - 1]] : []; + if (ALL_ARGS.has(cmd)) return args; + return []; +} + +function badWriteTarget(cmd: string): string | null { + for (const seg of cmd.split(/;|\|\||&&|\||&/)) { + for (const t of writeTargets(seg)) { + if (t.startsWith("/tmp/") || t === "/dev/null") continue; + if (!inside(t)) return t; + } + } + return null; +} +const REDIRECT_OK = new Set(["/dev/null", "/dev/stderr", "/dev/stdout"]); + +/** 找出所有输出重定向目标(> >> 2> &>),有一个落在 workspace 外就拒。相对路径以 cwd=workspace 解析;`cd` 过的命令无法追踪,靠文件权限兜底。 */ +function badRedirect(cmd: string): string | null { + const re = /(?:^|[\s;|&(])(?:\d?>>?|&>)\s*['"]?([^\s'"|;&)]+)/g; + let m: RegExpExecArray | null; + while ((m = re.exec(cmd)) !== null) { + const target = m[1]; + if (target.startsWith("&")) continue; // 2>&1 + if (REDIRECT_OK.has(target)) continue; + if (target.startsWith("/proc/self/fd/")) continue; + if (!inside(target)) return target; + } + return null; +} + +export default function (pi: ExtensionAPI) { + pi.on("tool_call", async (event: any, _ctx: any) => { + const name: string = event.toolName ?? event.tool ?? ""; + const input: any = event.input ?? {}; + + if (name === "write" || name === "edit") { + const p: string = input.path ?? input.file_path ?? ""; + if (!p || !inside(p)) { + log("deny-" + name, p); + return { block: true, reason: `path-guard: 只能写 workspace/ 内的文件(${p})` }; + } + } + if (name === "bash") { + const cmd: string = input.command ?? ""; + const bad = badRedirect(cmd); + if (bad) { + log("deny-bash-redirect", cmd.slice(0, 500)); + return { block: true, reason: `path-guard: 重定向到 workspace 之外(${bad})。写文件只在 workspace/。` }; + } + const forbidden = CMD_POSITION.exec(cmd); + if (forbidden) { + log("deny-bash-cmd", cmd.slice(0, 500)); + return { block: true, reason: `path-guard: 不要直接调用 ${forbidden[1]}。编译/上板/评测走 bench stage run(它会选对 venv、拿资源池、落日志)。只读地 grep 这个词是允许的。` }; + } + const badTarget = badWriteTarget(cmd); + if (badTarget) { + log("deny-bash-write", cmd.slice(0, 500)); + return { block: true, reason: `path-guard: 写到 workspace 之外(${badTarget})。读外面的文件可以,写只能在 workspace/(或 /tmp)。` }; + } + for (const rx of WRITE_PATTERNS) { + if (rx.test(cmd)) { + log("deny-bash", cmd.slice(0, 500)); + return { block: true, reason: `path-guard: 命令被拒(${rx})。重阶段用 bench stage run,写文件只在 workspace/。` }; + } + } + } + return undefined; + }); +} diff --git a/agent/models.json.tmpl b/agent/models.json.tmpl new file mode 100644 index 0000000..80be7af --- /dev/null +++ b/agent/models.json.tmpl @@ -0,0 +1,37 @@ +{ + "_comment": "渲染到 ~/.pi/agent/models.json(chmod 600)。{TEACHER_API_KEY}/{STUDENT_*} 由 `bench agent setup` 从 bench/.env.local 填。字段名对照 pi 版本文档(P3 核对:providers..{baseUrl, api, apiKey, models[]})。", + "providers": { + "dr-gateway": { + "baseUrl": "https://ai-api.d-robotics.cc/v1", + "api": "openai-completions", + "apiKey": "{TEACHER_API_KEY}", + "models": [ + { + "id": "glm-5.3-flash", + "name": "GLM 5.3 Flash (teacher)", + "reasoning": false, + "input": ["text"], + "contextWindow": 128000, + "maxTokens": 16384, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 } + } + ] + }, + "student-vllm": { + "baseUrl": "{STUDENT_BASE_URL}", + "api": "openai-completions", + "apiKey": "{STUDENT_API_KEY}", + "models": [ + { + "id": "{STUDENT_MODEL}", + "name": "student (vLLM)", + "reasoning": false, + "input": ["text"], + "contextWindow": 65536, + "maxTokens": 8192, + "cost": { "input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0 } + } + ] + } + } +} diff --git a/agent/oe-doc.sh b/agent/oe-doc.sh new file mode 100755 index 0000000..efa12c4 --- /dev/null +++ b/agent/oe-doc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# oe-doc.sh — 只读文档检索。P3 接到 oe-mcp 的 CLI;现在退化为在 .horizon 里 grep。 +set -uo pipefail +Q="${1:-}" +[ -z "$Q" ] && { echo "usage: oe-doc.sh "; exit 2; } +ROOT="${BENCH_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}" +HZ="$ROOT/../.horizon" +if command -v oe-mcp-cli >/dev/null 2>&1; then + exec oe-mcp-cli search "$Q" +fi +grep -rIl --include='*.md' -i -- "$Q" "$HZ" 2>/dev/null | head -8 | while read -r f; do + echo "=== $f" + grep -n -i -B2 -A8 -- "$Q" "$f" | head -60 +done diff --git a/agent/prompts/e2e-occ.md b/agent/prompts/e2e-occ.md new file mode 100644 index 0000000..813b874 --- /dev/null +++ b/agent/prompts/e2e-occ.md @@ -0,0 +1,18 @@ +# 任务:把 {model}(3D 占据)编译到 {march}({backend}),本轮目标是编译通过 + 板端可运行,不评精度 + +job:`{job_id}` run:`{run_id}` +job.json:`{job_json}`(只读) 你的可写区:当前目录 + +## 本轮门 +- G1 compile:`.hbm` 产出、march = {march}、`hb_verifier` 通过 +- G2 numeric:余弦 ≥ {g2_threshold}(校准来源为 synthetic 时只作参考) +- G3:本轮不评(na),不要尝试跑评测 +- G4:板端 perf 照常测,这是本轮最有价值的数字 + +## 校准输入 +harness 按 repo_demo → nuscenes_128 → synthetic 三级回退,`bench stage run calibrate` 的输出会告诉你用了哪级。你可以在 workspace 里自己准备更好的校准 npy 并在 compile.yaml 里指向它(必须在 workspace 内)。 + +## 预期 +BEV pooling / 体素化 / 稀疏卷积类算子在 {march} 上很可能不支持。`.horizon` 没有 occ 专用 skill。如果三种不同方法(改导出、拆算子、改输入规格)都过不了编译,报 BLOCKED 并给出 error_class 与具体算子名——这本身就是有价值的结果。 + +预算:{max_turns} 轮、{max_tokens} tokens;剩 5 轮会收到提醒,那时先 claim。 diff --git a/agent/prompts/e2e.md b/agent/prompts/e2e.md new file mode 100644 index 0000000..91dff7f --- /dev/null +++ b/agent/prompts/e2e.md @@ -0,0 +1,21 @@ +# 任务:把 {model}({task})量化并编译到 {march}({backend} / {qcfg}),通过全部必选门 + +job:`{job_id}` run:`{run_id}` +job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的可写区:当前目录 + +## 目标 +1. G1 compile:产出 `.hbm`,march = {march},输入 {input_spec},`hb_verifier` 通过 +2. G2 numeric:校准外样本逐输出余弦 ≥ {g2_threshold} +3. G3 task:{g3_metric} 在 T1 上相对 FP32 基线退化不超过 {g3_tol}({g3_kind}) +4. G4 perf:只记录,不判 + +## 你手上有什么 +- FP32 ONNX:`{onnx}`(03-export 已产出) +- FP32 基线:`{baseline_dir}/metrics.json` +- 校准集:`{calib_dir}`({calib_n} 个 npy,与 val 零交集) +- T1 评测子集:`{t1_tar}`({t1_n} 张) + +## 流程 +先读 job.json 和 `.horizon/skills/` 里与 {task} / {backend} 相关的 skill,再决定编译配置。提交 `bench stage run compile --config workspace/<你的>.yaml`,`bench stage wait`。失败就读日志改配置;同一方法最多重试 1 次。编译过了跑 `bench stage run hbm`,再 `bench stage run eval`。最后调用 `claim`。 + +预算:{max_turns} 轮、{max_tokens} tokens。等待编译不消耗预算。剩 5 轮会收到提醒——那时先 claim,别等到耗尽。 diff --git a/agent/prompts/repair.md b/agent/prompts/repair.md new file mode 100644 index 0000000..a4928a9 --- /dev/null +++ b/agent/prompts/repair.md @@ -0,0 +1,15 @@ +# 任务:修复一次失败的量化编译 + +job:`{job_id}` run:`{run_id}` 目标 {march} / {backend} / {qcfg} +job.json:`{job_json}`(只读) 你的可写区:当前目录 + +## 失败现场(来自确定性流水线 oracle,不是别人的尝试) +- 失败阶段:`{oracle_stage_failed}` +- error_class:`{oracle_error_class}` +- 日志:`{oracle_stage_dir}/stderr.log`、`stdout.log`(只读) +- oracle 用的编译配置:`{oracle_compile_yaml}`(只读,复制到 workspace 再改) + +## 要做的 +读懂失败原因,改配置或改模型导出方式,重新提交 `bench stage run compile --config workspace/<你的>.yaml`,直到 G1 通过,然后跑 `hbm` 与 `eval`。同一方法最多重试 1 次;三种方法都不行 → BLOCKED,说明你认为 oracle 的失败是否为工具链固有限制。 + +最后调用 `claim`。预算:{max_turns} 轮、{max_tokens} tokens;剩 5 轮会收到提醒,那时先 claim。 diff --git a/agent/settings.json.tmpl b/agent/settings.json.tmpl new file mode 100644 index 0000000..3e38328 --- /dev/null +++ b/agent/settings.json.tmpl @@ -0,0 +1,17 @@ +{ + "_comment": "渲染到 /workspace/.pi/settings.json。{PROJECT_ROOT}/{BENCH_ROOT}/{MODEL} 由 harness/rollout.py 替换。compaction 关:训练数据里出现 CompactionSummaryMessage 是脏的,上下文靠 max_turns 控制。", + "defaultProvider": "dr-gateway", + "defaultModel": "{MODEL}", + "compaction": { "enabled": false }, + "skills": [ + "{PROJECT_ROOT}/.horizon/skills/*/*", + "{PROJECT_ROOT}/.horizon/skills/horizon-router" + ], + "extensions": [ + "{BENCH_ROOT}/agent/extensions/path-guard.ts", + "{BENCH_ROOT}/agent/extensions/claim-card.ts", + "{BENCH_ROOT}/agent/extensions/budget-kill.ts", + "{BENCH_ROOT}/agent/extensions/oe-doc.ts" + ], + "tools": ["read", "edit", "write", "bash", "grep"] +} diff --git a/bin/bench b/bin/bench new file mode 100755 index 0000000..75cb523 --- /dev/null +++ b/bin/bench @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# bench CLI 入口。harness 只依赖 pyyaml;工具链在各自 venv 里由 harness 子进程调用。 +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export BENCH_ROOT="${BENCH_ROOT:-$(dirname "$HERE")}" +export PYTHONPATH="$BENCH_ROOT${PYTHONPATH:+:$PYTHONPATH}" +PY="${BENCH_PYTHON:-python3}" +exec "$PY" -m harness.cli "$@" diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..d9cf557 --- /dev/null +++ b/config.yaml @@ -0,0 +1,81 @@ +# bench/config.yaml — 运行时配置(不进 job.json;job.json 只快照与结果相关的东西) +version: 1 + +paths: + # 项目根;harness 默认取本文件所在目录的上一级 + project_root: null + # 两套 venv。harness 按 backend 选,agent 无权改 + venv: + ptq: ~/toolchains/oe-3.7.0/venv # hb_compile / hb_model_info / hb_verifier(无 pip,不装东西) + qat: ~/toolchains/oellm2/venv # plugin + hbdk4 4.12 + hbm_infer(不装东西) + bench: ~/toolchains/bench-venv # 我们自己的:torch/torchvision/ultralytics/pycocotools/pyarrow — 权重、导 ONNX、FP32 基线、评测打分、数据入池 + # harness 自身用的解释器(只需 pyyaml),默认 python3 + harness_python: python3 + # hb_compile 的 working_dir 根目录(中间产物几百 MB/个)。必须放本地盘: + # 实测 JuiceFS 写 1 MB=360 ms vs 本地盘 3 ms,大模型编译会卡在 D 状态被看门狗误判挂死。 + # 留空则退回 runs///oracle/stages/05-compile/work(JuiceFS 上,只适合小规模)。 + work_root: /home/ming01.wang/bench-work + # .horizon skill pack(只读) + horizon: ../.horizon + # agent 运行时(scripts/setup_agent.sh → ~/toolchains/node;pi 是 node 脚本,node 必须在 PATH 或这里给全路径) + pi_bin: ~/toolchains/node/bin/pi + node_bin: ~/toolchains/node/bin/node + +pools: + # 机器是共享的:2026-09-14 实测 8 张卡,4/5/7 被别人跑满、6 占了 29 GB;0–3 空闲。 + # 槽位 i → devices[i] → CUDA_VISIBLE_DEVICES。别人的卡不碰。 + # reserve_for_agent:只给 rollout 的 `bench stage run` 用的槽位,oracle 拿不到。 + # 实测(2026-09-15):两个 oracle 调度进程把 cpu 8/8 占满,第一条 rollout 的编译在队列里饿了 60 分钟。 + gpu: { workers: 4, devices: [0, 1, 2, 3], reserve_for_agent: 1 } # baseline / eval / qat-calibrate + cpu: { workers: 8, reserve_for_agent: 2 } # export / hb_compile + board: { workers: 1 } # 一块板 = 1,严格串行,带锁(不预留:agent 与 oracle 都得排队) + agent: { workers: 3, rpm: 60 } # 受 LLM 端点限速 + +watchdog: + stall_s: 900 # 日志不增长 且 进程树 CPU < cpu_idle_pct 持续 stall_s → STALLED + cpu_idle_pct: 5 + poll_s: 30 + hard_ceiling_s: 86400 # 24h 纯保险 + soft_alert: p95x3 # 超过历史 p95 × 3 只告警,不杀 + +perf: + warmup: 20 + runs: 200 + stats: [mean, p95] + +rollouts: + k: 4 + max_turns: 40 + max_tokens: 200000 + hard_ceiling_s: 43200 # 12h 只防 agent 进程挂死 + # --tools 是白名单,同时作用于内置工具与 extension 注册的工具。 + # 踩过(2026-09-15 第一条轨迹):claim / oe_doc 没列进来 → claim-card 注册了工具但被白名单关掉, + # agent 全程无法 claim,213 次 bash 调用一次 claim 都没有。extension 工具必须显式列出。 + tools: [read, edit, write, bash, grep, claim, oe_doc] + +retry: + transient: 2 # oom / board_offline + stalled: 1 # 进 long 车道 + deterministic: 0 + +scheduling: + priority_order: [A, B, C] + c_lane: low # C 类走独立低优先级车道 + estimate_source: history_then_params + +data: + default_tier: t1 + calib_n_default: 128 + +# LLM 端点。key 不写这里,写 bench/.env.local(gitignore) +endpoints: + teacher: + provider: openai-compatible + base_url: https://ai-api.d-robotics.cc/v1 + model: glm-5.3-flash + api_key_env: TEACHER_API_KEY + student: + provider: openai-compatible + base_url: null # vLLM 起来后填 + model: null + api_key_env: STUDENT_API_KEY diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/export/redact.py b/export/redact.py new file mode 100644 index 0000000..b0e47f5 --- /dev/null +++ b/export/redact.py @@ -0,0 +1,115 @@ +"""脱敏。规则移植自 data_collect/redact_archive.py 的思路,不 import 那边任何东西。 + +agent 的 bash 输出会进轨迹:用户目录、主机名、IP、密钥、token、邮箱都要抹。 +同一条轨迹内占位符稳定(同一个用户名每次映射到同一个 ),保证轨迹可读。 +""" +from __future__ import annotations + +import json +import os +import re +import sys +from pathlib import Path + +RULES: list[tuple[str, re.Pattern, str]] = [ + ("api_key", re.compile(r"\bsk-[A-Za-z0-9_\-]{10,}\b"), ""), + ("hf_token", re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"), ""), + ("bearer", re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._\-]{16,}"), r"\1"), + ("kv_secret", re.compile(r"(?i)\b((?:api[_-]?key|token|secret|passw(?:or)?d|authorization)\s*[=:]\s*)['\"]?[^\s'\"]{6,}"), r"\1"), + ("ssh_target", re.compile(r"\b[a-z][a-z0-9._-]{0,31}@(?:\d{1,3}(?:\.\d{1,3}){3}|[a-z0-9.-]+\.[a-z]{2,})\b"), ""), + ("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), ""), + ("ipv4", re.compile(r"\b(?!(?:0|127|10|192\.168)\.)\d{1,3}(?:\.\d{1,3}){3}\b"), ""), + ("hostname_dr", re.compile(r"\b[a-z0-9-]+\.d-robotics\.cc\b"), ""), + ("instance", re.compile(r"\binstance-[a-z0-9]{6,}\b"), ""), +] +USER_DIR = re.compile(r"(/mnt/data/|/home/)([A-Za-z0-9][A-Za-z0-9._-]{1,31})(?=/|\b)") + + +class Redactor: + def __init__(self): + self.users: dict[str, str] = {} + self.counts: dict[str, int] = {} + + def _user(self, m: re.Match) -> str: + name = m.group(2) + if name not in self.users: + self.users[name] = f"" + self.counts["user_dir"] = self.counts.get("user_dir", 0) + 1 + return m.group(1) + self.users[name] + + def text(self, s: str) -> str: + if not isinstance(s, str) or not s: + return s + s = USER_DIR.sub(self._user, s) + for name, rx, rep in RULES: + s, n = rx.subn(rep, s) + if n: + self.counts[name] = self.counts.get(name, 0) + n + return s + + def obj(self, o): + if isinstance(o, str): + return self.text(o) + if isinstance(o, list): + return [self.obj(x) for x in o] + if isinstance(o, dict): + return {k: self.obj(v) for k, v in o.items()} + return o + + # ---- 第二遍:裸用户名 + # 第一遍只抹路径里的用户名(/mnt/data/、/home/)。但 agent 的 bash 输出里到处是 + # `ls -la` 的属主/属组列("drwxr-xr-x 4 ming01.wang ming01.wang")、`whoami`、进程表的 USER 列—— + # 那些不是路径,第一遍抓不到。实测第一条轨迹:路径替换 64 处,裸用户名仍残留 62 处。 + def bare_user_regex(self, extra: set[str] | None = None) -> re.Pattern | None: + names = set(self.users) | (extra or set()) + names = {n for n in names if len(n) >= 3 and not n.isdigit()} + if not names: + return None + return re.compile(r"(? str: + def rep(m: re.Match) -> str: + name = m.group(1) + if name not in self.users: + self.users[name] = f"" + self.counts["user_bare"] = self.counts.get("user_bare", 0) + 1 + return self.users[name] + return rx.sub(rep, s) + if isinstance(o, str): + return sub(o) + if isinstance(o, list): + return [self.bare_pass(x, rx) for x in o] + if isinstance(o, dict): + return {k: self.bare_pass(v, rx) for k, v in o.items()} + return o + + +def local_usernames() -> set[str]: + """本机可能出现的用户名:当前用户 + /home 下的目录名。用于裸用户名替换的种子。""" + out = set() + for k in ("USER", "LOGNAME"): + v = os.environ.get(k) + if v: + out.add(v) + try: + out |= {p.name for p in Path("/home").iterdir() if p.is_dir()} + except OSError: + pass + return {n for n in out if len(n) >= 3} + + +def redact_messages(messages: list[dict]) -> tuple[list[dict], dict]: + r = Redactor() + out = r.obj(messages) + rx = r.bare_user_regex(local_usernames()) + if rx is not None: + out = r.bare_pass(out, rx) + return out, {"counts": r.counts, "users": len(r.users)} + + +if __name__ == "__main__": + for p in sys.argv[1:]: + d = json.loads(Path(p).read_text()) + out, stats = redact_messages(d if isinstance(d, list) else [d]) + print(json.dumps({"file": p, **stats}, ensure_ascii=False)) diff --git a/export/reward.py b/export/reward.py new file mode 100644 index 0000000..a0a6c3c --- /dev/null +++ b/export/reward.py @@ -0,0 +1,79 @@ +"""reward v0 配方。离线算,只读 verdict.json;配方版本化,可全量重打分。 + +r = 1.0·[G1] + 1.0·[G2] + 2.0·[G3] 绝对门 +r += 1.0·clip((Δ_agent − Δ_oracle)/|tol|, −1, +1) 相对 oracle 的精度 +r += 0.5·clip((lat_oracle − lat_agent)/lat_oracle, −1, +1) 相对 oracle 的延时 +r −= 0.5·turns/max_turns + 0.5·tokens/max_tokens 代价 +r −= 2.0·[claim ≠ verdict] 说谎 +r −= 0.25·[claim 缺席] +BLOCKED: +1.0 若 oracle 同阶段同类失败;否则 −2.0 +TIMEOUT: −1.0 +G3 = na:其 2.0 权重按比例分给 G1/G2(各 2.0),精度相对项取 0 +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +RECIPE_VERSION = "v0.1" +PASSING = {"pass", "pass-screen", "pass-weak"} +W = {"G1": 1.0, "G2": 1.0, "G3": 2.0, "acc_rel": 1.0, "lat_rel": 0.5, "cost_turns": 0.5, "cost_tokens": 0.5, + "lie": 2.0, "claim_absent": 0.25, "blocked_ok": 1.0, "blocked_bad": 2.0, "timeout": 1.0} + + +def clip(x: float, lo: float = -1.0, hi: float = 1.0) -> float: + return max(lo, min(hi, x)) + + +def compute(verdict: dict, max_turns: int = 40, max_tokens: int = 200000) -> dict: + terms: dict[str, float] = {} + g = verdict.get("gates") or {} + terminal = verdict.get("terminal") + claim = verdict.get("claim") or {} + oref = verdict.get("oracle_ref") or {} + cost = verdict.get("cost") or {} + + if terminal == "TIMEOUT": + terms["timeout"] = -W["timeout"] + elif terminal == "BLOCKED": + terms["blocked"] = W["blocked_ok"] if verdict.get("blocked_correct") else -W["blocked_bad"] + else: + g3_na = (g.get("G3") or {}).get("verdict") == "na" + w1, w2, w3 = (2.0, 2.0, 0.0) if g3_na else (W["G1"], W["G2"], W["G3"]) + terms["G1"] = w1 * float((g.get("G1") or {}).get("verdict") in PASSING) + terms["G2"] = w2 * float((g.get("G2") or {}).get("verdict") in PASSING) + terms["G3"] = w3 * float((g.get("G3") or {}).get("verdict") in PASSING) + # 相对 oracle 的精度(都用同一 kind 的 delta;lower-is-better 已在 gates.g3 里翻转成"越大越好") + g3 = g.get("G3") or {} + if not g3_na and g3.get("delta") is not None and oref.get("G3_delta") is not None and g3.get("tolerance"): + terms["acc_rel"] = W["acc_rel"] * clip((g3["delta"] - oref["G3_delta"]) / abs(g3["tolerance"])) + else: + terms["acc_rel"] = 0.0 + g4 = g.get("G4") or {} + if g4.get("latency_ms_mean") and oref.get("latency_ms_mean") and not g4.get("contended"): + terms["lat_rel"] = W["lat_rel"] * clip((oref["latency_ms_mean"] - g4["latency_ms_mean"]) / oref["latency_ms_mean"]) + else: + terms["lat_rel"] = 0.0 + + turns, tokens = cost.get("turns") or 0, cost.get("tokens") or 0 + if cost.get("tokens_reported") is False or tokens == 0: + # 端点不回 usage(dr-gateway 实测全 0):代价项只按 turns 算,权重合并,别把"没报"当成"很省" + terms["cost"] = -((W["cost_turns"] + W["cost_tokens"]) * min(1.0, turns / max_turns)) + terms_note = "tokens unavailable" + else: + terms["cost"] = -(W["cost_turns"] * min(1.0, turns / max_turns) + W["cost_tokens"] * min(1.0, tokens / max_tokens)) + terms_note = None + if claim.get("terminal") == "ABSENT" or not claim: + terms["claim_absent"] = -W["claim_absent"] + elif verdict.get("claim_consistent") is False: + terms["lie"] = -W["lie"] + r = sum(terms.values()) + return {"reward": round(r, 4), "terms": {k: round(v, 4) for k, v in terms.items()}, "recipe": RECIPE_VERSION, + "terminal": terminal, "review": bool(verdict.get("review")), "note": terms_note} + + +if __name__ == "__main__": + for p in sys.argv[1:]: + v = json.loads(Path(p).read_text()) + print(p, json.dumps(compute(v), ensure_ascii=False)) diff --git a/export/to_agentrl.py b/export/to_agentrl.py new file mode 100644 index 0000000..d8d3cfc --- /dev/null +++ b/export/to_agentrl.py @@ -0,0 +1,259 @@ +"""L4 数据物化:扫描 rollouts → 脱敏、清洗(R1–R8)、线性化 → 与 verdict / oracle 配对 → reward → datasets//。 + + python export/to_agentrl.py --runs r20260914-01 [r…] [--policy teacher] [--include-timeout] + +清洗规则(移植自 data_collect/clean_for_agentrl.py 的经验,独立实现): + R1 双方言归一:OpenAI / Anthropic 形状 → {role, content, tool_calls?, tool_call_id?} + R2 剥离 thinking / reasoning 块 + R3 system 只在首条且只有一条(多条合并) + R4 截断悬空尾轮:结尾的 assistant tool_call 没有对应 tool 结果 → 连同该 assistant 一起去掉 + R5 去空消息 + R6 合并相邻同 role 的纯文本消息 + R7 tool 结果截断(头 6k + 尾 2k 字符) + R8 丢弃:无 assistant 消息,或 turns < 2 +轨迹源:events.jsonl 最后一条 agent_end.messages[](已线性化);session.jsonl 是树,只做备份。 +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +BENCH = HERE.parent +sys.path.insert(0, str(HERE)) +from redact import redact_messages # noqa: E402 +from reward import compute as compute_reward, RECIPE_VERSION # noqa: E402 + +TOOL_HEAD, TOOL_TAIL = 6000, 2000 + + +# ---------------------------------------------------------------- 轨迹提取 + +def system_prompt(rd: Path) -> tuple[str, str]: + """取这条轨迹的系统提示 → (text, source)。 + + pi 的 --append-system-prompt 内容**不出现在** agent_end.messages[] 里(实测 0.73.1): + 导出时若不补回来,SFT 样本就只有 user 任务 + assistant 动作,没有"只能写 workspace / 必须 claim"这些规则, + 等于训练一个看不见规则的策略。优先读 rollout 自己的快照,其次按 cmd.json 里的路径读当前文件(会漂移,故记 source)。 + """ + snap = rd / "agent" / "system.md" + if snap.exists(): + return snap.read_text(errors="replace"), "snapshot" + cmd = rd / "agent" / "cmd.json" + if cmd.exists(): + try: + argv = json.loads(cmd.read_text())["cmd"] + i = argv.index("--append-system-prompt") + f = Path(argv[i + 1]) + if f.exists(): + return f.read_text(errors="replace"), "live-file" + except (ValueError, KeyError, IndexError, OSError): + pass + return "", "missing" + + +def load_messages(rd: Path) -> list[dict] | None: + ev = rd / "agent" / "events.jsonl" + if not ev.exists(): + return None + last = None + for line in ev.read_text(errors="replace").splitlines(): + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + if e.get("type") == "agent_end" and isinstance(e.get("messages"), list): + last = e["messages"] + return last + + +# ---------------------------------------------------------------- R1–R8 + +def _text_of(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [] + for c in content: + if isinstance(c, dict): + t = c.get("type") + if t in ("text", "output_text") and c.get("text"): + parts.append(c["text"]) + elif t in ("thinking", "reasoning", "redacted_thinking"): + continue # R2 + elif isinstance(c, str): + parts.append(c) + return "\n".join(parts) + return "" if content is None else str(content) + + +def r1_normalize(m: dict) -> list[dict]: + """一条原始消息 → 0..n 条归一消息(Anthropic 的 tool_use/tool_result 混在 content 里要拆)。""" + role = m.get("role") + content = m.get("content") + out: list[dict] = [] + if role in ("toolResult", "tool_result", "tool"): + return [{"role": "tool", "tool_call_id": m.get("toolCallId") or m.get("tool_call_id"), + "content": _text_of(content) or json.dumps(m.get("output") or m.get("result") or "", ensure_ascii=False)}] + if role == "assistant": + tool_calls = [] + if isinstance(content, list): + for c in content: + if isinstance(c, dict) and c.get("type") in ("toolCall", "tool_use", "function_call"): + tool_calls.append({"id": c.get("id"), "name": c.get("name"), + "arguments": c.get("arguments") if "arguments" in c else c.get("input")}) + for tc in m.get("tool_calls") or []: + fn = tc.get("function") or {} + tool_calls.append({"id": tc.get("id"), "name": fn.get("name") or tc.get("name"), + "arguments": fn.get("arguments") if fn else tc.get("arguments")}) + msg = {"role": "assistant", "content": _text_of(content)} + if tool_calls: + msg["tool_calls"] = tool_calls + return [msg] + if role == "custom": + # pi 扩展塞进会话的消息(如 budget-kill 的预算提醒):模型是当 user 输入看到的,训练样本也要有 + t = _text_of(content) + return [{"role": "user", "content": f"[{m.get('customType', 'extension')}] {t}"}] if t else [] + if role in ("user", "system"): + if isinstance(content, list) and any(isinstance(c, dict) and c.get("type") == "tool_result" for c in content): + for c in content: + if isinstance(c, dict) and c.get("type") == "tool_result": + out.append({"role": "tool", "tool_call_id": c.get("tool_use_id"), "content": _text_of(c.get("content"))}) + else: + t = _text_of([c]) + if t: + out.append({"role": "user", "content": t}) + return out + return [{"role": role, "content": _text_of(content)}] + return [] + + +def clean(messages: list[dict]) -> tuple[list[dict], dict]: + stats = {"in": len(messages)} + msgs: list[dict] = [] + for m in messages: + msgs.extend(r1_normalize(m)) + # R3 system 合并到首条 + systems = [m["content"] for m in msgs if m["role"] == "system"] + msgs = [m for m in msgs if m["role"] != "system"] + if systems: + msgs.insert(0, {"role": "system", "content": "\n\n".join(s for s in systems if s)}) + # R5 去空 + msgs = [m for m in msgs if m.get("content") or m.get("tool_calls")] + # R7 tool 结果截断 + for m in msgs: + if m["role"] == "tool" and len(m["content"]) > TOOL_HEAD + TOOL_TAIL: + m["content"] = m["content"][:TOOL_HEAD] + f"\n…[truncated {len(m['content']) - TOOL_HEAD - TOOL_TAIL} chars]…\n" + m["content"][-TOOL_TAIL:] + stats["r7_truncated"] = stats.get("r7_truncated", 0) + 1 + # R6 合并相邻同 role 纯文本 + merged: list[dict] = [] + for m in msgs: + if merged and m["role"] == merged[-1]["role"] and m["role"] in ("user", "assistant") \ + and not m.get("tool_calls") and not merged[-1].get("tool_calls"): + merged[-1]["content"] += "\n" + m["content"] + stats["r6_merged"] = stats.get("r6_merged", 0) + 1 + else: + merged.append(m) + msgs = merged + # R4 悬空尾轮 + while msgs and msgs[-1]["role"] == "assistant" and msgs[-1].get("tool_calls"): + msgs.pop(); stats["r4_dangling"] = stats.get("r4_dangling", 0) + 1 + while msgs and msgs[-1]["role"] in ("user", "tool") and len(msgs) > 2: + msgs.pop(); stats["r4_trailing_user"] = stats.get("r4_trailing_user", 0) + 1 + # R8 + n_assist = sum(m["role"] == "assistant" for m in msgs) + stats.update({"out": len(msgs), "assistant_turns": n_assist, "drop": n_assist == 0 or len(msgs) < 3}) + return msgs, stats + + +# ---------------------------------------------------------------- 主流程 + +def scan(run_ids: list[str]) -> list[Path]: + out = [] + for rid in run_ids: + for v in (BENCH / "runs" / rid).glob("*/rollouts/*/verdict.json"): + out.append(v.parent) + return sorted(out) + + +def next_dataset_id() -> str: + d = time.strftime("%Y%m%d") + existing = sorted((BENCH / "datasets").glob(f"ds{d}-*")) + return f"ds{d}-{len(existing) + 1:02d}" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--runs", nargs="+", required=True) + ap.add_argument("--policy", choices=["teacher", "student"]) + ap.add_argument("--include-timeout", action="store_true") + ap.add_argument("--max-turns", type=int, default=40) + ap.add_argument("--max-tokens", type=int, default=200000) + ap.add_argument("--dataset-id") + a = ap.parse_args() + + ds_id = a.dataset_id or next_dataset_id() + out_dir = BENCH / "datasets" / ds_id + out_dir.mkdir(parents=True, exist_ok=True) + files = {"teacher": open(out_dir / "teacher.jsonl", "w"), "student": open(out_dir / "student.jsonl", "w")} + stats = {"scanned": 0, "written": 0, "dropped": {}, "review": 0, "by_terminal": {}} + for rd in scan(a.runs): + stats["scanned"] += 1 + verdict = json.loads((rd / "verdict.json").read_text()) + rj = json.loads((rd / "rollout.json").read_text()) if (rd / "rollout.json").exists() else {} + policy = verdict.get("policy") or rj.get("policy") or "teacher" + if a.policy and policy != a.policy: + continue + if verdict["terminal"] == "TIMEOUT" and not a.include_timeout: + stats["dropped"]["timeout"] = stats["dropped"].get("timeout", 0) + 1; continue + raw = load_messages(rd) + if not raw: + stats["dropped"]["no_trajectory"] = stats["dropped"].get("no_trajectory", 0) + 1; continue + msgs, cst = clean(raw) + sys_text, sys_src = system_prompt(rd) + if sys_text: + if msgs and msgs[0]["role"] == "system": + msgs[0]["content"] = sys_text + "\n\n" + msgs[0]["content"] + else: + msgs.insert(0, {"role": "system", "content": sys_text}) + cst["system_source"] = sys_src + if cst["drop"]: + stats["dropped"]["r8"] = stats["dropped"].get("r8", 0) + 1; continue + msgs, rst = redact_messages(msgs) + rw = compute_reward(verdict, a.max_turns, a.max_tokens) + jd = rd.parent.parent + job = json.loads((jd / "job.json").read_text()) + rec = { + "id": hashlib.sha1(f"{verdict['run_id']}/{verdict['job_id']}/{verdict['rollout']}".encode()).hexdigest()[:16], + "run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"], + "mode": verdict.get("mode") or rj.get("mode"), "policy": policy, "model": rj.get("model"), + "task": job["task"], "march": job["march"], "backend": job["backend"], + "messages": msgs, + "verdict": {k: verdict[k] for k in ("terminal", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict}, + "gates": {g: (verdict["gates"].get(g) or {}).get("verdict") for g in ("G1", "G2", "G3", "G4")}, + "oracle_ref": verdict.get("oracle_ref"), + "reward": rw["reward"], "reward_terms": rw["terms"], "reward_recipe": rw["recipe"], + "cost": verdict.get("cost"), + "system_source": sys_src, + "system_sha1": hashlib.sha1(sys_text.encode()).hexdigest()[:12] if sys_text else None, + "clean_stats": cst, "redact_stats": rst, + } + files[policy].write(json.dumps(rec, ensure_ascii=False) + "\n") + stats["written"] += 1 + stats["review"] += int(bool(verdict.get("review"))) + stats["by_terminal"][verdict["terminal"]] = stats["by_terminal"].get(verdict["terminal"], 0) + 1 + for f in files.values(): + f.close() + manifest = {"dataset_id": ds_id, "runs": a.runs, "reward_recipe": RECIPE_VERSION, "created_at": time.time(), + "filters": {"policy": a.policy, "include_timeout": a.include_timeout}, "stats": stats} + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False)) + print(json.dumps(manifest, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/__init__.py b/harness/__init__.py new file mode 100644 index 0000000..c322ffa --- /dev/null +++ b/harness/__init__.py @@ -0,0 +1,17 @@ +"""bench harness — L1 oracle 流水线 + L3 verdict。 + +模块: + paths 根目录、config.yaml、.env.local + registry registry/* 加载、matrix 展开、job_id、job.json 快照 + db state/jobs.sqlite 唯一真相 + errors stderr → error_class + watchdog 无固定超时的子进程执行器 + gates G1–G4 → verdict + stages 01–08,每阶段一个子目录落 stdout/stderr/exit/meta + backends ptq / qat:按 backend 选 venv,agent 无权改 + scheduler 资源池、板卡锁、耗时估算 + pipeline oracle 主流程 + preflight + stagerun agent 可用的 `bench stage run / wait` + cli `bench` 命令 +""" +__version__ = "0.4.0" diff --git a/harness/adapters/__init__.py b/harness/adapters/__init__.py new file mode 100644 index 0000000..58b59f8 --- /dev/null +++ b/harness/adapters/__init__.py @@ -0,0 +1,2 @@ +"""custom 模型的构建适配器:每个模型一个 .py,提供 build(weights: Path, input_spec: str) -> nn.Module。 +非 torchvision / ultralytics / timm 的模型走这里。""" diff --git a/harness/backends/__init__.py b/harness/backends/__init__.py new file mode 100644 index 0000000..f15db27 --- /dev/null +++ b/harness/backends/__init__.py @@ -0,0 +1,12 @@ +"""按 backend 选 venv 与命令模板。harness 的职责;agent 无权改。""" +from __future__ import annotations + +from . import ptq, qat + +BACKENDS = {"ptq": ptq, "qat": qat} + + +def get(backend: str): + if backend not in BACKENDS: + raise KeyError(f"未知 backend: {backend}") + return BACKENDS[backend] diff --git a/harness/backends/ptq.py b/harness/backends/ptq.py new file mode 100644 index 0000000..3fc073d --- /dev/null +++ b/harness/backends/ptq.py @@ -0,0 +1,111 @@ +"""PTQ 后端:~/toolchains/oe-3.7.0/venv(hbdk4 4.7.5 · horizon_tc_ui 3.5.3 · hmct 2.6.5)。 + +流程:ONNX → hb_compile(含校准)→ .hbm → hb_model_info / hb_verifier。 +字段名对照 OE 3.7 horizon_tc_ui 文档;P2 切片时逐项核对,核对完把 VERIFIED 改 True。 +""" +from __future__ import annotations + +from pathlib import Path + +import yaml + +from .. import paths + +NAME = "ptq" +VERIFIED = False # P2:hb_compile yaml 字段与工具名核对后置 True + +TOOLS = { + "compile": "hb_compile", + "model_info": "hb_model_info", + "verifier": "hb_verifier", + "perf": "hb_perf", +} + + +def python() -> Path: + return paths.venv_python(NAME) + + +def tool(name: str) -> str: + return str(paths.venv_bin(NAME, TOOLS[name])) + + +def export_cmd(job: dict, onnx_out: Path, weights: Path) -> list[str]: + return [str(paths.venv_python("bench")), "-m", "harness.tools.export_onnx", # 导 ONNX 用 bench venv(torch+torchvision+ultralytics) + "--model", job["model"], "--source", job["model_spec"]["source"], + "--weights", str(weights), "--input-spec", job["model_spec"]["input_spec"], + "--out", str(onnx_out)] + + +def compile_config(job: dict, onnx: Path, calib_dir: Path, work_dir: Path, agent_overrides: dict | None = None) -> dict: + """hb_compile 的 yaml。agent 在 rollout 里可以提交自己的 yaml(bench stage run --config), + 但 march / working_dir / output prefix 由 harness 钉死,覆盖不了。""" + prefix = job["job_id"] + cfg = { + "model_parameters": { + "onnx_model": str(onnx), + "march": job["march"], + "working_dir": str(work_dir), + "output_model_file_prefix": prefix, + "layer_out_dump": False, + }, + "input_parameters": { + # 实测 hb_compile 3.5.3:norm_type / cal_data_type 已弃用(由 mean/scale 与 npy 决定);featuremap 输入让预处理留在主机侧,口径与评测器一致 + "input_type_rt": "featuremap", + "input_type_train": "featuremap", + "input_layout_train": "NCHW", + }, + "calibration_parameters": { + "cal_data_dir": str(calib_dir), # 只含 .npy 的目录,后缀 _f32(OE 约定) + "calibration_type": "default", + }, + "compiler_parameters": { + "compile_mode": "latency", + "optimize_level": "O2", + "debug": False, + "jobs": 8, + }, + } + if job["qcfg"] == "fp16int8": + # hmct quant_config(schema:calibration_parameters.quant_config: str|dict)。实测 3.5.3 不接受 model_parameters.quant_config。 + # 定义 fp16int8 = 主体 int8 + 敏感算子与模型输出 float16(nash-p 有 fp16 算力)。qtype 可选:int8/int16/float16/… + cfg["calibration_parameters"]["quant_config"] = { + "model_config": {"all_node_type": "int8", "model_output_type": "float16"}, + "op_config": {op: {"qtype": "float16"} for op in ("Softmax", "LayerNormalization", "Gelu", "Erf", "Exp", "Log", "Div", "Pow")}, + } + if agent_overrides: + # 允许 agent 改:input_parameters / calibration_parameters / compiler_parameters / quant_config + for sect in ("input_parameters", "calibration_parameters", "compiler_parameters"): + cfg[sect].update(agent_overrides.get(sect) or {}) + if agent_overrides.get("calibration_parameters", {}).get("quant_config") is not None: + cfg["calibration_parameters"]["quant_config"] = agent_overrides["calibration_parameters"]["quant_config"] + # 钉死项 + cfg["model_parameters"]["march"] = job["march"] + cfg["model_parameters"]["working_dir"] = str(work_dir) + cfg["model_parameters"]["output_model_file_prefix"] = prefix + cfg["model_parameters"]["onnx_model"] = str(onnx) + return cfg + + +def write_compile_config(cfg: dict, path: Path) -> Path: + path.write_text(yaml.safe_dump(cfg, allow_unicode=True, sort_keys=False)) + return path + + +def compile_cmd(yaml_path: Path) -> list[str]: + return [tool("compile"), "-c", str(yaml_path)] + + +def model_info_cmd(hbm: Path) -> list[str]: + return [tool("model_info"), str(hbm)] + + +def verifier_cmd(ptq_onnx: Path, bc: Path, sample_npy: Path) -> list[str]: + """hb_verifier 3.5.3:-m 逗号分隔的 onnx/bc/hbm,-i 输入文件。hbm 需要板 ip,x86 侧比 ptq_model.onnx vs quantized_model.bc。 + 输出逐层余弦表,rc=0;G1 取最后一行(模型输出)的余弦。""" + return [tool("verifier"), "-m", f"{ptq_onnx},{bc}", "-i", str(sample_npy)] + + +def infer_cmd(hbm: Path, inputs_npz: Path, out_npz: Path) -> list[str]: + """x86 侧用 hbm 推理拿输出张量(G2 与 G3 用)。P2 确认 PTQ venv 里是 hbm_infer 还是 hb_model_infer。""" + return [str(python()), "-m", "harness.tools.infer_hbm", "--hbm", str(hbm), "--inputs", str(inputs_npz), "--out", str(out_npz)] diff --git a/harness/backends/qat.py b/harness/backends/qat.py new file mode 100644 index 0000000..a6b50b9 --- /dev/null +++ b/harness/backends/qat.py @@ -0,0 +1,52 @@ +"""QAT 后端:~/toolchains/oellm2/venv(hbdk4 4.12.3a2 · horizon_plugin_pytorch 3.3.5 · hbm_infer 3.15.3)。 + +第一轮只做 calibration-only(plugin 的 calib 模式),不做训练。P4 之后再接。 +流程:torch 模型 → plugin prepare(calib) → export hbir → hbdk4 compile → .hbm。 +""" +from __future__ import annotations + +from pathlib import Path + +from .. import paths + +NAME = "qat" +VERIFIED = False + +TOOLS = { + "model_info": "hb_model_info", # P4 确认 hbdk4 4.12 里的工具名 + "verifier": "hb_verifier", +} + + +def python() -> Path: + return paths.venv_python(NAME) + + +def tool(name: str) -> str: + return str(paths.venv_bin(NAME, TOOLS[name])) + + +def export_cmd(job: dict, onnx_out: Path, weights: Path) -> list[str]: + # QAT 路径不需要 ONNX,但 03-export 阶段统一产出一份 ONNX 供 FP32 基线对齐 + return [str(paths.venv_python("bench")), "-m", "harness.tools.export_onnx", # 导 ONNX 用 bench venv(torch+torchvision+ultralytics) + "--model", job["model"], "--source", job["model_spec"]["source"], + "--weights", str(weights), "--input-spec", job["model_spec"]["input_spec"], + "--out", str(onnx_out)] + + +def calib_compile_cmd(job: dict, weights: Path, calib_dir: Path, work_dir: Path, agent_config: Path | None) -> list[str]: + cmd = [str(python()), "-m", "harness.tools.qat_calib_compile", + "--model", job["model"], "--weights", str(weights), "--march", job["march"], + "--calib", str(calib_dir), "--out", str(work_dir / f"{job['job_id']}.hbm"), + "--input-spec", job["model_spec"]["input_spec"]] + if agent_config: + cmd += ["--config", str(agent_config)] + return cmd + + +def model_info_cmd(hbm: Path) -> list[str]: + return [tool("model_info"), str(hbm)] + + +def infer_cmd(hbm: Path, inputs_npz: Path, out_npz: Path) -> list[str]: + return [str(python()), "-m", "harness.tools.infer_hbm", "--hbm", str(hbm), "--inputs", str(inputs_npz), "--out", str(out_npz)] diff --git a/harness/cli.py b/harness/cli.py new file mode 100644 index 0000000..dabe5df --- /dev/null +++ b/harness/cli.py @@ -0,0 +1,258 @@ +"""`bench` 命令。 + + bench preflight [--modes oracle,e2e] + bench new-run [--note ...] + bench expand [--run r…] [--modes …] + bench run [--run r…] [--job id …] [--dry-run] [--parallel N] + bench status [--run r…] [--fail] + bench show [--run r…] + bench annotate --note "…" + bench report [--run r…] + bench verify + bench stage run [--config path] (rollout 内,agent 用) + bench stage wait [--poll 60] + bench stage status +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from . import paths, db + + +def _run_id(con, given: str | None) -> str: + rid = given or db.latest_run_id(con) + if not rid: + raise SystemExit("没有 run:先 bench new-run") + return rid + + +def cmd_preflight(a) -> int: + from .pipeline import preflight + errs, warns = preflight(a.modes.split(",") if a.modes else None, oracle_only=a.oracle_only) + for w in warns: + print(f"WARN {w}") + for e in errs: + print(f"ERROR {e}") + print(f"preflight: {len(errs)} errors, {len(warns)} warnings") + return 1 if errs else 0 + + +def cmd_new_run(a) -> int: + from .pipeline import new_run + print(new_run(a.note or "")) + return 0 + + +def cmd_expand(a) -> int: + from .pipeline import expand + with db.connect() as con: + rid = _run_id(con, a.run) + print(json.dumps({"run_id": rid, **expand(rid, a.modes.split(",") if a.modes else None, a.marches.split(",") if a.marches else None)})) + return 0 + + +def cmd_run(a) -> int: + from .pipeline import run_oracle + with db.connect() as con: + rid = _run_id(con, a.run) + s = run_oracle(rid, a.job or None, dry_run=a.dry_run, parallel=a.parallel) + print(json.dumps({"run_id": rid, **s})) + return 0 + + +def cmd_status(a) -> int: + with db.connect() as con: + rid = _run_id(con, a.run) + rows = db.status_counts(con, rid) + print(f"run {rid}") + if a.fail: + for r in db.error_class_counts(con, rid): + print(f" {r['n']:4d} {r['error_class']:<20} {r['stage_failed'] or ''}") + for j in db.list_jobs(con, rid, state="FAIL"): + print(f" FAIL {j['job_id']:<50} {j['error_class'] or '':<18} {j['stage_failed'] or ''}") + return 0 + agg: dict = {} + for r in rows: + agg.setdefault(r["state"], {}).setdefault(r["task"], {})[r["march"]] = r["n"] + for st, tasks in agg.items(): + total = sum(sum(m.values()) for m in tasks.values()) + print(f" {st:<15} {total:5d}") + for t, m in tasks.items(): + print(f" {t:<8} " + " ".join(f"{k}={v}" for k, v in m.items())) + ro = con.execute("SELECT state, COUNT(*) n FROM rollouts WHERE run_id=? GROUP BY state", (rid,)).fetchall() + if ro: + print(" rollouts: " + " ".join(f"{r['state']}={r['n']}" for r in ro)) + from .scheduler import pool_usage, pool_size + from .registry import status_counts + print(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items())) + print(" registry: " + " ".join(f"{k}={v}" for k, v in status_counts().items()) + " (只有 ready 会被展开)") + return 0 + + +def cmd_show(a) -> int: + with db.connect() as con: + rid = _run_id(con, a.run) + j = db.get_job(con, rid, a.job_id) + if not j: + raise SystemExit(f"{rid} 里没有 {a.job_id}") + print(f"{a.job_id} [{rid}] state={j['state']} ec={j['error_class']} stage={j['stage_failed']} est={j['estimate_s']}") + st = con.execute("SELECT stage, attempt, lane, state, exit_code, error_class, duration_s FROM stages" + " WHERE run_id=? AND job_id=? AND rollout=-1 ORDER BY stage, attempt", (rid, a.job_id)).fetchall() + for s in st: + print(f" {s['stage']:<14} a{s['attempt']:02d} {s['lane']:<6} {s['state']:<8} exit={s['exit_code']!s:<4} {s['error_class'] or '':<18} {s['duration_s'] or 0:8.1f}s") + jd = paths.RUNS / rid / a.job_id + if j["stage_failed"]: + for p in sorted((jd / "oracle" / "stages" / j["stage_failed"]).glob("a*/stderr.log")): + print(f"--- {p} (tail 50) ---") + print("\n".join(p.read_text(errors="replace").splitlines()[-50:])) + rs = db.list_rollouts(con, rid, a.job_id) + if rs: + print("rollouts:") + for r in rs: + print(f" {r['kk']:02d} {r['mode']:<6} {r['policy'] or '':<8} {r['state']:<10} claim={r['claim_terminal']} consistent={r['claim_consistent']} review={r['review']} turns={r['turns']} tokens={r['tokens']}") + notes = con.execute("SELECT ts, note FROM annotations WHERE run_id=? AND job_id=? ORDER BY ts", (rid, a.job_id)).fetchall() + for n in notes: + print(f" note: {n['note']}") + return 0 + + +def cmd_annotate(a) -> int: + with db.connect() as con: + rid = _run_id(con, a.run) + db.annotate(con, rid, a.job_id, a.note) + print("ok") + return 0 + + +def cmd_report(a) -> int: + from .report import render + with db.connect() as con: + rid = _run_id(con, a.run) + print(render(rid)) + return 0 + + +def cmd_watch(a) -> int: + from .watch import watch + with db.connect() as con: + rid = _run_id(con, a.run) + return watch(rid, a.interval, a.once) + + +def cmd_reset_stale(a) -> int: + from .pipeline import reset_stale + with db.connect() as con: + rid = _run_id(con, a.run) + print(json.dumps(reset_stale(rid, a.force))) + return 0 + + +def cmd_board_backfill(a) -> int: + from .pipeline import board_backfill + with db.connect() as con: + rid = _run_id(con, a.run) + print(json.dumps(board_backfill(rid, a.job or None), ensure_ascii=False)) + return 0 + + +def cmd_rejudge(a) -> int: + from .pipeline import rejudge + with db.connect() as con: + rid = _run_id(con, a.run) + ids = [a.job_id] if a.job_id not in ("all", "--all") else [j["job_id"] for j in db.list_jobs(con, rid, mode="oracle") if j["state"] in ("PASS", "FAIL", "UNVERIFIED")] + from collections import Counter + c = Counter() + for jid in ids: + r = rejudge(rid, jid) + c[(r["terminal"], r.get("failed_gate"))] += 1 + if a.job_id not in ("all", "--all"): + print(json.dumps(r, ensure_ascii=False)) + if a.job_id in ("all", "--all"): + print(json.dumps({"rejudged": len(ids), "by_terminal_gate": {f"{k[0]}/{k[1]}": v for k, v in c.items()}}, ensure_ascii=False)) + return 0 + + +def cmd_verify(a) -> int: + from .verify import verify_rollout + v = verify_rollout(Path(a.rollout_dir)) + print(json.dumps({k: v[k] for k in ("job_id", "rollout", "terminal", "failed_gate", "claim_consistent", "review")})) + return 0 + + +def cmd_collect(a) -> int: + from .pipeline import collect_rollouts + with db.connect() as con: + rid = _run_id(con, a.run) + s = collect_rollouts(rid, a.job or None, a.modes.split(",") if a.modes else None, + parallel=a.parallel, dry_run=a.dry_run, limit=a.limit) + print(json.dumps({"run_id": rid, **s}, ensure_ascii=False)) + return 0 + + +def cmd_rollout(a) -> int: + from .rollout import run_rollout + with db.connect() as con: + rid = _run_id(con, a.run) + print(json.dumps(run_rollout(rid, a.job_id, a.kk, a.mode, a.policy, dry_run=a.dry_run), ensure_ascii=False)) + return 0 + + +def cmd_agent(a) -> int: + from .rollout import setup_models_json + if a.op == "setup": + print(setup_models_json(force=a.force)) + return 0 + raise SystemExit("agent op: setup") + + +def cmd_stage(a) -> int: + from . import stagerun + if a.op == "run": + stagerun.submit(a.stage, a.config) + return 0 + if a.op == "wait": + return stagerun.wait(a.stage, a.poll) + if a.op == "status": + return stagerun.status(a.stage) + raise SystemExit("stage op: run | wait | status") + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="bench") + sp = p.add_subparsers(dest="cmd", required=True) + + x = sp.add_parser("preflight"); x.add_argument("--modes"); x.add_argument("--oracle-only", action="store_true", help="只跑 benchmark 那一半时放行 agent 运行时缺失"); x.set_defaults(f=cmd_preflight) + x = sp.add_parser("new-run"); x.add_argument("--note"); x.set_defaults(f=cmd_new_run) + x = sp.add_parser("expand"); x.add_argument("--run"); x.add_argument("--modes"); x.add_argument("--marches", help="只展开这些 march,逗号分隔"); x.set_defaults(f=cmd_expand) + x = sp.add_parser("run"); x.add_argument("--run"); x.add_argument("--job", action="append") + x.add_argument("--dry-run", action="store_true"); x.add_argument("--parallel", type=int); x.set_defaults(f=cmd_run) + x = sp.add_parser("status"); x.add_argument("--run"); x.add_argument("--fail", action="store_true"); x.set_defaults(f=cmd_status) + x = sp.add_parser("show"); x.add_argument("job_id"); x.add_argument("--run"); x.set_defaults(f=cmd_show) + x = sp.add_parser("annotate"); x.add_argument("job_id"); x.add_argument("--note", required=True); x.add_argument("--run"); x.set_defaults(f=cmd_annotate) + x = sp.add_parser("report"); x.add_argument("--run"); x.set_defaults(f=cmd_report) + x = sp.add_parser("verify"); x.add_argument("rollout_dir"); x.set_defaults(f=cmd_verify) + x = sp.add_parser("rejudge"); x.add_argument("job_id", help="job_id 或 all(该 run 所有已结束的 job)"); x.add_argument("--run"); x.set_defaults(f=cmd_rejudge) + x = sp.add_parser("board-backfill"); x.add_argument("--run"); x.add_argument("--job", action="append"); x.set_defaults(f=cmd_board_backfill) + x = sp.add_parser("watch"); x.add_argument("--run"); x.add_argument("--interval", type=int, default=30); x.add_argument("--once", action="store_true"); x.set_defaults(f=cmd_watch) + x = sp.add_parser("reset-stale"); x.add_argument("--run"); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_reset_stale) + x = sp.add_parser("collect"); x.add_argument("--run"); x.add_argument("--job", action="append") + x.add_argument("--modes", help="e2e,repair"); x.add_argument("--parallel", type=int) + x.add_argument("--limit", type=int, help="先采 N 条试水"); x.add_argument("--dry-run", action="store_true") + x.set_defaults(f=cmd_collect) + x = sp.add_parser("rollout"); x.add_argument("job_id"); x.add_argument("--kk", type=int, default=0) + x.add_argument("--mode", choices=["e2e", "repair"], default="e2e"); x.add_argument("--policy", choices=["teacher", "student"], default="teacher") + x.add_argument("--run"); x.add_argument("--dry-run", action="store_true"); x.set_defaults(f=cmd_rollout) + x = sp.add_parser("agent"); x.add_argument("op", choices=["setup"]); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_agent) + x = sp.add_parser("stage"); x.add_argument("op", choices=["run", "wait", "status"]); x.add_argument("stage") + x.add_argument("--config"); x.add_argument("--poll", type=int, default=60); x.set_defaults(f=cmd_stage) + + a = p.parse_args(argv) + return a.f(a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/db.py b/harness/db.py new file mode 100644 index 0000000..6376f37 --- /dev/null +++ b/harness/db.py @@ -0,0 +1,238 @@ +"""state/jobs.sqlite — 状态、error_class、耗时历史的唯一真相。文件系统只存内容。""" +from __future__ import annotations + +import json +import os +import sqlite3 +import time +from contextlib import contextmanager +from pathlib import Path + +from .paths import DB_PATH, STATE + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, + created_at REAL NOT NULL, + manifest TEXT NOT NULL, -- json:venv pip-freeze sha、板固件、registry 指纹、数据 manifest sha + note TEXT +); +CREATE TABLE IF NOT EXISTS jobs ( + run_id TEXT NOT NULL, + job_id TEXT NOT NULL, + task TEXT, model TEXT, march TEXT, backend TEXT, qcfg TEXT, priority TEXT, + mode TEXT NOT NULL, -- oracle | e2e | repair + policy TEXT, -- teacher | student | NULL + k INTEGER DEFAULT 1, + state TEXT NOT NULL, -- PENDING ORACLE_RUNNING PASS FAIL UNVERIFIED + error_class TEXT, + stage_failed TEXT, + estimate_s REAL, + started_at REAL, finished_at REAL, + result TEXT, -- oracle/result.json 副本 + note TEXT, + PRIMARY KEY (run_id, job_id, mode, policy) +); +CREATE TABLE IF NOT EXISTS stages ( + run_id TEXT NOT NULL, job_id TEXT NOT NULL, + rollout INTEGER NOT NULL DEFAULT -1, -- -1 = oracle + stage TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + lane TEXT DEFAULT 'normal', -- normal | long + state TEXT NOT NULL, -- RUNNING OK FAIL STALLED SKIPPED + exit_code INTEGER, + error_class TEXT, + started_at REAL, finished_at REAL, duration_s REAL, + PRIMARY KEY (run_id, job_id, rollout, stage, attempt) +); +CREATE TABLE IF NOT EXISTS rollouts ( + run_id TEXT NOT NULL, job_id TEXT NOT NULL, kk INTEGER NOT NULL, + mode TEXT, policy TEXT, + state TEXT NOT NULL, -- QUEUED RUNNING VERIFYING PASS FAIL BLOCKED TIMEOUT + claim_terminal TEXT, + verdict_terminal TEXT, + claim_consistent INTEGER, + review INTEGER DEFAULT 0, -- oracle FAIL / rollout PASS → 1,进人工抽查 + turns INTEGER, tokens INTEGER, wall_s REAL, + verdict TEXT, + started_at REAL, finished_at REAL, + PRIMARY KEY (run_id, job_id, kk) +); +CREATE TABLE IF NOT EXISTS annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT, job_id TEXT, ts REAL, note TEXT +); +CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(run_id, state); +CREATE INDEX IF NOT EXISTS idx_stages_hist ON stages(job_id, stage, state); +""" + + +def _open(path: Path) -> sqlite3.Connection: + con = sqlite3.connect(path, timeout=60) + con.row_factory = sqlite3.Row + con.execute("PRAGMA busy_timeout=60000") + return con + + +@contextmanager +def connect(path: Path = DB_PATH): + STATE.mkdir(parents=True, exist_ok=True) + con = _open(path) + # 默认 TRUNCATE:WAL 在 FUSE(JuiceFS 等)上会 disk I/O error(-shm 需要 mmap)。 + # state/ 放本地盘(BENCH_STATE_DIR=/home//bench-state)时可设 BENCH_SQLITE_JOURNAL=WAL。 + jm = os.environ.get("BENCH_SQLITE_JOURNAL", "TRUNCATE") + con.execute(f"PRAGMA journal_mode={jm}") + con.executescript(SCHEMA) + try: + yield con + con.commit() + finally: + con.close() + + +# ---------------------------------------------------------------- runs + +def create_run(con, run_id: str, manifest: dict, note: str = "") -> None: + if con.execute("SELECT 1 FROM runs WHERE run_id=?", (run_id,)).fetchone(): + raise RuntimeError(f"run_id 已存在(append-only,不允许覆盖): {run_id}") + con.execute("INSERT INTO runs VALUES (?,?,?,?)", + (run_id, time.time(), json.dumps(manifest, ensure_ascii=False), note)) + + +def get_run(con, run_id: str) -> dict | None: + r = con.execute("SELECT * FROM runs WHERE run_id=?", (run_id,)).fetchone() + return dict(r) if r else None + + +def latest_run_id(con) -> str | None: + r = con.execute("SELECT run_id FROM runs ORDER BY created_at DESC LIMIT 1").fetchone() + return r[0] if r else None + + +def next_run_id(con, date_str: str) -> str: + rows = con.execute("SELECT run_id FROM runs WHERE run_id LIKE ?", (f"r{date_str}-%",)).fetchall() + seq = 1 + max([int(r[0].rsplit("-", 1)[1]) for r in rows] or [0]) + return f"r{date_str}-{seq:02d}" + + +# ---------------------------------------------------------------- jobs + +def upsert_job(con, run_id: str, spec_row: dict, estimate_s: float | None) -> bool: + """展开幂等:已存在的 (run_id, job_id, mode, policy) 不动。返回是否新插入。""" + key = (run_id, spec_row["job_id"], spec_row["mode"], spec_row.get("policy")) + if con.execute("SELECT 1 FROM jobs WHERE run_id=? AND job_id=? AND mode=? AND policy IS ?", key).fetchone(): + return False + con.execute( + "INSERT INTO jobs (run_id, job_id, task, model, march, backend, qcfg, priority, mode, policy, k, state, estimate_s)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (run_id, spec_row["job_id"], spec_row["task"], spec_row["model"], spec_row["march"], + spec_row["backend"], spec_row["qcfg"], spec_row["priority"], spec_row["mode"], + spec_row.get("policy"), spec_row.get("k", 1), "PENDING", estimate_s)) + return True + + +def claim_job(con, run_id: str, job_id: str, mode: str = "oracle", force: bool = False) -> bool: + """原子认领:只有 PENDING 的 job 能被置成 ORACLE_RUNNING。两个 `bench run` 进程同时挑到同一个 job 时只有一个赢。 + force=True 允许重跑已结束的 job(只给 --job 显式指定时用;结果仍是原地覆盖 result.json,慎用)。""" + allowed = "('PENDING')" if not force else "('PENDING','PASS','FAIL','UNVERIFIED')" + cur = con.execute(f"UPDATE jobs SET state='ORACLE_RUNNING', started_at=? WHERE run_id=? AND job_id=? AND mode=? AND state IN {allowed}", + (time.time(), run_id, job_id, mode)) + con.commit() + return cur.rowcount == 1 + + +def set_job_state(con, run_id: str, job_id: str, state: str, *, mode: str = "oracle", + error_class: str | None = None, stage_failed: str | None = None, + result: dict | None = None) -> None: + now = time.time() + if state == "ORACLE_RUNNING": + con.execute("UPDATE jobs SET state=?, started_at=? WHERE run_id=? AND job_id=? AND mode=?", + (state, now, run_id, job_id, mode)) + else: + con.execute("UPDATE jobs SET state=?, finished_at=?, error_class=?, stage_failed=?, result=?" + " WHERE run_id=? AND job_id=? AND mode=?", + (state, now, error_class, stage_failed, + json.dumps(result, ensure_ascii=False) if result else None, run_id, job_id, mode)) + + +def get_job(con, run_id: str, job_id: str, mode: str = "oracle") -> dict | None: + r = con.execute("SELECT * FROM jobs WHERE run_id=? AND job_id=? AND mode=?", (run_id, job_id, mode)).fetchone() + return dict(r) if r else None + + +def list_jobs(con, run_id: str, mode: str | None = None, state: str | None = None) -> list[dict]: + q, args = "SELECT * FROM jobs WHERE run_id=?", [run_id] + if mode: + q += " AND mode=?"; args.append(mode) + if state: + q += " AND state=?"; args.append(state) + q += " ORDER BY CASE priority WHEN 'A' THEN 0 WHEN 'B' THEN 1 ELSE 2 END, estimate_s, job_id" + return [dict(r) for r in con.execute(q, args)] + + +# ---------------------------------------------------------------- stages + +def stage_start(con, run_id, job_id, stage, attempt=0, rollout=-1, lane="normal") -> None: + con.execute("INSERT OR REPLACE INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, started_at)" + " VALUES (?,?,?,?,?,?,'RUNNING',?)", (run_id, job_id, rollout, stage, attempt, lane, time.time())) + + +def stage_finish(con, run_id, job_id, stage, attempt, state, exit_code=None, error_class=None, rollout=-1) -> None: + now = time.time() + r = con.execute("SELECT started_at FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?", + (run_id, job_id, rollout, stage, attempt)).fetchone() + started = r[0] if r and r[0] else now + con.execute("UPDATE stages SET state=?, exit_code=?, error_class=?, finished_at=?, duration_s=?" + " WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?", + (state, exit_code, error_class, now, now - started, run_id, job_id, rollout, stage, attempt)) + + +def stage_history(con, job_id: str, stage: str) -> list[float]: + """同 job_id(跨 run)同阶段的历史成功耗时,用于估算。""" + return [r[0] for r in con.execute( + "SELECT duration_s FROM stages WHERE job_id=? AND stage=? AND state='OK' AND rollout=-1 AND duration_s IS NOT NULL", + (job_id, stage))] + + +def stage_history_by_model(con, model: str, stage: str) -> list[float]: + return [r[0] for r in con.execute( + "SELECT s.duration_s FROM stages s JOIN jobs j ON s.run_id=j.run_id AND s.job_id=j.job_id" + " WHERE j.model=? AND s.stage=? AND s.state='OK' AND s.rollout=-1 AND s.duration_s IS NOT NULL", + (model, stage))] + + +# ---------------------------------------------------------------- rollouts + +def rollout_upsert(con, run_id, job_id, kk, mode, policy, state="QUEUED") -> None: + con.execute("INSERT OR IGNORE INTO rollouts (run_id, job_id, kk, mode, policy, state) VALUES (?,?,?,?,?,?)", + (run_id, job_id, kk, mode, policy, state)) + + +def rollout_set(con, run_id, job_id, kk, **fields) -> None: + if not fields: + return + cols = ", ".join(f"{k}=?" for k in fields) + con.execute(f"UPDATE rollouts SET {cols} WHERE run_id=? AND job_id=? AND kk=?", + (*fields.values(), run_id, job_id, kk)) + + +def list_rollouts(con, run_id, job_id) -> list[dict]: + return [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? AND job_id=? ORDER BY kk", (run_id, job_id))] + + +# ---------------------------------------------------------------- annotations / status + +def annotate(con, run_id, job_id, note) -> None: + con.execute("INSERT INTO annotations (run_id, job_id, ts, note) VALUES (?,?,?,?)", (run_id, job_id, time.time(), note)) + + +def status_counts(con, run_id: str, mode: str = "oracle") -> list[dict]: + return [dict(r) for r in con.execute( + "SELECT state, task, march, COUNT(*) n FROM jobs WHERE run_id=? AND mode=? GROUP BY state, task, march ORDER BY state, task, march", + (run_id, mode))] + + +def error_class_counts(con, run_id: str) -> list[dict]: + return [dict(r) for r in con.execute( + "SELECT COALESCE(error_class,'(none)') error_class, stage_failed, COUNT(*) n FROM jobs WHERE run_id=? AND state='FAIL'" + " GROUP BY error_class, stage_failed ORDER BY n DESC", (run_id,))] diff --git a/harness/errors.py b/harness/errors.py new file mode 100644 index 0000000..48cdecb --- /dev/null +++ b/harness/errors.py @@ -0,0 +1,46 @@ +"""stderr → error_class。正则规则库,unknown 进人工分类队列后在这里补规则。 + +规则顺序即优先级:先匹配到的赢。瞬态类在 TRANSIENT 里,决定重试策略。 +""" +from __future__ import annotations + +import re + +DETERMINISTIC = ["unsupported_op", "shape_infer", "export_fail", "calib_nan", "verifier_mismatch", "config_error", + # 门失败(gates.classify_gate_failure) + "accuracy_collapse", "accuracy_drop", "numeric_mismatch", "march_mismatch", "perf_over_budget"] +TRANSIENT = ["oom", "board_offline", "stalled"] +ALL = DETERMINISTIC + TRANSIENT + ["unknown"] + +# (error_class, compiled regex)。写规则时对着真实 stderr 写,别猜。 +RULES: list[tuple[str, re.Pattern]] = [ + ("config_error", re.compile(r"Key '[^']+' error:|Wrong key '|Missing key|yaml.*(invalid|error)|schema.*error", re.I)), # hb_compile yaml 校验不过:harness/agent 配置写错 + ("oom", re.compile(r"CUDA out of memory|OutOfMemoryError|std::bad_alloc|Cannot allocate memory|MemoryError|Killed process .* out of memory", re.I)), + ("board_offline", re.compile(r"Connection (refused|timed out|reset by peer)|No route to host|ssh: connect to host|board .*offline|hrt_.*: failed to connect", re.I)), + ("unsupported_op", re.compile(r"(unsupported|not supported|Unsupported) (op|operator|node|type|layer)|No support for op|op .* is not supported on (nash|bpu)|不支持的算子", re.I)), + ("shape_infer", re.compile(r"shape infer|ShapeInference|dynamic (shape|axis|dim)|Cannot infer shape|shape mismatch|unknown dimension", re.I)), + ("export_fail", re.compile(r"torch\.onnx|onnx export|symbolic_trace|fx\.trace|TracerWarning: .*failed|Exporting the operator .* is not supported|torch\.export", re.I)), + ("calib_nan", re.compile(r"\bnan\b.*(calib|quant|scale)|(calib|quant|scale).*\bnan\b|overflow encountered|inf.*threshold|threshold.*inf", re.I)), + ("verifier_mismatch", re.compile(r"hb_verifier.*(fail|mismatch)|cosine.*(below|less than|<)|consistency check failed|verify.*mismatch", re.I)), +] + +STALLED_MARK = "__BENCH_WATCHDOG_STALLED__" + + +def classify(stderr_text: str, stdout_text: str = "", exit_code: int | None = None) -> str: + text = (stderr_text or "") + "\n" + (stdout_text or "") + if STALLED_MARK in text: + return "stalled" + for cls, rx in RULES: + if rx.search(text): + return cls + return "unknown" + + +def is_transient(error_class: str) -> bool: + return error_class in TRANSIENT + + +def same_class(a: str | None, b: str | None) -> bool: + """BLOCKED 判定用:同类错误。unknown 不与任何东西同类(避免误判为正确 BLOCKED)。""" + return bool(a) and a == b and a != "unknown" diff --git a/harness/eval/__init__.py b/harness/eval/__init__.py new file mode 100644 index 0000000..ac8c563 --- /dev/null +++ b/harness/eval/__init__.py @@ -0,0 +1,11 @@ +"""逐任务评测器。agent 不可写;verdict 的唯一依据。FP32 基线与量化实测走同一份代码、同一子集、同一预处理。 + +EVALUATOR_VERSION 任何口径变化都要升;job.json 钉死它,基线缓存按它失效。 +""" +EVALUATOR_VERSION = "0.4.0" + +TASK_MODULES = { + "cls": "harness.eval.cls", + "det": "harness.eval.det", + # P4+:seg iseg pose mde stereo flow feat occ +} diff --git a/harness/eval/cls.py b/harness/eval/cls.py new file mode 100644 index 0000000..3ff2de1 --- /dev/null +++ b/harness/eval/cls.py @@ -0,0 +1,57 @@ +"""T01 分类:Top-1(主)/ Top-5(记录)。ImageNet-1K val T1=2000。 +配对 delta 的标准误由不一致对决定:SE(Δ) ≈ sqrt((p10 + p01)/n)·100 pp。""" +from __future__ import annotations + +import numpy as np + +from .common import T1, preprocess_cls, run_batched + +METRIC = "top1" +BATCH = 256 + + +def _size(input_spec: str) -> int: + return int(input_spec.split("x")[-1]) + + +def predict(runner, t1: T1, input_spec: str, items: list[dict] | None = None, resize: int = 0) -> tuple[np.ndarray, np.ndarray]: + """返回 (top5_pred[n,5], labels[n])。resize=0 → size*256/224(经典 224/256);torchvision 各权重的 resize 由 models.csv 给。""" + items = items or t1.items + size = _size(input_spec) + resize = resize or round(size * 256 / 224) + preds, labels = [], [] + for i in range(0, len(items), BATCH): + chunk = items[i:i + BATCH] + xs = [preprocess_cls(t1.image(it), size=size, resize=resize) for it in chunk] + for it, outs in zip(chunk, run_batched(runner, xs)): + logits = outs[0].reshape(-1) + preds.append(np.argsort(-logits)[:5]) + labels.append(int(it["label"])) + return np.stack(preds), np.asarray(labels) + + +def metrics(top5: np.ndarray, labels: np.ndarray) -> dict: + top1 = float((top5[:, 0] == labels).mean() * 100) + t5 = float(np.any(top5 == labels[:, None], axis=1).mean() * 100) + return {"top1": top1, "top5": t5, "n": int(len(labels))} + + +def paired_se(top5_a: np.ndarray, top5_b: np.ndarray, labels: np.ndarray) -> float: + ca, cb = top5_a[:, 0] == labels, top5_b[:, 0] == labels + p_disc = float((ca != cb).mean()) + return float(np.sqrt(p_disc / len(labels)) * 100) + + +def run_fp32(runner, t1: T1, input_spec: str, out: dict, resize: int = 0) -> None: + top5, labels = predict(runner, t1, input_spec, resize=resize) + out["metrics"] = metrics(top5, labels) + out["_pred"] = top5.tolist() + + +def run_quant(runner, t1: T1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: + top5, labels = predict(runner, t1, input_spec, resize=resize) + m = metrics(top5, labels) + se = paired_se(np.asarray(baseline["_pred"]), top5, labels) if baseline.get("_pred") else None + out["metrics"] = m + out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"]["top1"], "quant": m["top1"], + "tier": "t1", "n": m["n"], "se": se, "also": {"top5": m["top5"], "fp32_top5": baseline["metrics"]["top5"]}} diff --git a/harness/eval/common.py b/harness/eval/common.py new file mode 100644 index 0000000..e920f54 --- /dev/null +++ b/harness/eval/common.py @@ -0,0 +1,153 @@ +"""T1 tar 读取、预处理、推理后端抽象。所有任务共用。""" +from __future__ import annotations + +import hashlib +import io +import json +import tarfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) + + +@dataclass +class T1: + manifest: dict + tar: tarfile.TarFile + + @classmethod + def open(cls, path: str | Path, expect_sha: str | None = None) -> "T1": + p = Path(path) + if expect_sha and expect_sha != "None": + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + if h.hexdigest() != expect_sha: + raise RuntimeError(f"T1 sha 不一致: {p} {h.hexdigest()[:12]} != {expect_sha[:12]}") + t = tarfile.open(p, "r") + man = json.loads(t.extractfile("manifest.json").read()) + return cls(man, t) + + @property + def items(self) -> list[dict]: + return self.manifest["items"] + + def image(self, item: dict): + from PIL import Image + data = self.tar.extractfile(item["file"]).read() + return Image.open(io.BytesIO(data)).convert("RGB") + + def g2_subset(self, n: int, seed: int = 20260914) -> list[dict]: + rng = np.random.default_rng(seed) + idx = rng.permutation(len(self.items))[:n] + return [self.items[i] for i in sorted(idx)] + + +# ---------------------------------------------------------------- 预处理(口径钉死,fp32 与 quant 同一份) + +def preprocess_cls(img, size: int = 224, resize: int = 256) -> np.ndarray: + from PIL import Image + w, h = img.size + s = resize / min(w, h) + img = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BILINEAR) + w, h = img.size + l, t = (w - size) // 2, (h - size) // 2 + img = img.crop((l, t, l + size, t + size)) + x = np.asarray(img, dtype=np.float32) / 255.0 + x = (x - IMAGENET_MEAN) / IMAGENET_STD + return x.transpose(2, 0, 1)[None] # 1x3xHxW + + +def letterbox(img, size: int = 640): + """YOLO 风格 letterbox。返回 (1x3xSxS float32 [0,1], scale, (padx, pady))。""" + from PIL import Image + w, h = img.size + s = size / max(w, h) + nw, nh = round(w * s), round(h * s) + img = img.resize((nw, nh), Image.BILINEAR) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + px, py = (size - nw) // 2, (size - nh) // 2 + canvas.paste(img, (px, py)) + x = np.asarray(canvas, dtype=np.float32) / 255.0 + return x.transpose(2, 0, 1)[None], s, (px, py) + + +# ---------------------------------------------------------------- 推理后端 + +class TorchRunner: + def __init__(self, model, source, weights, input_spec): + import torch + from harness.tools.export_onnx import load_model + self.torch = torch + self.m = load_model(model, source, Path(weights), input_spec) + self.dev = "cuda" if torch.cuda.is_available() else "cpu" + self.m.to(self.dev) + + def __call__(self, x: np.ndarray) -> list[np.ndarray]: + with self.torch.no_grad(): + out = self.m(self.torch.from_numpy(x).to(self.dev)) + outs = out if isinstance(out, (list, tuple)) else [out] + flat = [] + for o in outs: + if isinstance(o, (list, tuple)): + flat.extend(o) + else: + flat.append(o) + return [o.detach().float().cpu().numpy() for o in flat if hasattr(o, "detach")] + + +class HbmRunner: + """x86 侧 hbm 推理。hbm 运行时(hbm_infer)在 runtime venv 里,本进程在 bench venv 里, + 所以按块起子进程:一块 64 张 → npz → runtime venv 的 harness.tools.infer_hbm → npz。 + 单张调用也走这条路(慢),评测器应该用 batch()。""" + + CHUNK = 64 + + def __init__(self, hbm: str, runtime_python: str, workdir: str | None = None): + import subprocess, tempfile + self.hbm = str(hbm) + self.py = runtime_python + self.tmp = Path(workdir or tempfile.mkdtemp(prefix="hbm-eval-")) + self.subprocess = subprocess + self._n = 0 + + def batch(self, xs: list[np.ndarray]) -> list[list[np.ndarray]]: + outs: list[list[np.ndarray]] = [] + for i in range(0, len(xs), self.CHUNK): + chunk = xs[i:i + self.CHUNK] + self._n += 1 + fin, fout = self.tmp / f"in{self._n:04d}.npz", self.tmp / f"out{self._n:04d}.npz" + np.savez(fin, **{f"s{j:04d}": x for j, x in enumerate(chunk)}) + env = dict(**__import__("os").environ) + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[2]) + r = self.subprocess.run([self.py, "-m", "harness.tools.infer_hbm", "--model", self.hbm, + "--inputs", str(fin), "--out", str(fout)], capture_output=True, text=True, env=env) + if r.returncode != 0: + raise RuntimeError(f"infer_hbm failed rc={r.returncode}: {r.stderr[-2000:]}") + o = np.load(fout) + for j in range(len(chunk)): + keys = sorted(k for k in o.files if k.startswith(f"s{j:04d}/")) + outs.append([o[k] for k in keys]) + fin.unlink(missing_ok=True); fout.unlink(missing_ok=True) + return outs + + def __call__(self, x: np.ndarray) -> list[np.ndarray]: + return self.batch([x])[0] + + +def run_batched(runner, xs: list[np.ndarray]) -> list[list[np.ndarray]]: + """TorchRunner 逐张、HbmRunner 分块——评测器只认这个入口。""" + if hasattr(runner, "batch"): + return runner.batch(xs) + return [runner(x) for x in xs] + + +def cosine(a: np.ndarray, b: np.ndarray) -> float: + a = a.astype(np.float64).ravel(); b = b.astype(np.float64).ravel() + na, nb = np.linalg.norm(a), np.linalg.norm(b) + return float(a @ b / (na * nb)) if na and nb else 0.0 diff --git a/harness/eval/det.py b/harness/eval/det.py new file mode 100644 index 0000000..e822f49 --- /dev/null +++ b/harness/eval/det.py @@ -0,0 +1,101 @@ +"""T02 检测:box mAP@[.5:.95](主)/ AP50(记录)。COCO val2017 T1=1000,pycocotools。 + +口径钉死:letterbox 640、conf 0.001、NMS IoU 0.7、每图最多 300 框、pycocotools 版本记进 metrics。 +后处理默认按 YOLOv8 输出 [1, 4+nc, N];其他头在 adapters 里给 decode()。 +""" +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +import numpy as np + +from .common import T1, letterbox, run_batched + +METRIC = "map_50_95" +CONF, IOU, MAX_DET = 0.001, 0.7, 300 + + +def _nms(boxes: np.ndarray, scores: np.ndarray, iou: float) -> np.ndarray: + try: + import torch, torchvision + keep = torchvision.ops.nms(torch.from_numpy(boxes).float(), torch.from_numpy(scores).float(), iou) + return keep.numpy() + except ImportError: + order = scores.argsort()[::-1] + keep = [] + while order.size: + i = order[0]; keep.append(i) + xx1 = np.maximum(boxes[i, 0], boxes[order[1:], 0]); yy1 = np.maximum(boxes[i, 1], boxes[order[1:], 1]) + xx2 = np.minimum(boxes[i, 2], boxes[order[1:], 2]); yy2 = np.minimum(boxes[i, 3], boxes[order[1:], 3]) + inter = np.clip(xx2 - xx1, 0, None) * np.clip(yy2 - yy1, 0, None) + a = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + o = inter / (a[i] + a[order[1:]] - inter + 1e-9) + order = order[1:][o < iou] + return np.asarray(keep) + + +def decode_yolov8(out: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """[1, 4+nc, N] → boxes xyxy(letterbox 坐标), scores, cls""" + p = out[0].T # N x (4+nc) + xywh, cls_scores = p[:, :4], p[:, 4:] + cls = cls_scores.argmax(1); score = cls_scores.max(1) + m = score > CONF + xywh, cls, score = xywh[m], cls[m], score[m] + boxes = np.stack([xywh[:, 0] - xywh[:, 2] / 2, xywh[:, 1] - xywh[:, 3] / 2, + xywh[:, 0] + xywh[:, 2] / 2, xywh[:, 1] + xywh[:, 3] / 2], 1) + # 按类偏移做 class-aware NMS + off = boxes + cls[:, None] * 4096 + keep = _nms(off, score, IOU)[:MAX_DET] + return boxes[keep], score[keep], cls[keep] + + +BATCH = 64 + + +def predict(runner, t1: T1, input_spec: str, decode=decode_yolov8) -> list[dict]: + size = int(input_spec.split("x")[-1]) + cat_map = t1.manifest.get("cat_map") # 模型类别 idx → coco category_id + dets = [] + items = t1.items + for i in range(0, len(items), BATCH): + chunk = items[i:i + BATCH] + pre = [letterbox(t1.image(it), size) for it in chunk] + for it, (x, s, (px, py)), outs in zip(chunk, pre, run_batched(runner, [p[0] for p in pre])): + boxes, scores, cls = decode(outs[0]) + boxes = (boxes - np.array([px, py, px, py])) / s + for b, sc, c in zip(boxes, scores, cls): + cid = cat_map[str(int(c))] if cat_map else int(c) + dets.append({"image_id": it["id"], "category_id": cid, "score": float(sc), + "bbox": [float(b[0]), float(b[1]), float(b[2] - b[0]), float(b[3] - b[1])]}) + return dets + + +def coco_eval(t1: T1, dets: list[dict]) -> dict: + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + import pycocotools + ann = json.loads(t1.tar.extractfile("annotations.json").read()) + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump(ann, f); gt_path = f.name + gt = COCO(gt_path) + if not dets: + return {"map_50_95": 0.0, "ap50": 0.0, "n": len(t1.items), "pycocotools": getattr(pycocotools, "__version__", "?")} + dt = gt.loadRes(dets) + e = COCOeval(gt, dt, "bbox") + e.params.imgIds = [it["id"] for it in t1.items] + e.evaluate(); e.accumulate(); e.summarize() + return {"map_50_95": float(e.stats[0] * 100), "ap50": float(e.stats[1] * 100), "n": len(t1.items), + "pycocotools": getattr(pycocotools, "__version__", "?")} + + +def run_fp32(runner, t1: T1, input_spec: str, out: dict, resize: int = 0) -> None: + out["metrics"] = coco_eval(t1, predict(runner, t1, input_spec)) + + +def run_quant(runner, t1: T1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: + m = coco_eval(t1, predict(runner, t1, input_spec)) + out["metrics"] = m + out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"]["map_50_95"], "quant": m["map_50_95"], + "tier": "t1", "n": m["n"], "se": None, "also": {"ap50": m["ap50"], "fp32_ap50": baseline["metrics"]["ap50"]}} diff --git a/harness/eval/run.py b/harness/eval/run.py new file mode 100644 index 0000000..1a1542a --- /dev/null +++ b/harness/eval/run.py @@ -0,0 +1,104 @@ +"""评测入口(在 backend venv 里跑)。 + +--mode fp32 : 02-baseline。T1 上算指标 + 抽 G2 样本保存输入与 FP32 输出。→ /metrics.json, g2_inputs.npz, g2_fp32.npz +--mode quant: 08-eval。hbm 在 G2 样本上算余弦、在 T1 上算指标、与基线配对。→ (eval.json) +""" +from __future__ import annotations + +import argparse +import importlib +import json +import sys +from pathlib import Path + +import numpy as np + +from . import EVALUATOR_VERSION, TASK_MODULES +from .common import T1, TorchRunner, HbmRunner, cosine, preprocess_cls, letterbox, run_batched + + +def g2_inputs(task: str, t1: T1, input_spec: str, n: int, resize: int = 0) -> dict[str, np.ndarray]: + size = int(input_spec.split("x")[-1]) + xs = {} + for i, it in enumerate(t1.g2_subset(n)): + img = t1.image(it) + if task == "cls": + xs[f"s{i:03d}"] = preprocess_cls(img, size=size, resize=resize or round(size * 256 / 224)) + else: + xs[f"s{i:03d}"] = letterbox(img, size)[0] + return xs + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--mode", choices=["fp32", "quant"], required=True) + ap.add_argument("--task", required=True) + ap.add_argument("--model", required=True) + ap.add_argument("--source") + ap.add_argument("--weights") + ap.add_argument("--hbm") + ap.add_argument("--onnx") + ap.add_argument("--backend") + ap.add_argument("--input-spec", required=True) + ap.add_argument("--resize", type=int, default=0, help="cls:resize 短边(torchvision 默认权重各不同);0 = size*256/224") + ap.add_argument("--t1", required=True) + ap.add_argument("--t1-sha") + ap.add_argument("--baseline") + ap.add_argument("--g2-samples", type=int, default=32) + ap.add_argument("--calib-source", default="t0") + ap.add_argument("--runtime-python", help="hbm 运行时所在 venv 的 python(quant 模式必填)") + ap.add_argument("--evaluator-version", default=EVALUATOR_VERSION) + ap.add_argument("--out", required=True) + a = ap.parse_args() + + if a.evaluator_version != EVALUATOR_VERSION: + print(f"evaluator version mismatch: job wants {a.evaluator_version}, code is {EVALUATOR_VERSION}", file=sys.stderr) + return 66 + if a.task not in TASK_MODULES: + print(f"task {a.task} evaluator not implemented (P4+)", file=sys.stderr) + return 65 + mod = importlib.import_module(TASK_MODULES[a.task]) + t1 = T1.open(a.t1, a.t1_sha) + out: dict = {"task": a.task, "model": a.model, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest.get("sha256"), "n": len(t1.items)} + + if a.mode == "fp32": + runner = TorchRunner(a.model, a.source, a.weights, a.input_spec) + od = Path(a.out); od.mkdir(parents=True, exist_ok=True) + xs = g2_inputs(a.task, t1, a.input_spec, a.g2_samples, a.resize) + np.savez(od / "g2_inputs.npz", **xs) + np.savez(od / "g2_fp32.npz", **{f"{k}/out{i}": o for k, x in xs.items() for i, o in enumerate(runner(x))}) + mod.run_fp32(runner, t1, a.input_spec, out, resize=a.resize) + (od / "metrics.json").write_text(json.dumps(out, ensure_ascii=False, indent=2)) + print(json.dumps(out["metrics"])) + return 0 + + # quant + if not a.runtime_python: + print("--runtime-python required in quant mode", file=sys.stderr); return 2 + runner = HbmRunner(a.hbm, a.runtime_python) + bdir = Path(a.baseline) + baseline = json.loads((bdir / "metrics.json").read_text()) + if baseline.get("t1_manifest_sha") != t1.manifest.get("sha256"): + print("baseline T1 sha != current T1 sha:基线与实测子集不一致,delta 不可比", file=sys.stderr) + return 67 + xs = np.load(bdir / "g2_inputs.npz") + fp = np.load(bdir / "g2_fp32.npz") + per_out: dict[str, list[float]] = {} + keys = list(xs.files) + for k, outs in zip(keys, run_batched(runner, [xs[k] for k in keys])): + for i, o in enumerate(outs): + ref = fp[f"{k}/out{i}"] + per_out.setdefault(f"out{i}", []).append(cosine(ref, o)) + cos_per_output = {k: float(min(v)) for k, v in per_out.items()} + out["G2"] = {"cos_min": min(cos_per_output.values()) if cos_per_output else None, + "cos_per_output": cos_per_output, "n": len(xs.files), "source": a.calib_source} + mod.run_quant(runner, t1, a.input_spec, baseline, out, resize=a.resize) + Path(a.out).parent.mkdir(parents=True, exist_ok=True) + Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=2)) + print(json.dumps({"G2": out["G2"], "G3": out.get("G3")})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/gates.py b/harness/gates.py new file mode 100644 index 0000000..3ae3809 --- /dev/null +++ b/harness/gates.py @@ -0,0 +1,175 @@ +"""四道门:原始测量值 + job.json 门快照 → 逐门 verdict + terminal。 + +verdict 取值:pass | pass-screen(T1 放宽)| pass-weak(synthetic 校准源)| fail | unverified | na +terminal:PASS | FAIL | UNVERIFIED +oracle 与 rollout 走同一份代码;这里不读文件、不跑工具,纯函数,便于人肉核对。 +""" +from __future__ import annotations + +import math + +PASSING = {"pass", "pass-screen", "pass-weak"} +LOWER_IS_BETTER = {"absrel", "rmse", "epe", "epe2d", "d1_all", "fl_all"} + + +def g1(m: dict | None, cfg: dict) -> dict: + if not m: + return {"verdict": "unverified", "reason": "no measurement"} + checks = { + "hbm_exists": m.get("hbm_exists"), + "march_ok": m.get("march_ok") if cfg.get("check_march", True) else True, + "io_ok": m.get("io_ok") if cfg.get("check_input_dtype_layout", True) else True, + "verifier_ok": m.get("verifier_ok") if cfg.get("run_hb_verifier", True) else True, + "log_clean": (not m.get("log_has_error")) if m.get("log_has_error") is not None else None, + } + if any(v is False for v in checks.values()): + failed = [k for k, v in checks.items() if v is False] + return {"verdict": "fail", "failed": failed, **checks, "march_read": m.get("march_read")} + if any(v is None for v in checks.values()): + return {"verdict": "unverified", "missing": [k for k, v in checks.items() if v is None], **checks} + return {"verdict": "pass", **checks, "march_read": m.get("march_read")} + + +def g2(m: dict | None, cfg: dict) -> dict: + if not m or (m.get("cos_min") is None and not m.get("cos_per_output")): + return {"verdict": "unverified", "reason": "no measurement"} + per = cfg.get("per_output") + if per: + outs = m.get("cos_per_output") or {} + results, ok = {}, True + for name, thr in per.items(): + v = outs.get(name) + if v is None: + return {"verdict": "unverified", "reason": f"output {name} missing", "cos_per_output": outs} + results[name] = {"cos": v, "threshold": thr, "ok": v >= thr} + ok = ok and v >= thr + cos_min = min(outs.values()) if outs else None + verdict = "pass" if ok else "fail" + out = {"verdict": verdict, "cos_min": cos_min, "per_output": results} + else: + thr = cfg["threshold"] + cos_min = m["cos_min"] + out = {"verdict": "pass" if cos_min >= thr else "fail", "cos_min": cos_min, "threshold": thr} + if out["verdict"] == "pass" and m.get("source") == "synthetic": + out["verdict"] = "pass-weak" + out["source"] = m.get("source", "t1") + out["n_samples"] = m.get("n") + return out + + +def g3(m: dict | None, cfg: dict) -> dict: + if cfg.get("verdict") == "na" or not cfg.get("required", True): + return {"verdict": "na", "reason": cfg.get("reason", "gate not required for this task")} + if not m or m.get("fp32") is None or m.get("quant") is None: + return {"verdict": "unverified", "reason": "no measurement", "metric": cfg.get("metric")} + kind = cfg.get("kind") + if kind not in ("abs_pp", "rel_pct"): + raise ValueError(f"G3 kind 必须显式 abs_pp|rel_pct,得到 {kind!r}") + metric = m.get("metric") or cfg.get("metric") + metric_name = metric if isinstance(metric, str) else (metric[0] if metric else "") + fp32, quant, tol = float(m["fp32"]), float(m["quant"]), float(cfg["tol"]) + lower_better = metric_name in LOWER_IS_BETTER + if kind == "abs_pp": + delta = quant - fp32 + if lower_better: + delta = -delta + eff_tol = tol + if m.get("tier") == "t1" and m.get("se"): + eff_tol = tol - cfg.get("t1_screen_widen_se", 2) * float(m["se"]) + ok = delta >= eff_tol + else: # rel_pct:劣化百分比 + if fp32 == 0: + return {"verdict": "unverified", "reason": "fp32 baseline is 0"} + degr = (quant - fp32) / abs(fp32) * 100.0 + if not lower_better: + degr = -degr + delta = degr + eff_tol = tol + if m.get("tier") == "t1" and m.get("se"): + eff_tol = tol + cfg.get("t1_screen_widen_se", 2) * float(m["se"]) + ok = degr <= eff_tol + verdict = ("pass-screen" if m.get("tier") == "t1" else "pass") if ok else "fail" + return {"verdict": verdict, "metric": metric_name, "kind": kind, "fp32": fp32, "quant": quant, + "delta": delta, "tolerance": tol, "effective_tolerance": eff_tol, "tier": m.get("tier"), + "n": m.get("n"), "se": m.get("se")} + + +def g4(m: dict | None, cfg: dict) -> dict: + rec = {k: (m or {}).get(k) for k in cfg.get("record", [])} + rec["contended"] = bool((m or {}).get("contended")) + budget = cfg.get("latency_budget_ms") + if not budget: + return {"verdict": "na", **rec} + if not m or m.get("latency_ms_mean") is None: + return {"verdict": "unverified", **rec} + return {"verdict": "pass" if m["latency_ms_mean"] <= budget else "fail", "budget_ms": budget, **rec} + + +def evaluate(measurements: dict, gates_cfg: dict) -> dict: + """返回 {"gates": {...}, "terminal": ...}。required 里的门决定 terminal。""" + res = { + "G1": g1(measurements.get("G1"), gates_cfg["G1"]), + "G2": g2(measurements.get("G2"), gates_cfg["G2"]), + "G3": g3(measurements.get("G3"), gates_cfg["G3"]), + "G4": g4(measurements.get("G4"), gates_cfg["G4"]), + } + required = list(gates_cfg.get("required", ["G1", "G2", "G3"])) + if gates_cfg["G4"].get("latency_budget_ms") and "G4" not in required: + required.append("G4") + verdicts = [res[g]["verdict"] for g in required] + if any(v == "fail" for v in verdicts): + terminal = "FAIL" + elif any(v == "unverified" for v in verdicts): + terminal = "UNVERIFIED" + else: + terminal = "PASS" + failed_gate = next((g for g in required if res[g]["verdict"] == "fail"), None) + return {"gates": res, "required": required, "terminal": terminal, "failed_gate": failed_gate} + + +def classify_gate_failure(ev: dict) -> str | None: + """终态 FAIL 但没有阶段崩:从门给一个 error_class(喂 repair 队列与 BLOCKED 判定)。 + march_mismatch G1 march 不符(作弊/配错) + verifier_mismatch G1 hb_verifier / hbm≡bc 不一致 + accuracy_collapse G3 掉点 ≥ 10×容差 或 量化后精度接近随机(PTQ 校准崩了,如 EfficientNet/MobileNetV3 默认 int8) + accuracy_drop G3 掉点超容差但没崩 + numeric_mismatch 只有 G2 余弦不达标 + perf_over_budget 只有 G4 超预算 + """ + g = ev["gates"] + if g["G1"]["verdict"] == "fail": + failed = g["G1"].get("failed", []) + if "march_ok" in failed: + return "march_mismatch" + if "verifier_ok" in failed and g["G3"].get("verdict") != "fail": + return "verifier_mismatch" + if g["G3"]["verdict"] == "fail": + d, tol = g["G3"].get("delta"), g["G3"].get("tolerance") + q, fp = g["G3"].get("quant"), g["G3"].get("fp32") + if d is not None and tol and (abs(d) >= 10 * abs(tol) or (fp and q is not None and q < 0.2 * fp)): + return "accuracy_collapse" + return "accuracy_drop" + if g["G1"]["verdict"] == "fail" and "verifier_ok" in g["G1"].get("failed", []): + return "verifier_mismatch" + if g["G2"]["verdict"] == "fail": + return "numeric_mismatch" + if g["G4"]["verdict"] == "fail": + return "perf_over_budget" + return None + + +def cosine(a, b) -> float: + """逐输出余弦。a, b 为同 shape 的一维可迭代(numpy 数组会先 ravel)。""" + try: + import numpy as np + a = np.asarray(a, dtype=np.float64).ravel() + b = np.asarray(b, dtype=np.float64).ravel() + na, nb = np.linalg.norm(a), np.linalg.norm(b) + if na == 0 or nb == 0: + return 0.0 + return float(a @ b / (na * nb)) + except ImportError: + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)) + nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) if na and nb else 0.0 diff --git a/harness/paths.py b/harness/paths.py new file mode 100644 index 0000000..0a500fa --- /dev/null +++ b/harness/paths.py @@ -0,0 +1,85 @@ +"""根目录、config.yaml、.env.local。所有路径解析只在这里做。""" +from __future__ import annotations + +import os +from functools import lru_cache +from pathlib import Path + +import yaml + +BENCH_ROOT = Path(os.environ.get("BENCH_ROOT") or Path(__file__).resolve().parent.parent) +PROJECT_ROOT = BENCH_ROOT.parent + +REGISTRY = BENCH_ROOT / "registry" +DATA = BENCH_ROOT / "data" +CACHE = BENCH_ROOT / "cache" +STATE = Path(os.environ["BENCH_STATE_DIR"]) if os.environ.get("BENCH_STATE_DIR") else BENCH_ROOT / "state" +RUNS = BENCH_ROOT / "runs" +REPORTS = BENCH_ROOT / "reports" +DATASETS = BENCH_ROOT / "datasets" +AGENT = BENCH_ROOT / "agent" +DB_PATH = STATE / "jobs.sqlite" + +STAGE_NAMES = [ + "01-fetch", "02-baseline", "03-export", # model-level → cache/ + "04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval", # target-level → job dir +] +MODEL_LEVEL = STAGE_NAMES[:3] +TARGET_LEVEL = STAGE_NAMES[3:] +TASKS = ["cls", "det", "seg", "iseg", "pose", "mde", "stereo", "flow", "feat", "occ"] +MARCHES = ["nash-e", "nash-m", "nash-p"] + + +def expand(p: str | os.PathLike | None, base: Path = BENCH_ROOT) -> Path | None: + if p is None: + return None + p = os.path.expandvars(os.path.expanduser(str(p))) + q = Path(p) + return q if q.is_absolute() else (base / q).resolve() + + +@lru_cache(maxsize=1) +def config() -> dict: + with open(BENCH_ROOT / "config.yaml", encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + cfg.setdefault("paths", {}) + if not cfg["paths"].get("project_root"): + cfg["paths"]["project_root"] = str(PROJECT_ROOT) + return cfg + + +def venv_python(backend: str) -> Path: + """按 backend 选解释器。这是 harness 的职责,agent 无权改。""" + v = config()["paths"]["venv"].get(backend) + if not v: + raise KeyError(f"config.yaml paths.venv 没有 backend={backend}") + return expand(v) / "bin" / "python" + + +def venv_bin(backend: str, tool: str) -> Path: + return expand(config()["paths"]["venv"][backend]) / "bin" / tool + + +@lru_cache(maxsize=1) +def load_env_local() -> dict: + """读取 bench/.env.local。只给 harness 用;rollout 的 agent 子进程环境由 stagerun/rollout 单独构造,永远不含这些。""" + env = {} + p = BENCH_ROOT / ".env.local" + if p.exists(): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + env[k.strip()] = v.strip().strip('"').strip("'") + return env + + +def secret(name: str) -> str | None: + return os.environ.get(name) or load_env_local().get(name) + + +def ensure_dirs() -> None: + for d in (STATE, RUNS, REPORTS, DATASETS, DATA / "pool", DATA / "t1", DATA / "calib", + CACHE / "weights", CACHE / "onnx", CACHE / "baseline"): + d.mkdir(parents=True, exist_ok=True) diff --git a/harness/pipeline.py b/harness/pipeline.py new file mode 100644 index 0000000..3ee35bf --- /dev/null +++ b/harness/pipeline.py @@ -0,0 +1,516 @@ +"""oracle 主流程:preflight → new-run → expand → run。 + +run:每 job 顺序跑 01–08;每阶段拿对应资源池;瞬态重试;STALLED 进 long 车道; +产出 oracle/result.json;状态写 jobs.sqlite。--dry-run 走完整状态机但不调工具链。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from . import paths, db, gates, scheduler +from .errors import is_transient +from .eval import EVALUATOR_VERSION +from .registry import (JobSpec, expand_matrix, build_job_json, datasets, models, model_row, + registry_fingerprint, matrix) +from .stages import ORDER, StageContext + + +def _stamped(msg: str) -> None: + print(time.strftime("%H:%M:%S"), msg, flush=True) + + +# ---------------------------------------------------------------- preflight + +def preflight(modes: list[str] | None = None, oracle_only: bool = False) -> tuple[list[str], list[str]]: + """返回 (errors, warnings)。errors 非空 → 不允许 run。 + agent 运行时(pi/node)缺失:matrix 里 e2e/repair 开着就是 ERROR——这个工厂的产品是轨迹,不是 benchmark; + 只想跑 oracle 时显式 --oracle-only。(踩过:之前只 WARN,start_run 一路跑下去,跑了一夜零轨迹。)""" + errs, warns = [], [] + cfg = paths.config() + + # 1. registry 自洽 + try: + rows = models() + specs = expand_matrix(modes) + except Exception as e: + return [f"registry: {e}"], warns + if not specs: + errs.append("matrix 展开为空") + + # 2. venv + for be, v in cfg["paths"]["venv"].items(): + py = paths.expand(v) / "bin" / "python" + if not py.exists(): + (errs if be in {s.backend for s in specs} else warns).append(f"venv[{be}] 缺: {py}") + + # 3. .horizon / pi / node + hz = paths.expand(cfg["paths"]["horizon"]) + if not hz.exists(): + warns.append(f".horizon 不在 {hz}(agent 模式需要)") + # pi 缺失对 oracle 无所谓,但只要 matrix 里开了 e2e/repair,就意味着"一条轨迹也采不到" → 必须是 ERROR。 + # 踩过:pi 没装,preflight 只 WARN,全量 run 跑了几小时只产出 benchmark,rollouts 全停在 QUEUED。 + pi = cfg["paths"].get("pi_bin") or "pi" + pi_missing = shutil.which(pi) is None and not Path(os.path.expanduser(pi)).exists() + if pi_missing: + (errs if (any(s.mode != "oracle" for s in specs) and not oracle_only) else warns).append( + f"pi 不在 PATH:{pi}(matrix 里 e2e/repair 开着 → 轨迹采不到;scripts/install_pi.sh)") + + # 4. 数据:展开后需要的每个数据集的 pool / t1 / calib + need: dict[tuple, dict] = {} + for s in specs: + r = model_row(s.model) + need[(r.get("dataset"), s.task, r["input_spec"])] = r + dsets = datasets().get("datasets") or {} + for (ds, task, ispec), r in sorted(need.items(), key=lambda kv: str(kv[0])): + if not ds: + errs.append(f"models.csv {r['model_id']}: dataset 为空") + continue + if ds not in dsets: + errs.append(f"datasets.yaml 没有 {ds}({r['model_id']} 需要)") + continue + d = dsets[ds] + ver = d.get("version", "") + pool = paths.DATA / "pool" / ds / ver + if not (pool / "manifest.json").exists(): + errs.append(f"pool 缺 {ds}/{ver}(scripts/ingest_pool.py --dataset {ds})") + t1n = (d.get("t1") or {}).get("n") + if task != "occ" and t1n not in (0, None): + t1 = paths.DATA / "t1" / ds / f"{(d.get('t1') or {}).get('seed', 0)}.tar" + if not t1.exists(): + errs.append(f"T1 缺 {t1}(scripts/make_t1.py --dataset {ds})") + calib = paths.DATA / "calib" / task / ispec + if not (calib / "manifest.json").exists(): + if task == "occ": + warns.append(f"calib 缺 {calib}:occ 将按阶梯回退,最终 synthetic") + else: + errs.append(f"calib 缺 {calib}(scripts/make_calib.py --task {task} --input-spec {ispec})") + + # 5. 凭证 + if not paths.secret("BOARD_SSH_KEY"): + warns.append("BOARD_SSH_KEY 未设:07-board 将 unverified") + if any(s.mode != "oracle" for s in specs) and not paths.secret("TEACHER_API_KEY"): + errs.append("TEACHER_API_KEY 未设,但 matrix 里 e2e/repair 是开的:轨迹一条也采不到") + + # 6. .env.oe-package 死路径(agent 会读它) + envf = paths.PROJECT_ROOT / ".horizon" / ".env.oe-package" + if not envf.exists(): + envf = paths.PROJECT_ROOT / ".env.oe-package" + if envf.exists(): + for line in envf.read_text(errors="replace").splitlines(): + if line.startswith("PYTHON_BIN="): + p = Path(os.path.expanduser(line.split("=", 1)[1].strip().strip('"'))) + if not p.is_absolute(): + p = paths.PROJECT_ROOT / p + if not p.exists(): + errs.append(f".env.oe-package PYTHON_BIN 指向不存在的路径 {p}(scripts/fix_env.sh)") + return errs, warns + + +# ---------------------------------------------------------------- run / manifest + +def _pip_freeze_sha(backend: str) -> str | None: + py = paths.venv_python(backend) + if not py.exists(): + return None + try: + r = subprocess.run([str(py), "-m", "pip", "freeze"], capture_output=True, text=True, timeout=120) + if r.returncode == 0 and r.stdout.strip(): + return hashlib.sha256(r.stdout.encode()).hexdigest() + except Exception: + pass + # PTQ venv 没有 pip:退到 site-packages 里 dist-info 目录名的哈希,效果等价 + sp = sorted(p.name for p in (py.parent.parent / "lib").glob("python*/site-packages/*.dist-info")) + return hashlib.sha256("\n".join(sp).encode()).hexdigest() if sp else None + + +def harness_code_sha() -> str: + """harness/ + export/ 全部 .py 的内容哈希。进 manifest;变了只告警(评测口径由 EVALUATOR_VERSION 单独钉)。""" + h = hashlib.sha256() + for p in sorted(list((paths.BENCH_ROOT / "harness").rglob("*.py")) + list((paths.BENCH_ROOT / "export").rglob("*.py"))): + h.update(str(p.relative_to(paths.BENCH_ROOT)).encode()); h.update(p.read_bytes()) + return h.hexdigest() + + +def build_manifest() -> dict: + cfg = paths.config() + return { + "created_at": time.time(), + "evaluator_version": EVALUATOR_VERSION, + "harness_code_sha256": harness_code_sha(), + "registry_fingerprint": registry_fingerprint(), + "venv": {be: {"path": str(paths.expand(v)), "pip_freeze_sha256": _pip_freeze_sha(be)} + for be, v in cfg["paths"]["venv"].items()}, + "board": {"host": paths.secret("BOARD_HOST"), "ucp": None, "board_type": None}, + "host": os.uname().nodename, + } + + +def new_run(note: str = "") -> str: + paths.ensure_dirs() + with db.connect() as con: + run_id = db.next_run_id(con, time.strftime("%Y%m%d")) + man = build_manifest() + db.create_run(con, run_id, man, note) + rd = paths.RUNS / run_id + rd.mkdir(parents=True, exist_ok=True) + (rd / "manifest.json").write_text(json.dumps(man, indent=2, ensure_ascii=False)) + return run_id + + +def check_manifest(run_id: str) -> list[str]: + """job 启动前比对:两套 venv 的 pip freeze 哈希、registry 指纹。不一致拒跑。""" + man = json.loads((paths.RUNS / run_id / "manifest.json").read_text()) + now = build_manifest() + bad = [] + if man["registry_fingerprint"] != now["registry_fingerprint"]: + bad.append("registry 改了:开新 run") + for be, v in man["venv"].items(): + if v["pip_freeze_sha256"] and v["pip_freeze_sha256"] != now["venv"].get(be, {}).get("pip_freeze_sha256"): + bad.append(f"venv[{be}] 变了:开新 run") + if man.get("evaluator_version") != now["evaluator_version"]: + bad.append("EVALUATOR_VERSION 变了:开新 run") + if man.get("harness_code_sha256") and man["harness_code_sha256"] != now["harness_code_sha256"]: + print(f"WARN harness 代码与 run {run_id} 创建时不同({man['harness_code_sha256'][:8]} → {now['harness_code_sha256'][:8]});" + "结果口径若受影响请开新 run", file=sys.stderr) + return bad + + +def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | None = None) -> dict: + specs = expand_matrix(modes) + if marches: + specs = [s for s in specs if s.march in marches] # 只展开部分 march(如补跑 nash-p),不改 registry + # 按 job_id 聚合 rollout 计划 + plans: dict[str, list[dict]] = {} + first: dict[str, JobSpec] = {} + for s in specs: + first.setdefault(s.job_id, s) + if s.mode != "oracle": + plans.setdefault(s.job_id, []).append({"mode": s.mode, "policy": s.policy, "k": s.k}) + n_new = 0 + with db.connect() as con: + for s in specs: + est = scheduler.estimate_job(con, s.model, s.task) + n_new += db.upsert_job(con, run_id, s.to_row(), est) + for i in range(s.k if s.mode != "oracle" else 0): + db.rollout_upsert(con, run_id, s.job_id, i, s.mode, s.policy) + n_dirs = 0 + for jid, s in first.items(): + jd = paths.RUNS / run_id / jid + jj = jd / "job.json" + if jj.exists(): + continue + jd.mkdir(parents=True, exist_ok=True) + (jd / "oracle" / "stages").mkdir(parents=True, exist_ok=True) + (jd / "oracle" / "artifacts").mkdir(parents=True, exist_ok=True) + jj.write_text(json.dumps(build_job_json(s, run_id, EVALUATOR_VERSION, plans.get(jid)), indent=2, ensure_ascii=False)) + os.chmod(jj, 0o444) + # data -> 软链只是给 agent 引用方便;那条"一个 job 一个目录"约束产出,不约束输入 + link = jd / "data" + if not link.exists(): + try: + link.symlink_to(paths.DATA) + except OSError: + pass + n_dirs += 1 + return {"jobs_total": len(specs), "jobs_new": n_new, "job_dirs_new": n_dirs, "job_ids": len(first)} + + +# ---------------------------------------------------------------- rollouts(轨迹采集主循环) + +def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[str] | None = None, + parallel: int | None = None, dry_run: bool = False, limit: int | None = None, + log=_stamped) -> dict: + """把这个 run 里所有 QUEUED 的 rollout 跑掉 —— 这是"采轨迹"的入口,oracle 的 run_oracle 只产 benchmark。 + + 前提:该 job 的 oracle 已结束(01–03 的 cache 齐了;repair 还要求 oracle FAIL)。 + 并发受 agent 池约束(config.yaml pools.agent.workers),与 oracle 共用同一组跨进程槽位锁。 + 每个 rollout 结束即 verify → verdict.json;轨迹落在 rollouts//agent/events.jsonl。 + """ + from .rollout import run_rollout, setup_models_json + mx = matrix() + repair_ecs = set(((mx.get("modes") or {}).get("repair") or {}).get("error_classes") or []) + if not dry_run: + setup_models_json() + with db.connect() as con: + rows = [dict(r) for r in con.execute( + "SELECT r.*, j.state AS job_state, j.error_class AS job_ec FROM rollouts r" + " LEFT JOIN jobs j ON j.run_id=r.run_id AND j.job_id=r.job_id AND j.mode='oracle'" + " WHERE r.run_id=? AND r.state='QUEUED' ORDER BY r.job_id, r.kk", (run_id,))] + todo, skipped = [], {} + for r in rows: + if job_ids and r["job_id"] not in job_ids: + continue + if modes and r["mode"] not in modes: + continue + if r["job_state"] not in ("PASS", "FAIL", "UNVERIFIED"): + skipped["oracle 未完成"] = skipped.get("oracle 未完成", 0) + 1 + continue + if r["mode"] == "repair": + if r["job_state"] != "FAIL": + skipped["repair 但 oracle 没失败"] = skipped.get("repair 但 oracle 没失败", 0) + 1 + continue + if repair_ecs and r["job_ec"] not in repair_ecs: + skipped[f"repair 的 error_class 不在采集范围({r['job_ec']})"] = skipped.get( + f"repair 的 error_class 不在采集范围({r['job_ec']})", 0) + 1 + continue + todo.append(r) + if limit: + todo = todo[:limit] + log(f"rollouts: {len(todo)} 条待采,跳过 {sum(skipped.values())}({skipped or '-'})") + + def one(r: dict) -> dict: + try: + with scheduler.acquire("agent"): + return run_rollout(run_id, r["job_id"], r["kk"], r["mode"], r["policy"], dry_run=dry_run) + except SystemExit as e: # 前置缺失等:记下来,不要让整批停 + log(f"[{r['job_id']}#{r['kk']:02d}] 跳过:{e}") + return {"terminal": "SKIPPED", "reason": str(e)} + except Exception as e: # noqa: BLE001 + log(f"[{r['job_id']}#{r['kk']:02d}] 异常:{type(e).__name__}: {e}") + return {"terminal": "ERROR", "reason": repr(e)} + + n = parallel or paths.config()["pools"]["agent"]["workers"] + summary: dict = {} + with ThreadPoolExecutor(max_workers=n) as ex: + for res in ex.map(one, todo): + t = res.get("terminal", "?") + summary[t] = summary.get(t, 0) + 1 + if t not in ("SKIPPED", "ERROR"): + log(f" → {res.get('rollout_dir', '')} {t} claim_consistent={res.get('claim_consistent')} review={res.get('review')}") + log(f"rollouts done: {summary}") + return summary + + +# ---------------------------------------------------------------- rejudge + +def rejudge(run_id: str, job_id: str) -> dict: + """不重跑,只从 oracle/stages 的测量值重新过四道门、重写 result.json 与 jobs.sqlite。 + 用途:harness 汇总 bug 修复后重判;门阈值以 job.json 快照为准,不会偷偷换阈值。""" + from .verify import collect_measurements + job, jd = _load_job(run_id, job_id) + rp = jd / "oracle" / "result.json" + old = json.loads(rp.read_text()) if rp.exists() else {} + meas, summ = collect_measurements(jd, jd / "oracle" / "stages") + if not meas.get("G4") and (old.get("measurements") or {}).get("G4"): + meas["G4"] = old["measurements"]["G4"] # 补测写进来的 G4 不能被 07 的旧 SKIPPED meta 冲掉 + ev = gates.evaluate(meas, job["gates"]) + terminal = ev["terminal"] + if old.get("stage_failed") and old.get("terminal") in ("FAIL", "UNVERIFIED") and old["stage_failed"] != "08-eval": + terminal = old["terminal"] # 阶段本身没跑成的,不因为重判变 PASS + ec = old.get("error_class") if old.get("stage_failed") else (gates.classify_gate_failure(ev) if terminal == "FAIL" else None) + result = {**old, "measurements": meas, "gates": ev["gates"], "required": ev["required"], "terminal": terminal, + "failed_gate": ev["failed_gate"], "error_class": ec, "rejudged_at": time.time(), "stage_summary": summ} + rp.write_text(json.dumps(result, indent=2, ensure_ascii=False, default=str)) + with db.connect() as con: + db.set_job_state(con, run_id, job_id, terminal, error_class=result.get("error_class"), + stage_failed=result.get("stage_failed"), result=result) + return {"job_id": job_id, "terminal": terminal, "failed_gate": ev["failed_gate"], + "gates": {g: v["verdict"] for g, v in ev["gates"].items()}} + + +# ---------------------------------------------------------------- board backfill + +def board_backfill(run_id: str, job_ids: list[str] | None = None, log=_stamped) -> dict: + """给已结束但没有 G4 的 job 补板端实测(job.json 展开时 boards.yaml 还没填板型的情况)。 + 只补 march 与沙箱板一致的 job;跑 07-board 到新的 attempt 目录,然后 rejudge。""" + from .registry import board_for_march + from .stages import Board + with db.connect() as con: + jobs = db.list_jobs(con, run_id, mode="oracle") + done, skipped = 0, {} + for j in jobs: + if j["state"] not in ("PASS", "FAIL", "UNVERIFIED") or (job_ids and j["job_id"] not in job_ids): + continue + board = board_for_march(j["march"]) + if not board: + skipped["no_board_for_march"] = skipped.get("no_board_for_march", 0) + 1; continue + job, jd = _load_job(run_id, j["job_id"]) + hbm = jd / "oracle" / "artifacts" / f"{j['job_id']}.hbm" + if not hbm.exists(): + skipped["no_hbm"] = skipped.get("no_hbm", 0) + 1; continue + res = json.loads((jd / "oracle" / "result.json").read_text()) if (jd / "oracle" / "result.json").exists() else {} + if (res.get("measurements") or {}).get("G4"): + skipped["has_G4"] = skipped.get("has_G4", 0) + 1; continue + job = dict(job); job["board"] = board # 运行时注入板信息;job.json 本身不改 + stage = Board() + prev = sorted((jd / "oracle" / "stages" / stage.name).glob("a*")) if (jd / "oracle" / "stages" / stage.name).exists() else [] + ctx = StageContext(run_id=run_id, job=job, job_dir=jd, stage_root=jd / "oracle" / "stages", + artifacts=jd / "oracle" / "artifacts", rollout=-1, attempt=len(prev)) + with db.connect() as con: + db.stage_start(con, run_id, j["job_id"], stage.name, ctx.attempt, -1, "backfill") + with scheduler.acquire("board") as slot: + ctx.extra["slot"] = slot + out = stage.run(ctx) + with db.connect() as con: + db.stage_finish(con, run_id, j["job_id"], stage.name, ctx.attempt, out.state, out.exit_code, out.error_class, -1) + if out.state == "OK" and out.measurements.get("G4"): + # 把 G4 写进 result.measurements,再重判 + res.setdefault("measurements", {})["G4"] = out.measurements["G4"] + res["board_backfilled_at"] = time.time() + (jd / "oracle" / "result.json").write_text(json.dumps(res, indent=2, ensure_ascii=False, default=str)) + rejudge(run_id, j["job_id"]) + g4 = out.measurements["G4"] + log(f"[{j['job_id']}] G4 backfilled: {g4.get('latency_ms_mean'):.3f} ms mean / {g4.get('latency_ms_p95'):.3f} p95") + done += 1 + else: + log(f"[{j['job_id']}] board stage {out.state} ec={out.error_class}") + skipped["board_fail"] = skipped.get("board_fail", 0) + 1 + return {"backfilled": done, "skipped": skipped} + + +# ---------------------------------------------------------------- oracle + +def _load_job(run_id: str, job_id: str) -> tuple[dict, Path]: + jd = paths.RUNS / run_id / job_id + return json.loads((jd / "job.json").read_text()), jd + + +def run_oracle_job(run_id: str, job_id: str, dry_run: bool = False, log=_stamped, force: bool = False) -> dict: + job, jd = _load_job(run_id, job_id) + cfg = paths.config() + with db.connect() as con: + if not db.claim_job(con, run_id, job_id, force=force): + log(f"[{job_id}] 已被别的进程认领或已完成,跳过") + return {"terminal": "SKIPPED", "job_id": job_id} + ctx = StageContext(run_id=run_id, job=job, job_dir=jd, stage_root=jd / "oracle" / "stages", + artifacts=jd / "oracle" / "artifacts", rollout=-1, dry_run=dry_run) + result = {"schema": "bench.result/1", "run_id": run_id, "job_id": job_id, "round": 1, + "stages": {}, "measurements": {}, "started_at": time.time()} + terminal, error_class, stage_failed = None, None, None + for S in ORDER: + stage = S() + attempt, lane = 0, "normal" + while True: + ctx.attempt, ctx.lane = attempt, lane + with db.connect() as con: + db.stage_start(con, run_id, job_id, stage.name, attempt, -1, lane) + if stage.level == "model": + # 同一模型的三个 march job 并行:只有一个真跑 01–03,其余进锁后看到缓存 → SKIPPED + with scheduler.model_lock(job["model"]), scheduler.acquire(stage.pool) as slot: + ctx.extra["slot"] = slot + out = stage.run(ctx) + else: + with scheduler.acquire(stage.pool) as slot: + ctx.extra["slot"] = slot + out = stage.run(ctx) + with db.connect() as con: + db.stage_finish(con, run_id, job_id, stage.name, attempt, out.state, out.exit_code, out.error_class, -1) + log(f"[{job_id}] {stage.name} a{attempt:02d} {lane}: {out.state} ec={out.error_class} {out.duration_s:.1f}s") + rec = result["stages"].setdefault(stage.name, {"attempts": []}) + rec["attempts"].append({"attempt": attempt, "lane": lane, "state": out.state, "exit_code": out.exit_code, + "error_class": out.error_class, "duration_s": round(out.duration_s, 2)}) + rec.update({"state": out.state, "error_class": out.error_class}) + if out.state in ("OK", "SKIPPED"): + for k, v in (out.measurements or {}).items(): + result["measurements"][k] = v + calib_meta = (out.measurements or {}).get("calib") + if calib_meta: + ctx.extra["calib_source"] = calib_meta.get("calib_source") + break + # 失败:决定重试 + if out.state == "STALLED" and attempt < cfg["retry"]["stalled"]: + attempt, lane = attempt + 1, "long"; continue + if out.state == "FAIL" and is_transient(out.error_class or "") and attempt < cfg["retry"]["transient"]: + attempt += 1; continue + break + if out.state not in ("OK", "SKIPPED"): + ec = out.error_class or ("stalled" if out.state == "STALLED" else "unknown") + if stage.name == "07-board": + result["measurements"]["G4"] = None # unverified,不阻塞 + result["stages"][stage.name]["note"] = "board failed after retries → G4 unverified" + continue + if stage.name == "08-eval": + terminal, error_class, stage_failed = "UNVERIFIED", ec, stage.name + break + # 01–06 失败 = G1 不可能过 + result["measurements"]["G1"] = {"hbm_exists": False} + terminal, error_class, stage_failed = "FAIL", ec, stage.name + break + + ev = gates.evaluate(result["measurements"], job["gates"]) + if not error_class and (terminal or ev["terminal"]) == "FAIL": + error_class = gates.classify_gate_failure(ev) + result.update({"gates": ev["gates"], "required": ev["required"], + "terminal": terminal or ev["terminal"], "failed_gate": ev["failed_gate"], + "error_class": error_class, "stage_failed": stage_failed, "finished_at": time.time()}) + if dry_run: + result["terminal"] = "UNVERIFIED"; result["dry_run"] = True + (jd / "oracle" / "result.json").write_text(json.dumps(result, indent=2, ensure_ascii=False, default=str)) + with db.connect() as con: + db.set_job_state(con, run_id, job_id, result["terminal"], error_class=error_class, + stage_failed=stage_failed, result=result) + log(f"[{job_id}] → {result['terminal']} ec={error_class} stage={stage_failed}") + return result + + +def scheduler_alive() -> bool: + """有没有 bench run 进程活着(持有 state/locks/run-*.lock)。""" + import fcntl + for p in (paths.STATE / "locks").glob("run-*.lock"): + try: + f = open(p, "w") + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(f, fcntl.LOCK_UN) + except BlockingIOError: + return True + finally: + f.close() + except OSError: + continue + return False + + +def reset_stale(run_id: str, force: bool = False) -> dict: + """调度进程崩了/被杀后,ORACLE_RUNNING 的 job 回 PENDING,RUNNING 的 stage 记 ABORTED。有活着的调度进程时拒绝(--force 除外)。""" + if scheduler_alive() and not force: + raise SystemExit("还有 bench run 进程活着,拒绝 reset(--force 可强制)") + with db.connect() as con: + n_jobs = con.execute("UPDATE jobs SET state='PENDING', started_at=NULL WHERE run_id=? AND state='ORACLE_RUNNING'", (run_id,)).rowcount + n_st = con.execute("UPDATE stages SET state='ABORTED', finished_at=? WHERE run_id=? AND state='RUNNING'", (time.time(), run_id)).rowcount + return {"jobs_reset": n_jobs, "stages_aborted": n_st} + + +def run_oracle(run_id: str, job_ids: list[str] | None = None, dry_run: bool = False, + parallel: int | None = None, log=_stamped) -> dict: + import fcntl + (paths.STATE / "locks").mkdir(parents=True, exist_ok=True) + lockf = open(paths.STATE / "locks" / f"run-{os.getpid()}.lock", "w") + fcntl.flock(lockf, fcntl.LOCK_EX) + if not dry_run: + bad = check_manifest(run_id) + if bad: + raise SystemExit("manifest 不一致,拒跑:" + "; ".join(bad)) + with db.connect() as con: + jobs = db.list_jobs(con, run_id, mode="oracle") + if job_ids: + jobs = [j for j in jobs if j["job_id"] in job_ids] # 显式指定:允许重跑(force) + else: + jobs = [j for j in jobs if j["state"] == "PENDING"] + jobs = scheduler.order_jobs(jobs) + n = parallel or paths.config()["pools"]["cpu"]["workers"] + summary = {"PASS": 0, "FAIL": 0, "UNVERIFIED": 0} + log(f"run {run_id}: {len(jobs)} jobs, parallel={n}") + with ThreadPoolExecutor(max_workers=n) as ex: + for r in ex.map(lambda j: run_oracle_job(run_id, j["job_id"], dry_run, log, force=bool(job_ids)), jobs): + summary[r["terminal"]] = summary.get(r["terminal"], 0) + 1 + log(f"run {run_id} done: {summary}") + try: + from .report import render + render(run_id) + except Exception as e: # noqa: BLE001 + log(f"report failed: {e}") + fcntl.flock(lockf, fcntl.LOCK_UN); lockf.close() + try: + (paths.STATE / "locks" / f"run-{os.getpid()}.lock").unlink() + except OSError: + pass + return summary diff --git a/harness/registry.py b/harness/registry.py new file mode 100644 index 0000000..d30d2ec --- /dev/null +++ b/harness/registry.py @@ -0,0 +1,303 @@ +"""registry/* 加载、校验、matrix 展开、job_id、job.json 快照。 + +展开是纯函数:同一份 registry 永远展开出同一组 job。 +""" +from __future__ import annotations + +import csv +import hashlib +import json +import re +from dataclasses import dataclass, field, asdict +from functools import lru_cache +from pathlib import Path + +import yaml + +from . import paths +from .paths import REGISTRY, TASKS, MARCHES + +SEG_RE = re.compile(r"^[a-z0-9.]+$") +JOB_ID_RE = re.compile(r"^[a-z0-9.]+--[a-z0-9.]+--nash-[emp]--[a-z0-9.]+--[a-z0-9.]+$") +# march 段本身含 '-'(nash-e),是唯一允许段内出现单横线的段 + + +def _yaml(name: str) -> dict: + with open(REGISTRY / name, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +@lru_cache(maxsize=None) +def models() -> list[dict]: + rows = [] + with open(REGISTRY / "models.csv", encoding="utf-8", newline="") as f: + for r in csv.DictReader(f): + r = {k.strip(): (v or "").strip() for k, v in r.items()} + if not r.get("model_id"): + continue + rows.append(r) + validate_models(rows) + return rows + + +def validate_models(rows: list[dict]) -> None: + seen = set() + for r in rows: + mid = r["model_id"] + if not SEG_RE.match(mid): + raise ValueError(f"models.csv model_id 非法(只允许 a-z0-9.): {mid!r}") + if mid in seen: + raise ValueError(f"models.csv model_id 重复: {mid}") + seen.add(mid) + if r["task"] not in TASKS: + raise ValueError(f"models.csv {mid}: task={r['task']!r} 不在 {TASKS}") + if r["priority"] not in ("A", "B", "C"): + raise ValueError(f"models.csv {mid}: priority={r['priority']!r}") + if not re.match(r"^\d+x\d+x\d+x\d+$", r.get("input_spec", "")): + raise ValueError(f"models.csv {mid}: input_spec={r.get('input_spec')!r} 需形如 1x3x224x224") + + +@lru_cache(maxsize=None) +def matrix() -> dict: + return _yaml("matrix.yaml") + + +@lru_cache(maxsize=None) +def gates() -> dict: + return _yaml("gates.yaml") + + +@lru_cache(maxsize=None) +def budgets() -> dict: + return _yaml("budgets.yaml") + + +@lru_cache(maxsize=None) +def datasets() -> dict: + return _yaml("datasets.yaml") + + +@lru_cache(maxsize=None) +def boards() -> dict: + return _yaml("boards.yaml") + + +def registry_fingerprint() -> str: + h = hashlib.sha256() + for name in sorted(p.name for p in REGISTRY.iterdir() if p.is_file()): + h.update(name.encode()) + h.update((REGISTRY / name).read_bytes()) + return h.hexdigest() + + +# ---------------------------------------------------------------- job_id + +def make_job_id(task: str, model: str, march: str, backend: str, qcfg: str) -> str: + for seg in (task, model, backend, qcfg): + if not SEG_RE.match(seg): + raise ValueError(f"job_id 段非法: {seg!r}") + if march not in MARCHES: + raise ValueError(f"march 非法: {march!r}") + jid = f"{task}--{model}--{march}--{backend}--{qcfg}" + assert JOB_ID_RE.match(jid), jid + return jid + + +def parse_job_id(jid: str) -> dict: + if not JOB_ID_RE.match(jid): + raise ValueError(f"job_id 非法: {jid!r}") + task, model, march, backend, qcfg = jid.split("--") + return dict(task=task, model=model, march=march, backend=backend, qcfg=qcfg) + + +# ---------------------------------------------------------------- expand + +@dataclass +class JobSpec: + job_id: str + task: str + model: str + march: str + backend: str + qcfg: str + priority: str + mode: str # oracle | e2e | repair + policy: str | None = None # teacher | student | None(oracle) + k: int = 1 + data_tier: str = "t1" + gates_required: list[str] = field(default_factory=lambda: ["G1", "G2", "G3"]) + overrides: dict = field(default_factory=dict) + + def to_row(self) -> dict: + d = asdict(self) + d["gates_required"] = json.dumps(self.gates_required) + d["overrides"] = json.dumps(self.overrides, ensure_ascii=False) + return d + + +def _select(row: dict, sel: dict | None) -> bool: + # status 默认只要 ready;matrix 里 select.status 可放宽(如 [ready, needs_eval] 做 compile-only) + allowed = (sel or {}).get("status", ["ready"]) + if (row.get("status") or "ready") not in allowed: + return False + if not sel: + return True + if "priority" in sel and row["priority"] not in sel["priority"]: + return False + if "tasks" in sel and row["task"] not in sel["tasks"]: + return False + if "models" in sel and row["model_id"] not in sel["models"]: + return False + return True + + +def _apply_overrides(row: dict, mx: dict) -> dict: + out: dict = {} + for ov in mx.get("overrides", []) or []: + m = ov.get("match", {}) + if all(row.get(k if k != "model" else "model_id") == v for k, v in m.items()): + out.update({k: v for k, v in ov.items() if k != "match"}) + return out + + +def expand_matrix(modes: list[str] | None = None) -> list[JobSpec]: + """把 registry 展开成 JobSpec 列表。repair 模式的 job 在 oracle FAIL 后由 pipeline 动态追加,这里不展开。""" + mx = matrix() + rows = models() + marches_all = mx.get("marches", MARCHES) + backends_all = mx.get("backends", ["ptq"]) + qcfg_by_march = mx.get("qcfg_by_march", {}) + tier = mx.get("data_tier", "t1") + out: list[JobSpec] = [] + for mode, mcfg in (mx.get("modes") or {}).items(): + if modes and mode not in modes: + continue + if not mcfg.get("enabled", True) or mcfg.get("when") == "oracle_fail": + continue + marches = marches_all if mcfg.get("marches", "all") == "all" else mcfg["marches"] + backends = backends_all if mcfg.get("backends", "all") == "all" else mcfg["backends"] + policies = mcfg.get("policies") or [None] + k = int(mcfg.get("k", 1)) + for row in rows: + if not _select(row, mcfg.get("select")): + continue + ov = _apply_overrides(row, mx) + for march in marches: + qcfg = ov.get("qcfg") or qcfg_by_march.get(march, "int8") + for backend in backends: + for policy in policies: + jid = make_job_id(row["task"], row["model_id"], march, backend, qcfg) + out.append(JobSpec( + job_id=jid, task=row["task"], model=row["model_id"], march=march, + backend=backend, qcfg=qcfg, priority=row["priority"], mode=mode, + policy=policy, k=k, data_tier=tier, + gates_required=list(ov.get("gates", ["G1", "G2", "G3"])), + overrides=ov, + )) + return out + + +# ---------------------------------------------------------------- job.json + +def status_counts() -> dict: + from collections import Counter + return dict(Counter((r.get("status") or "ready") for r in models())) + + +def model_row(model_id: str) -> dict: + for r in models(): + if r["model_id"] == model_id: + return r + raise KeyError(model_id) + + +def dataset_for(task: str, model_id: str) -> str: + r = model_row(model_id) + return r.get("dataset") or "" + + +def board_for_march(march: str) -> dict | None: + b = boards() + for bname, bcfg in (b.get("boards") or {}).items(): + if bcfg.get("march") == march: + return {"name": bname, **bcfg} + return None + + +def gate_snapshot(spec: JobSpec) -> dict: + g = gates() + task = spec.task + g2_thr = spec.overrides.get("g2_threshold") or g["G2"].get("overrides", {}).get(task) + if isinstance(g2_thr, dict): # feat:逐输出 + g2 = {"threshold": None, "per_output": g2_thr, "n_samples": g["G2"]["n_samples"]} + else: + g2 = {"threshold": g2_thr or g["G2"]["default_threshold"], "n_samples": g["G2"]["n_samples"]} + g3_task = dict(g["G3"]["tasks"].get(task, {})) + g3 = {"required": "G3" in spec.gates_required, **g3_task, + "t1_screen_widen_se": g["G3"].get("t1_screen_widen_se", 2)} + if spec.overrides.get("g3"): + g3.update(spec.overrides["g3"]) + return { + "required": spec.gates_required, + "G1": g["G1"], + "G2": g2, + "G3": g3, + "G4": {"mandatory": False, "latency_budget_ms": spec.overrides.get("latency_budget_ms"), + "record": g["G4"]["record"]}, + } + + +def data_snapshot(spec: JobSpec) -> dict: + """job.json.data:calib / t1 / t2 的 pool 路径与 manifest sha。缺的记 null,preflight 会报。""" + row = model_row(spec.model) + ds_name = row.get("dataset") or "" + ds = (datasets().get("datasets") or {}).get(ds_name, {}) + ver = ds.get("version", "") + pool = paths.DATA / "pool" / ds_name / ver + t1_spec = ds.get("t1", {}) or {} + seed = t1_spec.get("seed", 0) + t1_tar = paths.DATA / "t1" / ds_name / f"{seed}.tar" + calib_dir = paths.DATA / "calib" / spec.task / row["input_spec"] + + def sha_of(p: Path, key: str = "sha256") -> str | None: + m = p / "manifest.json" + if m.exists(): + try: + return json.loads(m.read_text()).get(key) + except Exception: + return None + return None + + return { + "tier": spec.data_tier, + "dataset": ds_name, + "version": ver, + "calib": {"path": str(calib_dir), "n": (ds.get("calib") or {}).get("n", 128), + "manifest_sha": sha_of(calib_dir), "source": None}, + "t1": {"path": str(t1_tar), "n": t1_spec.get("n"), "seed": seed, + "manifest_sha": sha_of(t1_tar.parent) if t1_tar.parent.exists() else None, # tar 文件 sha(T1.open 校验) + "content_sha": sha_of(t1_tar.parent, "content_sha256") if t1_tar.parent.exists() else None}, # 清单内容 sha(基线缓存键) + "t2": {"path": str(pool), "manifest_sha": sha_of(pool)}, + } + + +def build_job_json(spec: JobSpec, run_id: str, evaluator_version: str, plans: list[dict] | None = None) -> dict: + """job.json 与 mode 无关:一个 job_id 一份;oracle 与所有 rollout 共用。rollout 计划单独列。""" + row = model_row(spec.model) + board = board_for_march(spec.march) + return { + "schema": "bench.job/1", + "run_id": run_id, + "job_id": spec.job_id, + "task": spec.task, "model": spec.model, "march": spec.march, + "backend": spec.backend, "qcfg": spec.qcfg, "priority": spec.priority, + "rollout_plans": plans or [], # [{mode, policy, k}] + "model_spec": row, + "board": board, + "data": data_snapshot(spec), + "gates": gate_snapshot(spec), + "calib_source_ladder": spec.overrides.get("calib_source_ladder"), + "prompt": spec.overrides.get("prompt"), + "evaluator_version": evaluator_version, + "registry_fingerprint": registry_fingerprint(), + } diff --git a/harness/report.py b/harness/report.py new file mode 100644 index 0000000..041da70 --- /dev/null +++ b/harness/report.py @@ -0,0 +1,59 @@ +"""bench report:从 jobs.sqlite 生成静态 HTML 到 reports/.html,给不上机器的人看。""" +from __future__ import annotations + +import html +import time + +from . import paths, db + +CSS = """ +body{font:14px/1.5 system-ui,sans-serif;margin:24px;color:#111;background:#fafafa} +h1{font-size:20px}h2{font-size:16px;margin-top:28px} +table{border-collapse:collapse;background:#fff;font-variant-numeric:tabular-nums} +th,td{border:1px solid #ddd;padding:4px 8px;text-align:left;font-size:13px} +th{background:#f0f0f0}code{font-family:ui-monospace,monospace;font-size:12px} +.PASS{color:#2f6b46}.FAIL{color:#9e3223}.UNVERIFIED{color:#8a6516} +""" + + +def render(run_id: str) -> str: + with db.connect() as con: + run = db.get_run(con, run_id) + jobs = db.list_jobs(con, run_id, mode="oracle") + ec = db.error_class_counts(con, run_id) + ro = con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id, kk", (run_id,)).fetchall() + by_state: dict = {} + for j in jobs: + by_state[j["state"]] = by_state.get(j["state"], 0) + 1 + out = [f"bench {run_id}", + f"

bench · {html.escape(run_id)}

", + f"

generated {time.strftime('%Y-%m-%d %H:%M:%S')} · jobs {len(jobs)} · " + + " · ".join(f"{s} {n}" for s, n in by_state.items()) + "

"] + if ec: + out.append("

failures by error_class

") + out += [f"" for r in ec] + out.append("
error_classstagen
{html.escape(str(r['error_class']))}{html.escape(str(r['stage_failed'] or ''))}{r['n']}
") + out.append("

jobs

") + import json + for j in jobs: + r = json.loads(j["result"]) if j.get("result") else {} + g = r.get("gates") or {} + d = (g.get("G3") or {}).get("delta") + lat = (g.get("G4") or {}).get("latency_ms_mean") + out.append(f"" + f"" + f"" + f"") + out.append("
job_idpristateerror_classstageest sG3 Δlat ms
{html.escape(j['job_id'])}{j['priority']}{j['state']}{html.escape(j['error_class'] or '')}{html.escape(j['stage_failed'] or '')}{int(j['estimate_s'] or 0)}{'' if d is None else f'{d:+.2f}'}{'' if lat is None else f'{lat:.2f}'}
") + if ro: + out.append("

rollouts

") + for r in ro: + out.append(f"" + f"" + f"") + out.append("
job_idkkmodepolicystateclaimconsistentreviewturnstokens
{html.escape(r['job_id'])}{r['kk']:02d}{r['mode']}{r['policy'] or ''}{r['state']}{r['claim_terminal'] or ''}{r['claim_consistent']}{'⚠' if r['review'] else ''}{r['turns'] or ''}{r['tokens'] or ''}
") + html_text = "\n".join(out) + paths.REPORTS.mkdir(parents=True, exist_ok=True) + p = paths.REPORTS / f"{run_id}.html" + p.write_text(html_text) + return str(p) diff --git a/harness/rollout.py b/harness/rollout.py new file mode 100644 index 0000000..e452e02 --- /dev/null +++ b/harness/rollout.py @@ -0,0 +1,165 @@ +"""rollout 驱动器:起一个 pi 进程对一个 job 作答一次,然后独立核验。 + + bench rollout --kk 0 --mode e2e --policy teacher + bench agent setup # 从 .env.local 渲染 ~/.pi/agent/models.json + +每次 rollout 一个目录、全新上下文、看不到其他 rollout。agent 进程环境里没有任何凭证; +LLM key 在 ~/.pi/agent/models.json(600),板卡凭证只在 harness 的 07-board 子进程里。 +""" +from __future__ import annotations + +import json +import os +import shutil +import time +from pathlib import Path + +from . import paths, db, watchdog +from .registry import gates as gates_cfg +from .stages.s02_baseline import baseline_dir +from .stages.s03_export import onnx_path +from .verify import verify_rollout + + +def rollout_dir(run_id: str, job_id: str, kk: int) -> Path: + return paths.RUNS / run_id / job_id / "rollouts" / f"{kk:02d}" + + +def _render(tmpl: str, **kw) -> str: + out = tmpl + for k, v in kw.items(): + out = out.replace("{" + k + "}", "" if v is None else str(v)) + return out + + +def setup_models_json(force: bool = False) -> Path: + """渲染 ~/.pi/agent/models.json。key 只从 .env.local 来。""" + dst = Path(os.path.expanduser("~/.pi/agent/models.json")) + if dst.exists() and not force: + return dst + cfg = paths.config()["endpoints"] + tmpl = (paths.AGENT / "models.json.tmpl").read_text() + text = _render(tmpl, + TEACHER_API_KEY=paths.secret(cfg["teacher"]["api_key_env"]) or "", + STUDENT_BASE_URL=cfg["student"].get("base_url") or "http://127.0.0.1:8000/v1", + STUDENT_API_KEY=paths.secret(cfg["student"]["api_key_env"]) or "EMPTY", + STUDENT_MODEL=cfg["student"].get("model") or "student") + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text(text) + os.chmod(dst, 0o600) + return dst + + +def prepare(run_id: str, job_id: str, kk: int, mode: str, policy: str) -> tuple[Path, dict, str]: + jd = paths.RUNS / run_id / job_id + job = json.loads((jd / "job.json").read_text()) + rd = rollout_dir(run_id, job_id, kk) + if (rd / "rollout.json").exists(): + raise SystemExit(f"rollout 已存在(append-only):{rd}。想再答一次就用下一个 kk。") + for d in ("workspace", "agent", "stages", "handles", "workspace/.pi"): + (rd / d).mkdir(parents=True, exist_ok=True) + + # 前置:model-level 产物必须已在 cache 里(由 oracle 的 01–03 产出) + onnx = onnx_path(job) + bdir = baseline_dir(job) + missing = [str(p) for p in (onnx, bdir / "metrics.json") if not p or not p.exists()] + oracle = jd / "oracle" / "result.json" + if mode == "repair" and not oracle.exists(): + missing.append(f"{oracle}(repair 需要 oracle 失败现场)") + if missing: + raise SystemExit("rollout 前置缺失,先跑 oracle(至少 01–03):\n " + "\n ".join(missing)) + + cfg = paths.config() + ep = cfg["endpoints"][policy] + settings = _render((paths.AGENT / "settings.json.tmpl").read_text(), + PROJECT_ROOT=str(paths.PROJECT_ROOT), BENCH_ROOT=str(paths.BENCH_ROOT), MODEL=ep["model"]) + sj = json.loads(settings) + sj.pop("_comment", None) + sj["defaultProvider"] = "dr-gateway" if policy == "teacher" else "student-vllm" + sj["tools"] = cfg["rollouts"]["tools"] + (rd / "workspace" / ".pi" / "settings.json").write_text(json.dumps(sj, indent=2, ensure_ascii=False)) + + # prompt + tname = job.get("prompt") or (f"{mode}.md") + tmpl = (paths.AGENT / "prompts" / tname).read_text() + g = job["gates"] + ores = json.loads(oracle.read_text()) if oracle.exists() else {} + o_stage = ores.get("stage_failed") or "" + prompt = _render( + tmpl, job_id=job_id, run_id=run_id, model=job["model"], task=job["task"], march=job["march"], + backend=job["backend"], qcfg=job["qcfg"], job_json=str(jd / "job.json"), data_dir=str(jd / "data"), + input_spec=job["model_spec"]["input_spec"], onnx=str(onnx), baseline_dir=str(bdir), + calib_dir=job["data"]["calib"]["path"], calib_n=job["data"]["calib"]["n"], + t1_tar=job["data"]["t1"]["path"], t1_n=job["data"]["t1"]["n"], + g2_threshold=g["G2"].get("threshold"), g3_metric=g["G3"].get("metric"), g3_tol=g["G3"].get("tol"), + g3_kind=g["G3"].get("kind"), max_turns=cfg["rollouts"]["max_turns"], max_tokens=cfg["rollouts"]["max_tokens"], + oracle_stage_failed=o_stage, oracle_error_class=ores.get("error_class"), + oracle_stage_dir=str(jd / "oracle" / "stages" / o_stage) if o_stage else "", + oracle_compile_yaml=str(jd / "oracle" / "stages" / "04-calibrate" / "compile.yaml"), + ) + (rd / "agent" / "prompt.md").write_text(prompt) + # 系统提示快照:pi 的 --append-system-prompt 内容不会进 agent_end.messages[], + # 导出时要靠它补回 SFT 样本的 system 消息;SYSTEM.md 以后改了也不影响已采的轨迹。 + shutil.copy2(paths.AGENT / "SYSTEM.md", rd / "agent" / "system.md") + (rd / "rollout.json").write_text(json.dumps({"schema": "bench.rollout/1", "run_id": run_id, "job_id": job_id, + "kk": kk, "mode": mode, "policy": policy, "model": ep["model"], + "state": "RUNNING", "started_at": time.time()}, indent=2)) + with db.connect() as con: + db.rollout_upsert(con, run_id, job_id, kk, mode, policy) + db.rollout_set(con, run_id, job_id, kk, state="RUNNING", started_at=time.time()) + return rd, job, prompt + + +def agent_env(rd: Path) -> dict: + """agent 进程环境:只有路径与预算,没有凭证。""" + cfg = paths.config() + path_parts = [str(paths.BENCH_ROOT / "bin")] + if cfg["paths"].get("node_bin"): + path_parts.append(str(Path(paths.expand(cfg["paths"]["node_bin"])).parent)) + path_parts.append(os.environ.get("PATH", "/usr/bin:/bin")) + return { + "PATH": ":".join(path_parts), + "HOME": os.environ.get("HOME", "/tmp"), + "LANG": "C.UTF-8", + "TERM": "dumb", + "BENCH_ROOT": str(paths.BENCH_ROOT), + "BENCH_ROLLOUT_DIR": str(rd), + "BENCH_MAX_TURNS": str(cfg["rollouts"]["max_turns"]), + "BENCH_MAX_TOKENS": str(cfg["rollouts"]["max_tokens"]), + "PYTHONPATH": str(paths.BENCH_ROOT), + } + + +def pi_command(rd: Path, prompt: str) -> list[str]: + cfg = paths.config() + pi = str(paths.expand(cfg["paths"].get("pi_bin") or "pi")) if "/" in (cfg["paths"].get("pi_bin") or "") else (cfg["paths"].get("pi_bin") or "pi") + # 实测 pi-coding-agent 0.73.1(pi --help):--mode json 单独就是非交互 JSON 事件流;没有 -a 这个参数; + # --append-system-prompt 接受文本或文件路径;--tools 是逗号分隔白名单;extensions/skills 由 workspace/.pi/settings.json 发现。 + return [pi, "--mode", "json", "--session-dir", str(rd / "agent"), + "--append-system-prompt", str(rd / "agent" / "system.md"), # 用 rollout 内的快照,与导出补回的 system 消息逐字一致 + "--tools", ",".join(cfg["rollouts"]["tools"]), + prompt] + + +def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: bool = False) -> dict: + setup_models_json() + rd, job, prompt = prepare(run_id, job_id, kk, mode, policy) + cmd = pi_command(rd, prompt) + (rd / "agent" / "cmd.json").write_text(json.dumps({"cmd": cmd, "env_keys": sorted(agent_env(rd))}, indent=2, ensure_ascii=False)) + if dry_run: + print(f"[dry-run] rollout 已准备:{rd}\n cwd={rd / 'workspace'}\n cmd={' '.join(cmd[:8])} …") + return {"rollout_dir": str(rd), "dry_run": True} + cfg = paths.config()["rollouts"] + r = watchdog.run(cmd, cwd=rd / "workspace", stdout=rd / "agent" / "events.jsonl", stderr=rd / "agent" / "pi.stderr.log", + env=agent_env(rd), stall_s=cfg["hard_ceiling_s"], poll_s=30, hard_ceiling_s=cfg["hard_ceiling_s"]) + rj = json.loads((rd / "rollout.json").read_text()) + if r.state == "STALLED" or r.exit_code == 75: + rj["state"] = "TIMEOUT"; rj.setdefault("timeout_reason", r.stalled_reason or "budget") + elif rj.get("state") != "TIMEOUT": + rj["state"] = "DONE" + rj.update({"exit_code": r.exit_code, "wall_s": round(r.duration_s, 1), "finished_at": time.time()}) + (rd / "rollout.json").write_text(json.dumps(rj, indent=2)) + with db.connect() as con: + db.rollout_set(con, run_id, job_id, kk, state="VERIFYING") + v = verify_rollout(rd) + return {"rollout_dir": str(rd), "terminal": v["terminal"], "claim_consistent": v["claim_consistent"], "review": v["review"]} diff --git a/harness/scheduler.py b/harness/scheduler.py new file mode 100644 index 0000000..fc69d66 --- /dev/null +++ b/harness/scheduler.py @@ -0,0 +1,150 @@ +"""资源池、板卡锁、耗时估算、调度顺序。 + +资源池是**跨进程**的:每个池 N 个槽位,每个槽位一个 flock 文件(state/locks/-.lock)。 +oracle 流水线、agent 的 `bench stage run`、多个并行的 `bench run` 都通过同一组锁文件分配槽位, +所以不会因为开了两个进程就把 cpu 池当成 16 个、把板子当成两块。 +gpu 池的槽位号即 GPU 号(CUDA_VISIBLE_DEVICES)。board 池只有 1 个槽 = 严格串行。 +estimate 只用于排序与告警,永远不用于杀进程。 +注意:锁文件所在的 state 目录必须是所有进程共用的同一个(BENCH_STATE_DIR 要一致)。 +""" +from __future__ import annotations + +import fcntl +import os +import time +from contextlib import contextmanager +from pathlib import Path + +from . import paths +from .registry import budgets, model_row + + +def _lock_dir() -> Path: + d = paths.STATE / "locks" + d.mkdir(parents=True, exist_ok=True) + return d + + +def pool_size(pool: str) -> int: + return int(paths.config()["pools"].get(pool, {}).get("workers", 1)) + + +def pool_reserve(pool: str) -> int: + """给 agent 预留的槽位数:oracle 拿不到这些槽,rollout 的 `bench stage run` 优先拿。 + 实测(2026-09-15):两个 oracle 调度进程把 cpu 池 8/8 占满,第一条 rollout 的 05-compile 在队列里饿了 60 分钟。 + 轨迹是这个工厂的产品,benchmark 只是副产品——不能让 oracle 把 agent 饿死。""" + n = pool_size(pool) + r = int(paths.config()["pools"].get(pool, {}).get("reserve_for_agent", 0) or 0) + return max(0, min(r, n - 1)) + + +def _try_slots(pool: str, order: list[int]): + d = _lock_dir() + for i in order: + f = open(d / f"{pool}-{i}.lock", "w") + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + f.write(f"{os.getpid()} {time.time()}\n"); f.flush() + return f, i + except BlockingIOError: + f.close() + return None, None + + +@contextmanager +def acquire(pool: str, poll_s: float = 2.0, lane: str = "normal"): + """拿一个池槽位(跨进程)。yield 槽位号(gpu 池 = 槽位 → devices[slot])。 + + lane="normal"(oracle)只用 [reserve, n);lane="agent"(rollout 的 stage)可用全部,且先抢保留槽 [0, reserve)。 + """ + n, reserve = pool_size(pool), pool_reserve(pool) + order = (list(range(reserve)) + list(range(reserve, n))) if lane == "agent" else list(range(reserve, n)) + fh = slot = None + while fh is None: + fh, slot = _try_slots(pool, order) + if fh is None: + time.sleep(poll_s) + try: + yield slot + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + fh.close() + + +@contextmanager +def model_lock(model: str): + """model-level 阶段(01–03)按模型互斥:同一模型的三个 march job 并行时只有一个真跑,其余看到缓存 SKIPPED。""" + f = open(_lock_dir() / f"model-{model}.lock", "w") + fcntl.flock(f, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(f, fcntl.LOCK_UN) + f.close() + + +def pool_usage() -> dict[str, int]: + """每个池当前被占的槽位数(bench status 用)。""" + out = {} + d = _lock_dir() + for pool in paths.config()["pools"]: + busy = 0 + for i in range(pool_size(pool)): + p = d / f"{pool}-{i}.lock" + if not p.exists(): + continue + f = open(p, "w") + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(f, fcntl.LOCK_UN) + except BlockingIOError: + busy += 1 + finally: + f.close() + out[pool] = busy + return out + + +# ---------------------------------------------------------------- 估算 + +def _params_m(model: str) -> float | None: + r = model_row(model) + try: + return float(r.get("params_m") or "") + except ValueError: + return None + + +def estimate_stage(con, model: str, task: str, stage: str) -> float: + from .db import stage_history_by_model + hist = stage_history_by_model(con, model, stage) + if len(hist) >= 2: + hist = sorted(hist) + return hist[min(len(hist) - 1, int(0.95 * len(hist)))] + if len(hist) == 1: + return hist[0] + b = budgets() + pm = _params_m(model) or 30.0 + key = stage.split("-", 1)[1] + row = next((r for r in b["by_params_m"] if r["max"] is None or pm <= r["max"]), b["by_params_m"][-1]) + base = row.get(key, 300) + return base * b["task_multiplier"].get(task, 1.0) + + +def estimate_job(con, model: str, task: str) -> float: + return sum(estimate_stage(con, model, task, s) for s in ("05-compile", "07-board", "08-eval")) + + +def p95_alert_threshold(con, model: str, stage: str) -> float | None: + from .db import stage_history_by_model + hist = stage_history_by_model(con, model, stage) + if len(hist) < 3: + return None + hist = sorted(hist) + return 3 * hist[int(0.95 * (len(hist) - 1))] + + +def order_jobs(jobs: list[dict]) -> list[dict]: + """A/B/C 优先级,同级内估算短的先;C 类单独排最后(低优先级车道)。""" + pri = {"A": 0, "B": 1, "C": 2} + return sorted(jobs, key=lambda j: (pri.get(j.get("priority"), 3), j.get("estimate_s") or 1e9, j["job_id"])) diff --git a/harness/stagerun.py b/harness/stagerun.py new file mode 100644 index 0000000..690d18c --- /dev/null +++ b/harness/stagerun.py @@ -0,0 +1,161 @@ +"""agent 可用的 `bench stage run / wait / status`。 + +agent 不在 bash 里直接跑 hb_compile。它提交: + bench stage run compile --config workspace/compile.yaml → 打印 handle + bench stage wait --poll 60 → 阻塞到完成,打印结果与日志位置 +harness 在 agent 进程外执行阶段:按 backend 选 venv、拿资源池、看门狗、日志落 rollout/stages/。 +四个好处:编译时长与 pi 的 bash 超时脱钩;venv 不可能选错;预算与看门狗一视同仁;日志位置统一。 + +rollout 上下文由 env BENCH_ROLLOUT_DIR 给定(rollout 驱动器设置);agent 无法指向别的 job。 +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +from . import paths, db, scheduler +from .stages import BY_NAME, AGENT_RUNNABLE, StageContext, short + + +def rollout_dir() -> Path: + d = os.environ.get("BENCH_ROLLOUT_DIR") + if not d: + raise SystemExit("BENCH_ROLLOUT_DIR 未设:bench stage 只能在 rollout 内使用") + return Path(d).resolve() + + +def _ctx_from_rollout(rd: Path, agent_config: Path | None) -> StageContext: + jd = rd.parent.parent + job = json.loads((jd / "job.json").read_text()) + kk = int(rd.name) + return StageContext(run_id=job["run_id"], job=job, job_dir=jd, stage_root=rd / "stages", + artifacts=rd / "workspace" / "artifacts", rollout=kk, agent_config=agent_config) + + +def submit(stage: str, config: str | None, poll_hint: int = 60) -> str: + rd = rollout_dir() + name = short(stage) + if name not in AGENT_RUNNABLE: + raise SystemExit(f"阶段 {name} 不允许由 agent 提交(可用:{sorted(AGENT_RUNNABLE)})") + cfg_path = None + if config: + cfg_path = Path(config).resolve() + ws = (rd / "workspace").resolve() + if ws not in cfg_path.parents: + raise SystemExit(f"--config 必须在 workspace/ 内:{cfg_path}") + if not cfg_path.exists(): + raise SystemExit(f"--config 不存在:{cfg_path}") + hid = f"{name}-{time.strftime('%H%M%S')}-{uuid.uuid4().hex[:6]}" + hd = rd / "handles" / hid + hd.mkdir(parents=True, exist_ok=True) + (hd / "request.json").write_text(json.dumps({"stage": name, "config": str(cfg_path) if cfg_path else None, + "submitted_at": time.time()}, indent=2)) + (hd / "status.json").write_text(json.dumps({"state": "QUEUED"})) + # 脱离 agent 进程;环境里去掉凭证(07-board 自己从 .env.local 取) + env = {k: v for k, v in os.environ.items() if not k.endswith(("_API_KEY", "_TOKEN"))} + env["BENCH_ROOT"] = str(paths.BENCH_ROOT) + env["PYTHONPATH"] = str(paths.BENCH_ROOT) + py = paths.config()["paths"].get("harness_python", "python3") + with open(hd / "runner.log", "ab") as lf: + subprocess.Popen([py, "-m", "harness.stagerun", "--exec", str(hd)], cwd=str(paths.BENCH_ROOT), env=env, + stdout=lf, stderr=subprocess.STDOUT, start_new_session=True) + print(hid) + print(f"# wait: bench stage wait {hid} --poll {poll_hint}") + return hid + + +def _exec(hd: Path) -> int: + rd = hd.parent.parent + req = json.loads((hd / "request.json").read_text()) + ctx = _ctx_from_rollout(rd, Path(req["config"]) if req.get("config") else None) + S = BY_NAME[req["stage"]] + stage = S() + # attempt = 该 rollout 内此阶段已提交次数(agent 自己的重试是同一条轨迹) + prev = sorted((rd / "stages" / stage.name).glob("a*")) if (rd / "stages" / stage.name).exists() else [] + ctx.attempt = len(prev) + # 06-hbm 需要 04 的 calib_source 传给 08 + cm = rd / "stages" / "04-calibrate" / "meta.json" + if cm.exists(): + try: + ctx.extra["calib_source"] = json.loads(cm.read_text()).get("calib_source") + except Exception: + pass + # QUEUED = 已提交、在等池槽位;RUNNING = 真的在跑。两者分开,否则看板上分不出"排队 60 分钟"和"编译 60 分钟"。 + (hd / "status.json").write_text(json.dumps({"state": "QUEUED", "pool": stage.pool, "attempt": ctx.attempt, + "queued_at": time.time()})) + with db.connect() as con: + db.stage_start(con, ctx.run_id, ctx.job["job_id"], stage.name, ctx.attempt, ctx.rollout, lane="queued") + with scheduler.acquire(stage.pool, lane="agent") as slot: # agent 优先拿保留槽,不被 oracle 饿死 + (hd / "status.json").write_text(json.dumps({"state": "RUNNING", "pool": stage.pool, "slot": slot, + "attempt": ctx.attempt, "started_at": time.time()})) + with db.connect() as con: # 拿到槽位才算真的开始跑:看板据此区分"排队"与"在算" + con.execute("UPDATE stages SET lane='agent', started_at=? WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?", + (time.time(), ctx.run_id, ctx.job["job_id"], ctx.rollout, stage.name, ctx.attempt)) + ctx.extra["slot"] = slot + out = stage.run(ctx) + with db.connect() as con: + db.stage_finish(con, ctx.run_id, ctx.job["job_id"], stage.name, ctx.attempt, out.state, out.exit_code, out.error_class, ctx.rollout) + sd = rd / "stages" / stage.name / f"a{ctx.attempt:02d}" + status = {"state": "DONE", "result": out.state, "exit_code": out.exit_code, "error_class": out.error_class, + "duration_s": round(out.duration_s, 1), "attempt": ctx.attempt, + "stdout": str(sd / "stdout.log"), "stderr": str(sd / "stderr.log"), "meta": str(sd / "meta.json"), + "measurements": out.measurements, "finished_at": time.time()} + (hd / "status.json").write_text(json.dumps(status, indent=2, ensure_ascii=False, default=str)) + return 0 + + +def wait(handle: str, poll: int = 60, quiet: bool = False, max_wait_s: int | None = None) -> int: + """阻塞到该阶段结束。每轮打一行心跳(排队中 / 运行中 + 已耗时),这样 agent 侧看得见进展, + pi 的 bash 工具也不会因为"长时间无输出"而误判。max_wait_s 到点就返回 2 并提示继续等(不是失败)。""" + hd = rollout_dir() / "handles" / handle + if not hd.exists(): + raise SystemExit(f"handle 不存在:{handle}") + t0 = time.time() + while True: + st = json.loads((hd / "status.json").read_text()) + if st.get("state") == "DONE": + break + if not quiet: + since = st.get("started_at") or st.get("queued_at") or t0 + print(f"[{time.strftime('%H:%M:%S')}] {handle}: {st.get('state')}" + f"{'(等 ' + st.get('pool', '?') + ' 池槽位)' if st.get('state') == 'QUEUED' else ''}" + f" 已 {int(time.time() - since)}s", flush=True) + if max_wait_s and time.time() - t0 > max_wait_s: + print(f"(已等 {int(time.time() - t0)}s 仍未结束;这不是失败,再调一次 bench stage wait {handle} 继续等)") + return 2 + time.sleep(poll) + _print_status(st) + return 0 if st["result"] in ("OK", "SKIPPED") else 1 + + +def status(handle: str) -> int: + hd = rollout_dir() / "handles" / handle + st = json.loads((hd / "status.json").read_text()) + _print_status(st) + return 0 + + +def _print_status(st: dict) -> None: + if st.get("state") != "DONE": + print(json.dumps(st)); return + print(f"result: {st['result']} exit_code: {st['exit_code']} error_class: {st['error_class']} {st['duration_s']}s") + print(f"stdout: {st['stdout']}\nstderr: {st['stderr']}\nmeta: {st['meta']}") + if st.get("measurements"): + print("measurements: " + json.dumps(st["measurements"], ensure_ascii=False)) + if st["result"] != "OK": + try: + tail = Path(st["stderr"]).read_text(errors="replace").splitlines()[-30:] + print("--- stderr tail ---"); print("\n".join(tail)) + except OSError: + pass + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--exec": + sys.exit(_exec(Path(sys.argv[2]))) + sys.exit(2) diff --git a/harness/stages/__init__.py b/harness/stages/__init__.py new file mode 100644 index 0000000..0227a4c --- /dev/null +++ b/harness/stages/__init__.py @@ -0,0 +1,25 @@ +"""01–03 model-level(进 cache/,每模型一次);04–08 target-level(每 job 一遍)。""" +from .base import Stage, StageContext, StageOutcome +from .s01_fetch import Fetch +from .s02_baseline import Baseline +from .s03_export import Export +from .s04_calibrate import Calibrate +from .s05_compile import Compile +from .s06_hbm import Hbm +from .s07_board import Board +from .s08_eval import Eval + +ORDER: list[type[Stage]] = [Fetch, Baseline, Export, Calibrate, Compile, Hbm, Board, Eval] +BY_NAME: dict[str, type[Stage]] = {s.name: s for s in ORDER} +# agent 通过 `bench stage run` 可提交的阶段(重阶段由 harness 执行) +AGENT_RUNNABLE = {"04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"} + + +def short(name: str) -> str: + """'compile' → '05-compile';已带前缀则原样。""" + if name in BY_NAME: + return name + for k in BY_NAME: + if k.split("-", 1)[1] == name: + return k + raise KeyError(f"未知阶段: {name}") diff --git a/harness/stages/base.py b/harness/stages/base.py new file mode 100644 index 0000000..ed75205 --- /dev/null +++ b/harness/stages/base.py @@ -0,0 +1,149 @@ +"""Stage 基类:每阶段独立子目录,落 stdout / stderr / exit_code / meta.json / 耗时。 + +oracle 与 rollout 的 `bench stage run` 走同一份 Stage 代码,区别只在 ctx.stage_root 与 ctx.rollout。 +""" +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from pathlib import Path + +from .. import paths, watchdog +from ..errors import classify + + +@dataclass +class StageContext: + run_id: str + job: dict # job.json + job_dir: Path # runs// + stage_root: Path # …/oracle/stages 或 …/rollouts/kk/stages + artifacts: Path # …/oracle/artifacts 或 …/rollouts/kk/workspace/artifacts + rollout: int = -1 # -1 = oracle + attempt: int = 0 + lane: str = "normal" + dry_run: bool = False + agent_config: Path | None = None # rollout 提交的配置(workspace/x.yaml),oracle 为 None + extra: dict = field(default_factory=dict) + + @property + def cfg(self) -> dict: + return paths.config() + + def watchdog_params(self) -> dict: + wd = dict(self.cfg["watchdog"]) + from ..registry import budgets + ov = (budgets().get("watchdog_overrides") or {}).get(self.job["task"]) or {} + wd.update(ov) + if self.lane == "long": + wd["stall_s"] = max(wd["stall_s"], 3600) + return wd + + +@dataclass +class StageOutcome: + state: str # OK | FAIL | STALLED | SKIPPED + exit_code: int | None = None + error_class: str | None = None + duration_s: float = 0.0 + meta: dict = field(default_factory=dict) + measurements: dict = field(default_factory=dict) # 贡献给 gates.evaluate 的片段,如 {"G1": {...}} + + +class Stage: + name: str = "00-none" + level: str = "target" # model | target + pool: str = "cpu" # gpu | cpu | board + retry_transient: bool = True + + # ------------------------------------------------------------ 子类实现 + def cached(self, ctx: StageContext) -> dict | None: + """model-level 阶段:已有缓存则返回 meta(跳过执行)。""" + return None + + def command(self, ctx: StageContext) -> list[str] | None: + """返回要执行的命令;None 表示本阶段是纯 python(重写 native)。""" + return None + + def env(self, ctx: StageContext) -> dict: + """子进程环境。默认继承但去掉凭证;07-board 自己加板卡变量。""" + env = {k: v for k, v in os.environ.items() if not k.endswith(("_API_KEY", "_TOKEN")) and not k.startswith("BOARD_")} + env["BENCH_ROOT"] = str(paths.BENCH_ROOT) + env["BENCH_JOB_DIR"] = str(ctx.job_dir) + env["PYTHONPATH"] = str(paths.BENCH_ROOT) + (":" + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") + slot = ctx.extra.get("slot") + if self.pool == "gpu" and slot is not None: + devices = paths.config()["pools"]["gpu"].get("devices") or list(range(paths.config()["pools"]["gpu"]["workers"])) + env["CUDA_VISIBLE_DEVICES"] = str(devices[slot % len(devices)]) # 槽位 → 指定物理卡 + elif self.pool != "gpu": + env["CUDA_VISIBLE_DEVICES"] = "" # cpu / board 阶段不碰 GPU + return env + + def native(self, ctx: StageContext, stage_dir: Path) -> StageOutcome: + raise NotImplementedError + + def measure(self, ctx: StageContext, stage_dir: Path, outcome: StageOutcome) -> dict: + """执行完成后提取测量值片段。默认无。""" + return {} + + # ------------------------------------------------------------ 框架 + def stage_dir(self, ctx: StageContext) -> Path: + d = ctx.stage_root / self.name / f"a{ctx.attempt:02d}" + d.mkdir(parents=True, exist_ok=True) + return d + + def run(self, ctx: StageContext) -> StageOutcome: + sd = self.stage_dir(ctx) + t0 = time.time() + meta = {"stage": self.name, "level": self.level, "pool": self.pool, "attempt": ctx.attempt, + "lane": ctx.lane, "rollout": ctx.rollout, "started_at": t0, "dry_run": ctx.dry_run} + try: + c = self.cached(ctx) + if c is not None and not ctx.dry_run: + out = StageOutcome("SKIPPED", 0, None, 0.0, {"cached": c}) + elif ctx.dry_run: + try: + would = self.command(ctx) + except Exception as e: # 上游缓存不存在时 command() 会抛;dry-run 只记录 + would = f"" + out = StageOutcome("OK", 0, None, 0.0, {"dry_run": True, "would_run": would}) + else: + cmd = self.command(ctx) + if cmd is None: + out = self.native(ctx, sd) + else: + wd = ctx.watchdog_params() + r = watchdog.run(cmd, cwd=sd, stdout=sd / "stdout.log", stderr=sd / "stderr.log", + env=self.env(ctx), stall_s=wd["stall_s"], cpu_idle_pct=wd["cpu_idle_pct"], + poll_s=wd["poll_s"], hard_ceiling_s=wd["hard_ceiling_s"]) + ec = None + if r.state != "OK": + ec = classify(_tail(sd / "stderr.log"), _tail(sd / "stdout.log"), r.exit_code) + out = StageOutcome(r.state, r.exit_code, ec, r.duration_s, + {"cmd": cmd, "stalled": r.stalled_reason, + "io_blocked_s": getattr(r, "io_blocked_s", 0.0)}) + if out.state == "OK": + # native() 可能已经放了测量值(06-hbm 的 G1);measure() 只在有返回时覆盖。踩过:这里曾把 G1 抹成 {} → job UNVERIFIED + out.measurements = self.measure(ctx, sd, out) or out.measurements or {} + except Exception as e: # harness 自身的 bug 也要落盘,不能吞 + (sd / "stderr.log").open("a").write(f"\n[harness exception] {type(e).__name__}: {e}\n") + out = StageOutcome("FAIL", -1, "unknown", time.time() - t0, {"harness_exception": repr(e)}) + out.duration_s = out.duration_s or (time.time() - t0) + meta.update({"state": out.state, "exit_code": out.exit_code, "error_class": out.error_class, + "duration_s": round(out.duration_s, 3), "finished_at": time.time(), + "measurements": out.measurements, **out.meta}) + (sd / "exit_code").write_text(str(out.exit_code)) + (sd / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2, default=str)) + # 阶段级 meta.json = 最新 attempt 的副本 + (ctx.stage_root / self.name / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2, default=str)) + return out + + +def _tail(p: Path, n: int = 400) -> str: + try: + lines = p.read_text(errors="replace").splitlines() + return "\n".join(lines[-n:]) + except OSError: + return "" diff --git a/harness/stages/s01_fetch.py b/harness/stages/s01_fetch.py new file mode 100644 index 0000000..164ea92 --- /dev/null +++ b/harness/stages/s01_fetch.py @@ -0,0 +1,65 @@ +"""01-fetch:权重 → cache/weights//。content-addressed,跨 run 复用。""" +from __future__ import annotations + +import hashlib +import json +import shutil +from pathlib import Path + +from .. import paths +from .base import Stage, StageContext, StageOutcome + +INDEX = paths.CACHE / "weights" / "index.json" + + +def _index() -> dict: + return json.loads(INDEX.read_text()) if INDEX.exists() else {} + + +def weights_dir(model_id: str) -> Path | None: + sha = _index().get(model_id, {}).get("sha256") + return (paths.CACHE / "weights" / sha) if sha else None + + +def weights_file(model_id: str) -> Path | None: + d = weights_dir(model_id) + if not d: + return None + files = [p for p in d.iterdir() if p.name != "meta.json"] + return files[0] if files else None + + +class Fetch(Stage): + name = "01-fetch" + level = "model" + pool = "cpu" + + def cached(self, ctx: StageContext): + d = weights_dir(ctx.job["model"]) + return {"weights_dir": str(d)} if d and d.exists() else None + + def command(self, ctx: StageContext): + tmp = paths.CACHE / "weights" / f"_tmp-{ctx.job['model']}" + tmp.mkdir(parents=True, exist_ok=True) + # torchvision:// 与 hf:// 需要 torch/hf 环境 → bench venv;http(s) 直接下 + return [str(paths.venv_python("bench")), "-m", "harness.tools.fetch_weights", + "--model", ctx.job["model"], "--uri", ctx.job["model_spec"]["weights"], "--out-dir", str(tmp)] + + def measure(self, ctx: StageContext, stage_dir: Path, outcome: StageOutcome) -> dict: + tmp = paths.CACHE / "weights" / f"_tmp-{ctx.job['model']}" + files = [p for p in tmp.iterdir() if p.is_file()] + if not files: + raise RuntimeError("fetch 未产出文件") + f = files[0] + sha = hashlib.sha256(f.read_bytes()).hexdigest() + dst = paths.CACHE / "weights" / sha + if not dst.exists(): + dst.mkdir(parents=True) + shutil.move(str(f), dst / f.name) + (dst / "meta.json").write_text(json.dumps({"model": ctx.job["model"], "uri": ctx.job["model_spec"]["weights"], + "sha256": sha, "size": (dst / f.name).stat().st_size}, indent=2)) + shutil.rmtree(tmp, ignore_errors=True) + idx = _index() + idx[ctx.job["model"]] = {"sha256": sha, "file": f.name} + INDEX.write_text(json.dumps(idx, indent=2, ensure_ascii=False)) + return {"weights_sha256": sha} diff --git a/harness/stages/s02_baseline.py b/harness/stages/s02_baseline.py new file mode 100644 index 0000000..7527c9d --- /dev/null +++ b/harness/stages/s02_baseline.py @@ -0,0 +1,48 @@ +"""02-baseline:FP32 指标(T1)+ 逐输出张量样本(G2 用)→ cache/baseline/--/。 +基线自跑,不采信官方数。评测器与 08-eval 完全同一份代码(harness/eval/)。""" +from __future__ import annotations + +import json +from pathlib import Path + +from .. import paths +from .base import Stage, StageContext, StageOutcome +from .s01_fetch import weights_file + + +def baseline_dir(job: dict) -> Path: + return paths.CACHE / "baseline" / f"{job['model']}--{job['data']['dataset']}" + + +class Baseline(Stage): + name = "02-baseline" + level = "model" + pool = "gpu" + + def cached(self, ctx: StageContext): + d = baseline_dir(ctx.job) + m = d / "metrics.json" + if m.exists(): + mj = json.loads(m.read_text()) + # 基线必须与当前 T1 子集、评测器版本一致,否则 delta 不可比 + if mj.get("t1_manifest_sha") == ctx.job["data"]["t1"].get("content_sha") and \ + mj.get("evaluator_version") == ctx.job["evaluator_version"]: + return {"baseline_dir": str(d), "metrics": mj.get("metrics")} + return None + + def command(self, ctx: StageContext): + job = ctx.job + out = baseline_dir(job) + out.mkdir(parents=True, exist_ok=True) + return [str(paths.venv_python("bench")), "-m", "harness.eval.run", + "--mode", "fp32", "--task", job["task"], "--model", job["model"], + "--source", job["model_spec"]["source"], "--weights", str(weights_file(job["model"])), + "--input-spec", job["model_spec"]["input_spec"], "--resize", str(job["model_spec"].get("resize") or 0), + "--t1", job["data"]["t1"]["path"], "--t1-sha", str(job["data"]["t1"]["manifest_sha"]), + "--g2-samples", str(job["gates"]["G2"]["n_samples"]), + "--evaluator-version", job["evaluator_version"], + "--out", str(out)] + + def measure(self, ctx, stage_dir, outcome) -> dict: + m = json.loads((baseline_dir(ctx.job) / "metrics.json").read_text()) + return {"baseline": m.get("metrics")} diff --git a/harness/stages/s03_export.py b/harness/stages/s03_export.py new file mode 100644 index 0000000..8059a3b --- /dev/null +++ b/harness/stages/s03_export.py @@ -0,0 +1,52 @@ +"""03-export:权重 → ONNX → cache/onnx//.onnx。export_fail 是 model-level:一次影响该模型所有 job。""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from .. import paths +from ..backends import get as backend +from .base import Stage, StageContext, StageOutcome +from .s01_fetch import weights_file, _index + + +def onnx_path(job: dict) -> Path | None: + sha = _index().get(job["model"], {}).get("sha256") + return (paths.CACHE / "onnx" / sha / f"{job['model']}.onnx") if sha else None + + +class Export(Stage): + name = "03-export" + level = "model" + pool = "cpu" + + def cached(self, ctx: StageContext): + p = onnx_path(ctx.job) + if not (p and p.exists()): + return None + # 导出口径版本要一致,否则重导(踩过:ultralytics 旧导出留下的 ONNX 被新 run 直接复用) + from ..tools.export_onnx import EXPORTER_VERSION + meta = p.parent / "meta.json" + try: + if json.loads(meta.read_text()).get("exporter_version") != EXPORTER_VERSION: + return None + except Exception: + return None + return {"onnx": str(p)} + + def command(self, ctx: StageContext): + p = onnx_path(ctx.job) + if p is None: + raise RuntimeError("01-fetch 未登记权重 sha,无法定位 onnx 缓存目录") + p.parent.mkdir(parents=True, exist_ok=True) + return backend(ctx.job["backend"]).export_cmd(ctx.job, p, weights_file(ctx.job["model"])) + + def measure(self, ctx, stage_dir, outcome) -> dict: + p = onnx_path(ctx.job) + sha = hashlib.sha256(p.read_bytes()).hexdigest() + from ..tools.export_onnx import EXPORTER_VERSION + (p.parent / "meta.json").write_text(json.dumps({"model": ctx.job["model"], "onnx_sha256": sha, + "input_spec": ctx.job["model_spec"]["input_spec"], + "exporter_version": EXPORTER_VERSION}, indent=2)) + return {"onnx_sha256": sha} diff --git a/harness/stages/s04_calibrate.py b/harness/stages/s04_calibrate.py new file mode 100644 index 0000000..8d7d8a1 --- /dev/null +++ b/harness/stages/s04_calibrate.py @@ -0,0 +1,109 @@ +"""04-calibrate:确认校准集就位、按 occ 阶梯选来源、生成编译配置(PTQ 的 hb_compile yaml)。 + +纯 python,不跑工具。rollout 里 agent 提交的 --config 在这里与 harness 钉死项合并。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from .. import paths +from ..backends import ptq +from .base import Stage, StageContext, StageOutcome +from .s03_export import onnx_path + + +def calib_dir(job: dict) -> Path: + return Path(job["data"]["calib"]["path"]) + + +def calib_npy_dir(cd: Path) -> Path: + """hb_compile 的 cal_data_dir:只含 .npy 的子目录(manifest.json 在上一级)。""" + return cd / "npy_f32" if (cd / "npy_f32").exists() else cd + + +def compile_yaml_path(ctx: StageContext) -> Path: + return ctx.stage_root / "04-calibrate" / "compile.yaml" + + +def work_dir(ctx: StageContext) -> Path: + """hb_compile 的 working_dir:中间产物(几个 ONNX + bc + hbm,几百 MB)都写这里。 + + 默认放本地盘(config.yaml paths.work_root),不放 runs/。实测 2026-09-15: + JuiceFS 写 1 MB 要 360 ms,本地盘 3 ms(慢 118 倍),大模型编译会长时间卡在 D 状态, + 看起来就像挂死——EfficientNet-B4/B5 因此被看门狗误杀。最终产物由 06-hbm 收进 runs/ 的 artifacts/。 + work_root 为空则退回 runs/ 内(单机小规模时够用)。 + """ + root = paths.config()["paths"].get("work_root") + if not root: + return ctx.stage_root / "05-compile" / "work" + rel = f"{ctx.run_id}/{ctx.job['job_id']}" + (f"/r{ctx.rollout:02d}" if ctx.rollout >= 0 else "/oracle") + return paths.expand(root) / rel / "05-compile-work" + + +class Calibrate(Stage): + name = "04-calibrate" + level = "target" + pool = "cpu" + + def native(self, ctx: StageContext, stage_dir: Path) -> StageOutcome: + job = ctx.job + cd = calib_dir(job) + source = "t0" + ladder = job.get("calib_source_ladder") + if ladder: + # occ:repo_demo → nuscenes_128 → synthetic,逐级回退并记录用了哪级 + for lvl in ladder: + cand = cd.parent / f"{cd.name}--{lvl}" + if cand.exists() and (cand / "manifest.json").exists(): + cd, source = cand, lvl + break + else: + cd, source = cd.parent / f"{cd.name}--synthetic", "synthetic" + if not cd.exists(): + cd.mkdir(parents=True, exist_ok=True) + _make_synthetic(cd, job["model_spec"]["input_spec"], job["data"]["calib"]["n"]) + if not (cd / "manifest.json").exists(): + (stage_dir / "stderr.log").write_text(f"calib manifest missing: {cd}\n") + return StageOutcome("FAIL", 2, "unknown", 0.0, {"reason": "calib_missing", "calib_dir": str(cd)}) + manifest = json.loads((cd / "manifest.json").read_text()) + + agent_overrides = None + if ctx.agent_config and ctx.agent_config.exists(): + agent_overrides = yaml.safe_load(ctx.agent_config.read_text()) or {} + (stage_dir / "agent_config.yaml").write_text(ctx.agent_config.read_text()) + + meta = {"calib_dir": str(cd), "calib_npy_dir": str(calib_npy_dir(cd)), "calib_source": source, + "calib_n": manifest.get("count"), "calib_manifest_sha": manifest.get("sha256")} + if job["backend"] == "ptq": + cfg = ptq.compile_config(job, onnx_path(job), calib_npy_dir(cd), work_dir(ctx), agent_overrides) + yp = compile_yaml_path(ctx) + yp.parent.mkdir(parents=True, exist_ok=True) + ptq.write_compile_config(cfg, yp) + meta["compile_yaml"] = str(yp) + (stage_dir / "stdout.log").write_text(json.dumps(meta, indent=2, ensure_ascii=False) + "\n") + (stage_dir / "stderr.log").touch() + return StageOutcome("OK", 0, None, 0.0, meta, {"calib": meta}) + + +def _make_synthetic(out: Path, input_spec: str, n: int) -> None: + """占据类第三级回退:输入范围内的随机张量。G2 结果只作参考(pass-weak)。""" + import hashlib + try: + import numpy as np + shape = tuple(int(x) for x in input_spec.split("x")) + rng = np.random.default_rng(20260914) + h = hashlib.sha256() + (out / "npy_f32").mkdir(exist_ok=True) + for i in range(n): + arr = rng.uniform(-1, 1, size=shape).astype("float32") + p = out / "npy_f32" / f"{i:05d}.npy" + np.save(p, arr) + h.update(p.read_bytes()) + (out / "manifest.json").write_text(json.dumps({"count": n, "sha256": h.hexdigest(), "source": "synthetic", + "input_spec": input_spec}, indent=2)) + except ImportError: + (out / "manifest.json").write_text(json.dumps({"count": 0, "sha256": None, "source": "synthetic", + "error": "numpy unavailable"}, indent=2)) diff --git a/harness/stages/s05_compile.py b/harness/stages/s05_compile.py new file mode 100644 index 0000000..03fac9d --- /dev/null +++ b/harness/stages/s05_compile.py @@ -0,0 +1,27 @@ +"""05-compile:hb_compile(PTQ)/ plugin calib + hbdk4 compile(QAT)。CPU 密集,只有看门狗,没有超时。""" +from __future__ import annotations + +from pathlib import Path + +from ..backends import ptq, qat +from .base import Stage, StageContext +from .s01_fetch import weights_file +from .s04_calibrate import compile_yaml_path, work_dir, calib_dir + + +class Compile(Stage): + name = "05-compile" + level = "target" + pool = "cpu" + + def command(self, ctx: StageContext): + wd = work_dir(ctx) + wd.mkdir(parents=True, exist_ok=True) + if ctx.job["backend"] == "ptq": + return ptq.compile_cmd(compile_yaml_path(ctx)) + return qat.calib_compile_cmd(ctx.job, weights_file(ctx.job["model"]), calib_dir(ctx.job), wd, ctx.agent_config) + + def env(self, ctx: StageContext) -> dict: + env = super().env(ctx) + env["OMP_NUM_THREADS"] = "8" + return env diff --git a/harness/stages/s06_hbm.py b/harness/stages/s06_hbm.py new file mode 100644 index 0000000..22e5994 --- /dev/null +++ b/harness/stages/s06_hbm.py @@ -0,0 +1,212 @@ +"""06-hbm:收产物、hb_model_info 读 march / 输入规格 / 静态性能估算、hb_verifier 一致性、编译日志扫 ERROR → G1 测量值。 + +产物命名由 job_id 决定:artifacts/.hbm(.bc 也留一份)。march 不一致直接 fail——agent 最容易作弊的口子。 +实测(hb_model_info 3.5.3):I/O 表与 march 打在 stderr(logging),静态 perf 估算(FPS / latency / DDR)打在 stdout。 +实测(hb_verifier 3.5.3):-m ptq_model.onnx,quantized_model.bc -i ,逐层余弦表,最后一行是模型输出。 +""" +from __future__ import annotations + +import json +import re +import shutil +from pathlib import Path + +from .. import watchdog +from ..backends import get as backend +from ..registry import boards +from .base import Stage, StageContext, StageOutcome +from .s04_calibrate import work_dir, calib_npy_dir + +ANSI = re.compile(r"\x1b\[[0-9;]*m") + + +def hbm_path(ctx: StageContext) -> Path: + return ctx.artifacts / f"{ctx.job['job_id']}.hbm" + + +def bc_path(ctx: StageContext) -> Path: + return ctx.artifacts / f"{ctx.job['job_id']}.bc" + + +def _read(p: Path) -> str: + return ANSI.sub("", p.read_text(errors="replace")) if p.exists() else "" + + +class Hbm(Stage): + name = "06-hbm" + level = "target" + pool = "cpu" + + def native(self, ctx: StageContext, stage_dir: Path) -> StageOutcome: + job = ctx.job + wd = work_dir(ctx) + cands = sorted(wd.rglob("*.hbm"), key=lambda p: p.stat().st_mtime, reverse=True) if wd.exists() else [] + m: dict = {"hbm_exists": bool(cands)} + if not cands: + (stage_dir / "stderr.log").write_text(f"no .hbm under {wd}\n") + return StageOutcome("FAIL", 2, "unknown", 0.0, {"reason": "no_hbm"}, {"G1": m}) + ctx.artifacts.mkdir(parents=True, exist_ok=True) + dst = hbm_path(ctx) + shutil.copy2(cands[0], dst) + bc = next(iter(sorted(wd.rglob("*quantized_model.bc"))), None) or next(iter(sorted(wd.rglob("*.bc"))), None) + if bc: + shutil.copy2(bc, ctx.artifacts / f"{job['job_id']}.bc") + be = backend(job["backend"]) + wdp = ctx.watchdog_params() + prefix = job["job_id"] + + # ---- hb_model_info:stdout=静态 perf,stderr=I/O 表 + march + r = watchdog.run(be.model_info_cmd(dst), cwd=stage_dir, stdout=stage_dir / "model_info.log", + stderr=stage_dir / "model_info.err.log", env=self.env(ctx), stall_s=wdp["stall_s"], poll_s=5) + info = _read(stage_dir / "model_info.log") + "\n" + _read(stage_dir / "model_info.err.log") + m["model_info_ok"] = r.state == "OK" + m["march_read"] = _find_march(info) + m["march_ok"] = (m["march_read"] == job["march"]) if m["march_read"] else None + io = _parse_io(info) + m["io"] = io + m["io_ok"] = _check_io(io, job["model_spec"]["input_spec"]) if io else None + m["static_perf"] = _parse_static_perf(info) + + # ---- hb_verifier(PTQ:ptq_model.onnx vs quantized_model.bc,一张校准外样本) + if job["backend"] == "ptq" and bc: + ptq_onnx = next(iter(sorted(wd.rglob("*_ptq_model.onnx"))), None) + sample = next(iter(sorted(calib_npy_dir(Path(job["data"]["calib"]["path"])).glob("*.npy"))), None) + if ptq_onnx and sample: + rv = watchdog.run(be.verifier_cmd(ptq_onnx, bc, sample), cwd=stage_dir, + stdout=stage_dir / "verifier.log", stderr=stage_dir / "verifier.err.log", + env=self.env(ctx), stall_s=wdp["stall_s"], poll_s=10) + vtext = _read(stage_dir / "verifier.log") + "\n" + _read(stage_dir / "verifier.err.log") + cos = _verifier_output_cosine(vtext) + thr = job["gates"]["G2"].get("threshold") or 0.99 + m["verifier_cosine"] = cos + m["verifier_ok"] = (rv.state == "OK") and (cos is None or cos >= thr) + m["verifier_rc"] = rv.exit_code + else: + m["verifier_ok"] = None + m["verifier_note"] = "ptq_model.onnx or calib sample missing" + else: + m["verifier_ok"] = True # QAT 的一致性在 08-eval 的 G2 里判 + + # ---- hbm ≡ bc 抽查(08-eval 全量跑的是 .bc;这里用 2 张确认 hbm 的 run_sim 与 bc 逐位一致) + if bc and job["backend"] == "ptq": + m["hbm_bc_cos_min"] = _hbm_bc_spotcheck(ctx, stage_dir, dst, ctx.artifacts / f"{job['job_id']}.bc", be, wdp) + if m["hbm_bc_cos_min"] is not None and m["hbm_bc_cos_min"] < 0.999: + m["verifier_ok"] = False + m["verifier_note"] = f"hbm vs bc cosine {m['hbm_bc_cos_min']:.6f} < 0.999" + + # ---- 编译日志扫 ERROR(只看 hb_compile 自己的日志级别,不看 INFO 里的字样) + clog = "" + for f in sorted((ctx.stage_root / "05-compile").glob("a*/stderr.log")): + clog += _read(f) + clog += _read(wd / "hb_compile.log") + # 上一次尝试的 ERROR 不算:只看最后一次 "Start hb_compile" 之后 + last = clog.rfind("Start hb_compile") + tail = clog[last:] if last >= 0 else clog + m["log_has_error"] = bool(re.search(r"\bERROR\b", tail)) + m["hmct_output_cosine"] = _hmct_output_cosine(tail) # hmct 校准阶段自己报的模型输出余弦(校准/量化),PTQ 崩了这里先看到 + + (stage_dir / "stdout.log").write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n") + (stage_dir / "stderr.log").touch() + # work 目录可能在本地盘(config paths.work_root),把它记下来,事后还能找到中间产物 + return StageOutcome("OK", 0, None, 0.0, + {"hbm": str(dst), "size": dst.stat().st_size, "work_dir": str(wd)}, {"G1": m}) + + +def _hbm_bc_spotcheck(ctx: StageContext, stage_dir: Path, hbm: Path, bc: Path, be, wdp: dict, n: int = 2) -> float | None: + """同 n 张校准样本分别过 hbm(run_sim) 与 bc(HBIR run),返回逐输出余弦的最小值。任何一步失败返回 None(记录,不判)。""" + try: + import numpy as np + npys = sorted(calib_npy_dir(Path(ctx.job["data"]["calib"]["path"])).glob("*.npy"))[:n] + if not npys: + return None + fin = stage_dir / "spot_in.npz" + np.savez(fin, **{f"s{i:04d}": np.load(p) for i, p in enumerate(npys)}) + outs = {} + for tag, model in (("hbm", hbm), ("bc", bc)): + fo = stage_dir / f"spot_{tag}.npz" + r = watchdog.run([str(be.python()), "-m", "harness.tools.infer_hbm", "--model", str(model), "--inputs", str(fin), "--out", str(fo)], + cwd=stage_dir, stdout=stage_dir / f"spot_{tag}.log", stderr=stage_dir / f"spot_{tag}.err.log", + env=self_env(ctx), stall_s=wdp["stall_s"], poll_s=5) + if r.state != "OK": + return None + outs[tag] = np.load(fo) + cos_min = 1.0 + for k in outs["hbm"].files: + a = outs["hbm"][k].astype(np.float64).ravel(); b = outs["bc"][k].astype(np.float64).ravel() + na, nb = np.linalg.norm(a), np.linalg.norm(b) + cos_min = min(cos_min, float(a @ b / (na * nb)) if na and nb else 0.0) + fin.unlink(missing_ok=True) + return cos_min + except Exception: + return None + + +def self_env(ctx: StageContext) -> dict: + return Hbm().env(ctx) + + +def _find_march(info: str) -> str | None: + low = info.lower() + strings = boards().get("march_strings", {}) + mm = re.search(r"march\s*[:=]?\s*\"?([a-z0-9-]+)", low) + if mm and mm.group(1).startswith("nash"): + return mm.group(1) + for march, names in strings.items(): + if any(n.lower() in low for n in names): + return march + return None + + +def _parse_io(info: str) -> dict: + """解析 'NAME TYPE SHAPE DATA_TYPE' 表:input input [1, 3, 224, 224] FLOAT32""" + io = {"inputs": [], "outputs": []} + for line in info.splitlines(): + mm = re.search(r"(\S+)\s+(input|output)\s+\[([0-9,\s]+)\]\s+([A-Z0-9_]+)", line) + if mm: + io["inputs" if mm.group(2) == "input" else "outputs"].append( + {"name": mm.group(1), "shape": [int(x) for x in mm.group(3).split(",")], "dtype": mm.group(4)}) + return io if io["inputs"] or io["outputs"] else {} + + +def _check_io(io: dict, input_spec: str) -> bool | None: + dims = [int(x) for x in input_spec.split("x")] + ins = io.get("inputs") or [] + if not ins: + return None + return any(i["shape"] == dims for i in ins) + + +def _parse_static_perf(info: str) -> dict: + """hb_model_info stdout:FPS=2318.51, latency = 431.3 us, DDR = 12709504 bytes(编译器静态估算,不是板端实测)""" + d = {} + mm = re.search(r"FPS\s*=\s*([\d.]+)", info) + if mm: d["fps_est"] = float(mm.group(1)) + mm = re.search(r"latency\s*=\s*([\d.]+)\s*(us|ms)", info) + if mm: d["latency_ms_est"] = float(mm.group(1)) / (1000.0 if mm.group(2) == "us" else 1.0) + mm = re.search(r"DDR\s*=\s*(\d+)\s*bytes", info) + if mm: d["ddr_bytes_est"] = int(mm.group(1)) + return d + + +def _hmct_output_cosine(text: str) -> dict | None: + """hb_compile 日志末尾的表:| TensorName | Calibrated Cosine | Quantized Cosine |""" + i = text.rfind("Calibrated Cosine | Quantized Cosine") + if i < 0: + return None + rows = re.findall(r"\|\s*(\S+)\s*\|\s*([0-9.\-]+)\s*\|\s*([0-9.\-]+)\s*\|", text[i:]) + if not rows: + return None + return {name: {"calibrated": float(c), "quantized": float(q)} for name, c, q in rows if _isnum(c) and _isnum(q)} + + +def _isnum(s: str) -> bool: + try: + float(s); return True + except ValueError: + return False + + +def _verifier_output_cosine(text: str) -> float | None: + """逐层余弦表最后一行(模型输出)的余弦;找不到返回 None(只靠 rc 判)。""" + rows = re.findall(r"\|\s*\S+\s*\|\s*\S+\s*\|\s*([0-9.]+)\s*\|", text) + return float(rows[-1]) if rows else None diff --git a/harness/stages/s07_board.py b/harness/stages/s07_board.py new file mode 100644 index 0000000..ce75f51 --- /dev/null +++ b/harness/stages/s07_board.py @@ -0,0 +1,54 @@ +"""07-board:推 hbm 上板,hrt_model_exec perf → G4(mean 与 p95 分开存,永不互推)。 + +唯一带板卡凭证的阶段;凭证只进这个子进程的 env,agent 进程永远拿不到。 +板卡 worker=1,锁在 scheduler 里。board_offline 归瞬态,重试后仍失败 → G4 unverified。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +from .. import paths +from .base import Stage, StageContext, StageOutcome +from .s06_hbm import hbm_path + + +class Board(Stage): + name = "07-board" + level = "target" + pool = "board" + + def command(self, ctx: StageContext): + if not ctx.job.get("board"): + return None # → native:没有这个 march 的板 → G4 unverified,不阻塞 + perf = ctx.cfg["perf"] + board = ctx.job.get("board") or {} + return [ctx.cfg["paths"].get("harness_python", "python3"), "-m", "harness.tools.board_perf", + "--hbm", str(hbm_path(ctx)), "--job-id", ctx.job["job_id"], + "--warmup", str(perf["warmup"]), "--runs", str(perf["runs"]), + "--runtime-dir", board.get("runtime_dir", "/userdata/bench"), + "--out", str(ctx.stage_root / self.name / "perf.json")] + + def native(self, ctx: StageContext, stage_dir: Path) -> StageOutcome: + msg = f"no board registered for march {ctx.job['march']} (registry/boards.yaml) → G4 unverified" + (stage_dir / "stdout.log").write_text(msg + "\n"); (stage_dir / "stderr.log").touch() + return StageOutcome("SKIPPED", 0, None, 0.0, {"reason": "no_board_for_march"}, {"G4": None}) + + def env(self, ctx: StageContext) -> dict: + env = super().env(ctx) + for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER", "BOARD_SSH_KEY"): + v = paths.secret(k) + if v: + env[k] = v + return env + + def measure(self, ctx, stage_dir, outcome) -> dict: + p = ctx.stage_root / self.name / "perf.json" + if not p.exists(): + return {"G4": None} + d = json.loads(p.read_text()) + return {"G4": {"latency_ms_mean": d.get("latency_ms_mean"), "latency_ms_p95": d.get("latency_ms_p95"), + "latency_ms_p50": d.get("latency_ms_p50"), "fps_single_thread": d.get("fps_single_thread"), + "bpu_util": d.get("bpu_util"), "ddr_bw_mbps": d.get("ddr_bw_mbps"), + "board_type": d.get("board_type"), "ucp": d.get("ucp"), + "contended": d.get("contended", False)}} # 有人抢板 → 数字可疑,reward 的 lat_rel 应忽略 diff --git a/harness/stages/s08_eval.py b/harness/stages/s08_eval.py new file mode 100644 index 0000000..7642df9 --- /dev/null +++ b/harness/stages/s08_eval.py @@ -0,0 +1,42 @@ +"""08-eval:量化模型在 T1 上跑同一评测器 → G2(余弦,校准外样本)+ G3(任务指标 − FP32 基线)。 + +评测器在 harness/eval/,agent 不可写,是 verdict 的唯一依据。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +from .. import paths +from .base import Stage, StageContext, StageOutcome +from .s02_baseline import baseline_dir +from .s06_hbm import hbm_path, bc_path +from .s03_export import onnx_path + + +class Eval(Stage): + name = "08-eval" + level = "target" + pool = "gpu" + + def command(self, ctx: StageContext): + job = ctx.job + out = ctx.stage_root / self.name / "eval.json" + return [str(paths.venv_python("bench")), "-m", "harness.eval.run", + "--runtime-python", str(paths.venv_python(job["backend"])), # hbm 用产出它的工具链 venv 跑 x86 模拟 + "--mode", "quant", "--task", job["task"], "--model", job["model"], + "--hbm", str(bc_path(ctx) if bc_path(ctx).exists() else hbm_path(ctx)), "--onnx", str(onnx_path(job)), + "--backend", job["backend"], "--input-spec", job["model_spec"]["input_spec"], "--resize", str(job["model_spec"].get("resize") or 0), + "--t1", job["data"]["t1"]["path"], "--t1-sha", str(job["data"]["t1"]["manifest_sha"]), + "--baseline", str(baseline_dir(job)), + "--g2-samples", str(job["gates"]["G2"]["n_samples"]), + "--calib-source", (ctx.extra.get("calib_source") or "t0"), + "--evaluator-version", job["evaluator_version"], + "--out", str(out)] + + def measure(self, ctx, stage_dir, outcome) -> dict: + p = ctx.stage_root / self.name / "eval.json" + if not p.exists(): + return {} + d = json.loads(p.read_text()) + return {"G2": d.get("G2"), "G3": d.get("G3")} diff --git a/harness/tools/__init__.py b/harness/tools/__init__.py new file mode 100644 index 0000000..edf4819 --- /dev/null +++ b/harness/tools/__init__.py @@ -0,0 +1 @@ +"""在工具链 venv 里执行的脚本(需要 torch / hbdk / hf 等)。harness 主进程只依赖 pyyaml,靠子进程调这些。""" diff --git a/harness/tools/board_bench.py b/harness/tools/board_bench.py new file mode 100644 index 0000000..8755954 --- /dev/null +++ b/harness/tools/board_bench.py @@ -0,0 +1,120 @@ +"""在板子上跑:加载 hbm,单线程连续推理,输出时延统计 + BPU 占用采样(JSON)。 + + python3 board_bench.py --hbm x.hbm --warmup 20 --runs 200 + +实测 RDK S100P 沙箱(UCP 3.13.6,Ubuntu 22.04 容器):没有 hrt_model_exec CLI,只有 python 包 hbm_runtime: + m = hbm_runtime.HB_HBMRuntime(path); name = m.model_names[0] + m.input_names[name] / m.output_names[name] / m.input_shapes[name][input] + outs = m.run({name: {input: np.ndarray}})[name] +只依赖 numpy + hbm_runtime,板上原生有。stderr 上 UCP 的日志噪声不管,结果只看 stdout 最后一行 JSON。 +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import threading +import time + + +def read_ratio() -> float | None: + for p in ("/sys/class/bpu/bpu_core0/ratio", "/sys/class/bpu/bpu/ratio"): + try: + return float(open(p).read().strip()) + except (OSError, ValueError): + continue + return None + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--hbm", required=True) + ap.add_argument("--warmup", type=int, default=20) + ap.add_argument("--runs", type=int, default=200) + ap.add_argument("--input-npy", help="可选:真实输入 .npy(默认随机)") + a = ap.parse_args() + import numpy as np + import hbm_runtime + + t_load = time.time() + m = hbm_runtime.HB_HBMRuntime(a.hbm) + name = m.model_names[0] + in_names = list(m.input_names[name]) + out_names = list(m.output_names[name]) + shapes = m.input_shapes[name] + types = getattr(m, "input_types", {}).get(name, {}) if hasattr(m, "input_types") else {} + load_s = time.time() - t_load + + def make_input(n): + shp = tuple(int(x) for x in shapes[n]) + dt = str(types.get(n, "")).lower() + if a.input_npy: + arr = np.load(a.input_npy) + return np.ascontiguousarray(arr.reshape(shp)) if arr.size == int(np.prod(shp)) else arr + if "int8" in dt or "uint8" in dt: + return np.random.randint(0, 255, size=shp, dtype=np.uint8 if "uint" in dt else np.int8) + return np.random.uniform(-1, 1, size=shp).astype(np.float32) + + feed = {name: {n: make_input(n) for n in in_names}} + + ratios: list[float] = [] + stop = threading.Event() + + def sampler(): + while not stop.is_set(): + r = read_ratio() + if r is not None: + ratios.append(r) + time.sleep(0.05) + + for _ in range(a.warmup): + m.run(feed) + th = threading.Thread(target=sampler, daemon=True); th.start() + lat = [] + t0 = time.time() + for _ in range(a.runs): + t = time.perf_counter() + m.run(feed) + lat.append((time.perf_counter() - t) * 1000.0) + wall = time.time() - t0 + stop.set(); th.join(timeout=1) + lat_np = np.asarray(lat) + out = { + "model": name, "inputs": {n: list(map(int, shapes[n])) for n in in_names}, "outputs": out_names, + "input_types": {n: str(types.get(n, "?")) for n in in_names}, + "latency_ms_mean": float(lat_np.mean()), "latency_ms_p50": float(np.percentile(lat_np, 50)), + "latency_ms_p95": float(np.percentile(lat_np, 95)), "latency_ms_min": float(lat_np.min()), + "fps_single_thread": a.runs / wall, "warmup": a.warmup, "runs": a.runs, "load_s": load_s, + "bpu_util": (float(np.mean(ratios)) / 100.0) if ratios else None, "bpu_ratio_samples": len(ratios), + "ucp": _ucp_version(), "board_type": _board_type(), "hostname": os.uname().nodename, + } + print(json.dumps(out)) + return 0 + + +def _ucp_version() -> str | None: + for p in ("/etc/ucp_version",): + try: + return open(p).read().strip() + except OSError: + pass + try: + import subprocess + o = subprocess.run(["python3", "-c", "import hbm_runtime"], capture_output=True, text=True, timeout=20) + import re + mm = re.search(r"UCP version = ([\d.]+)", o.stdout + o.stderr) + return mm.group(1) if mm else None + except Exception: + return None + + +def _board_type() -> str | None: + try: + return open("/proc/device-tree/model", "rb").read().decode(errors="replace").strip("\x00 \n") + except OSError: + return None + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/tools/board_perf.py b/harness/tools/board_perf.py new file mode 100644 index 0000000..da05bfa --- /dev/null +++ b/harness/tools/board_perf.py @@ -0,0 +1,90 @@ +"""07-board 子进程:scp hbm + board_bench.py 上板 → 板上 python3 跑 → 解析 JSON → perf.json。 + +凭证只从本进程 env 取(BOARD_HOST/PORT/USER/SSH_KEY),密钥登录。 +实测 RDK S100P 沙箱:无 hrt_model_exec,用 hbm_runtime python 包计时(见 board_bench.py)。 +板是共享的:开测前探测板上有没有别人的推理进程 / BPU ratio,最多等 --busy-wait-s,仍忙照测但标 contended。 +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def ssh_base() -> tuple[list[str], str]: + host, port, user, key = (os.environ.get(k) for k in ("BOARD_HOST", "BOARD_PORT", "BOARD_USER", "BOARD_SSH_KEY")) + user = user or "root" + if not (host and key): + print("BOARD_HOST/BOARD_SSH_KEY missing (see bench/.env.local)", file=sys.stderr) + raise SystemExit(4) + key = os.path.expanduser(key) + return ["-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=20", + "-i", key, "-p", port or "22"], f"{user}@{host}" + + +def run(cmd: list[str], timeout: int = 1800) -> str: + r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + if r.returncode != 0: + sys.stderr.write(r.stderr[-4000:]) + raise SystemExit(r.returncode) # ssh 连不上 → errors.classify 认出 board_offline + return r.stdout + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--hbm", required=True) + ap.add_argument("--job-id", required=True) + ap.add_argument("--warmup", type=int, default=20) + ap.add_argument("--runs", type=int, default=200) + ap.add_argument("--runtime-dir", default="/userdata/bench") + ap.add_argument("--busy-wait-s", type=int, default=600) + ap.add_argument("--out", required=True) + a = ap.parse_args() + opts, target = ssh_base() + scp_opts = [o if o != "-p" else "-P" for o in opts] + rdir = a.runtime_dir + remote_hbm = f"{rdir}/{a.job_id}.hbm" + run(["ssh", *opts, target, f"mkdir -p {rdir}"]) + + # 共享板:探测别人的推理进程 / BPU 占用 + contended, waited = False, 0 + while True: + probe = run(["ssh", *opts, target, + "pgrep -f 'board_bench|hrt_model_exec|hbm_runtime|dnn' | grep -v $$ | wc -l; " + "cat /sys/class/bpu/bpu_core0/ratio 2>/dev/null || cat /sys/class/bpu/bpu/ratio 2>/dev/null || echo 0"]).split() + others = int(probe[0]) if probe else 0 + ratio = float(probe[1]) if len(probe) > 1 else 0.0 + busy = others > 0 or ratio > 5 + if not busy or waited >= a.busy_wait_s: + contended = busy + break + time.sleep(20); waited += 20 + if contended: + print(f"WARN board busy after {waited}s (procs={others}, ratio={ratio}); measuring anyway, flag contended", file=sys.stderr) + + run(["scp", *scp_opts, "-q", a.hbm, f"{target}:{remote_hbm}"], timeout=1800) + run(["scp", *scp_opts, "-q", str(HERE / "board_bench.py"), f"{target}:{rdir}/board_bench.py"]) + out_txt = run(["ssh", *opts, target, + f"cd {rdir} && python3 board_bench.py --hbm {remote_hbm} --warmup {a.warmup} --runs {a.runs} 2>/dev/null | tail -1"], + timeout=3600) + run(["ssh", *opts, target, f"rm -f {remote_hbm}"]) + try: + d = json.loads(out_txt.strip().splitlines()[-1]) + except Exception: + sys.stderr.write("board_bench produced no JSON:\n" + out_txt[-2000:]) + return 5 + d.update({"contended": contended, "busy_waited_s": waited, "measured_at": time.time()}) + Path(a.out).parent.mkdir(parents=True, exist_ok=True) + Path(a.out).write_text(json.dumps(d, indent=2, ensure_ascii=False)) + print(json.dumps({k: d.get(k) for k in ("latency_ms_mean", "latency_ms_p95", "bpu_util", "board_type", "ucp", "contended")})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/tools/export_onnx.py b/harness/tools/export_onnx.py new file mode 100644 index 0000000..7c77c09 --- /dev/null +++ b/harness/tools/export_onnx.py @@ -0,0 +1,81 @@ +"""03-export 子进程:权重 → ONNX(静态 shape,opset 由 OE 3.7 要求决定,默认 11)。 + +按 source 分派:torchvision / ultralytics / timm / hf / custom(custom 走 adapters/.py)。 +export_fail 是 model-level,一次影响该模型所有 job。 +""" +from __future__ import annotations + +import argparse +import importlib +import sys +from pathlib import Path + +OPSET = 11 # 实测 OE 3.7 接受 opset 11 +EXPORTER_VERSION = "2" # 导出口径版本:变了缓存作废。v2 = ultralytics 走官方 exporter;torchvision 走 TorchScript exporter + arch 名映射 + + +def load_model(model: str, source: str, weights: Path, input_spec: str): + import torch + if source == "torchvision": + import torchvision.models as tvm + arch = model.replace(".", "_") # model_id 里 '.' 是 builder 名的 '_'(job_id 段不允许下划线以外的分隔) + m = tvm.get_model(arch, weights=None) + m.load_state_dict(torch.load(weights, map_location="cpu")) + return m.eval() + if source in ("ultralytics", "github") and weights.suffix == ".pt": + from ultralytics import YOLO + y = YOLO(str(weights)) + # ultralytics 自带 export;这里只取 nn.Module 走统一路径,保证 03-export 口径一致 + return y.model.eval().float() + if source == "timm": + import timm + m = timm.create_model(model, pretrained=False) + m.load_state_dict(torch.load(weights, map_location="cpu")) + return m.eval() + # custom:bench/harness/adapters/.py 里 def build(weights: Path, input_spec: str) -> nn.Module + mod = importlib.import_module(f"harness.adapters.{model.replace('.', '_').replace('-', '_')}") + return mod.build(weights, input_spec).eval() + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True) + ap.add_argument("--source", required=True) + ap.add_argument("--weights", required=True) + ap.add_argument("--input-spec", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--opset", type=int, default=OPSET) + a = ap.parse_args() + import torch + shape = tuple(int(x) for x in a.input_spec.split("x")) + out = Path(a.out) + out.parent.mkdir(parents=True, exist_ok=True) + if a.source == "ultralytics" or (Path(a.weights).suffix == ".pt" and a.source in ("github", "ultralytics")): + # 直接 torch.onnx.export(nn.Module) 会把 Detect 头里的 make_anchors/Cast 留在图里 → hb_compile 'hbir.cast_type Unsupported type' + # 官方 exporter 把 anchors / strides 常量化,输出 [1, 4+nc, N],与 eval/det.py 的解码一致 + from ultralytics import YOLO + y = YOLO(str(a.weights)) + exported = y.export(format="onnx", opset=a.opset, imgsz=shape[-1], simplify=True, dynamic=False, half=False) + import shutil + shutil.move(str(exported), str(out)) + import onnx + onnx.checker.check_model(str(out)) + print(out) + return 0 + m = load_model(a.model, a.source, Path(a.weights), a.input_spec) + x = torch.randn(*shape) + kw = dict(opset_version=a.opset, input_names=["input"], do_constant_folding=True, dynamic_axes=None) + with torch.no_grad(): + try: + # torch ≥ 2.9 默认 dynamo 导出器(要求 opset ≥ 18,图形态也不同);OE 3.7 走经典 TorchScript 导出器 + torch.onnx.export(m, x, str(out), dynamo=False, **kw) + except TypeError: + torch.onnx.export(m, x, str(out), **kw) + import onnx + onnx.checker.check_model(str(out)) + print(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/tools/fetch_weights.py b/harness/tools/fetch_weights.py new file mode 100644 index 0000000..f63702d --- /dev/null +++ b/harness/tools/fetch_weights.py @@ -0,0 +1,85 @@ +"""01-fetch 子进程:把 models.csv 的 weights URI 落成一个文件。 + +支持: + https://… 直接下载(带断点续传) + torchvision:/// 例 torchvision://resnet18/IMAGENET1K_V1 → 保存 state_dict .pth + hf:/// huggingface hub + file:///abs/path 本机已有 +""" +from __future__ import annotations + +import argparse +import shutil +import sys +import urllib.request +from pathlib import Path + + +def fetch_http(url: str, out: Path) -> Path: + tmp = out.with_suffix(out.suffix + ".part") + pos = tmp.stat().st_size if tmp.exists() else 0 + req = urllib.request.Request(url, headers={"Range": f"bytes={pos}-"} if pos else {}) + with urllib.request.urlopen(req, timeout=60) as r, open(tmp, "ab") as f: + shutil.copyfileobj(r, f, length=1 << 20) + tmp.rename(out) + return out + + +def fetch_torchvision(arch: str, enum: str, out: Path) -> Path: + import torch, torchvision.models as tvm + w = getattr(tvm.get_model_weights(arch), enum) # enum 可以是 DEFAULT + model = tvm.get_model(arch, weights=w) + torch.save(model.state_dict(), out) + return out + + +def fetch_hf(repo: str, filename: str, out: Path) -> Path: + from huggingface_hub import hf_hub_download + p = hf_hub_download(repo_id=repo, filename=filename) + shutil.copy2(p, out) + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True) + ap.add_argument("--uri", required=True) + ap.add_argument("--out-dir", required=True) + a = ap.parse_args() + out_dir = Path(a.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + uri = a.uri + if uri.startswith("ultralytics://"): + # 权重名自动下载(GitHub release assets);YOLO() 会把 .pt 放到 cwd + import os + wname = uri[len("ultralytics://"):] + os.chdir(out_dir) + from ultralytics import YOLO + y = YOLO(wname) + src = Path(getattr(y, "ckpt_path", None) or wname) + if not src.exists(): + src = next(out_dir.glob("*.pt")) + p = out_dir / src.name + if src.resolve() != p.resolve(): + shutil.copy2(src, p) + elif uri.startswith("torchvision://"): + arch, enum = uri[len("torchvision://"):].split("/", 1) + p = fetch_torchvision(arch, enum, out_dir / f"{a.model}.pth") + elif uri.startswith("hf://"): + repo, fn = uri[len("hf://"):].split("/", 2)[:2] if uri.count("/") == 3 else uri[len("hf://"):].rsplit("/", 1) + p = fetch_hf(repo, fn, out_dir / Path(fn).name) + elif uri.startswith("file://"): + src = Path(uri[len("file://"):]) + p = out_dir / src.name + shutil.copy2(src, p) + elif uri.startswith(("http://", "https://")): + p = fetch_http(uri, out_dir / Path(uri.split("?")[0]).name) + else: + print(f"unknown weights uri: {uri}", file=sys.stderr) + return 2 + print(p) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/tools/infer_hbm.py b/harness/tools/infer_hbm.py new file mode 100644 index 0000000..bb2d8a6 --- /dev/null +++ b/harness/tools/infer_hbm.py @@ -0,0 +1,65 @@ +"""x86 侧量化模型推理:inputs.npz → outputs.npz。在产出该模型的工具链 venv 里跑。 + +实测(2026-09-15,PTQ venv horizon_tc_ui 3.5.3,resnet18/nash-e): + .bc → HB_HBIRRuntime(bc).run(output_names, input_info) 0.04 s/张,与 hbm 的 run_sim **逐位一致**(cos=1.0, maxabs=0) + .hbm → HB_HBMRuntime(hbm).run_sim(output_names, input_info) ~20 s/张(慢,只用来抽查 hbm ≡ bc) + ptq_model.onnx → HB_ONNXRuntime(伪量化,cos≈0.9995,不逐位)——不用它出数 +所以 G3 全量走 .bc,06-hbm 用 2 张抽查 hbm ≡ bc。QAT 产物走 hbdk4(P4 核对)。 +都不可用就非零退出,让上游记 unverified 而不是假 pass。 +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def _runner(model: Path): + errs = [] + suf = model.suffix.lower() + try: + if suf == ".bc": + from horizon_tc_ui.hb_hbirruntime import HB_HBIRRuntime # type: ignore + rt = HB_HBIRRuntime(str(model)) + names, outs = list(rt.input_names), list(rt.output_names) + return lambda x: dict(zip(outs, rt.run(output_names=outs, input_info={names[0]: x}))) + if suf == ".hbm": + from horizon_tc_ui.hb_hbmruntime import HB_HBMRuntime # type: ignore + rt = HB_HBMRuntime(str(model)) + names, outs = list(rt.input_names), list(rt.output_names) + return lambda x: dict(zip(outs, rt.run_sim(output_names=outs, input_info={names[0]: x}))) + except Exception as e: # noqa: BLE001 + errs.append(f"horizon_tc_ui runtime: {type(e).__name__}: {e}") + try: + from hbdk4.compiler import load # type: ignore # QAT / hbdk4 4.12 路径,P4 核对 + m = load(str(model)) + fn = m.functions[0] + return lambda x: fn.feed([x]) + except Exception as e: # noqa: BLE001 + errs.append(f"hbdk4.load: {type(e).__name__}: {e}") + print("no runtime available for %s in this venv:\n %s" % (model, "\n ".join(errs)), file=sys.stderr) + raise SystemExit(3) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--model", "--hbm", dest="model", required=True, help=".bc(快,逐位等价)或 .hbm(慢)") + ap.add_argument("--inputs", required=True, help="npz: 每个 key 一个样本,shape 与 job 一致(含 batch 维)") + ap.add_argument("--out", required=True) + a = ap.parse_args() + import numpy as np + run = _runner(Path(a.model)) + ins = np.load(a.inputs) + outs = {} + for k in ins.files: + res = run(np.ascontiguousarray(ins[k], dtype=np.float32)) + items = list(res.items()) if isinstance(res, dict) else list(enumerate(res)) + for i, (_, v) in enumerate(items): + # 必须深拷贝:HB_HBIRRuntime.run 返回的数组引用内部缓冲区,下一次 run 会原地覆盖(实测踩坑:两张图输出变成同一份) + outs[f"{k}/out{i}"] = np.array(v, copy=True) + np.savez(a.out, **outs) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/tools/qat_calib_compile.py b/harness/tools/qat_calib_compile.py new file mode 100644 index 0000000..f107f6d --- /dev/null +++ b/harness/tools/qat_calib_compile.py @@ -0,0 +1,16 @@ +"""QAT 后端 05-compile 子进程(oellm2 venv):plugin calibration-only → hbir → hbdk4 compile → .hbm。 + +P4 之后实现。现在只占位:非零退出并写清楚原因,让 job 记 FAIL:unknown 而不是假 PASS。 +""" +from __future__ import annotations + +import sys + + +def main() -> int: + print("qat backend not implemented yet (P4). See bench/harness/tools/qat_calib_compile.py", file=sys.stderr) + return 64 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/harness/verify.py b/harness/verify.py new file mode 100644 index 0000000..31c220a --- /dev/null +++ b/harness/verify.py @@ -0,0 +1,152 @@ +"""L3:对一次 rollout 独立核验 → verdict.json。reward 的唯一来源。 + +输入:rollout/stages/*/meta.json(harness 自己跑的,agent 改不了)、claim.json(agent 自报,不可信)、 + oracle/result.json(参照系)、agent/events.jsonl(成本)。 +不读 workspace 里 agent 写的任何"结果"。 +""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +from . import db, gates +from .errors import same_class + + +def _latest_meta(stage_root: Path, name: str) -> dict | None: + p = stage_root / name / "meta.json" + return json.loads(p.read_text()) if p.exists() else None + + +def collect_measurements(rd: Path, stage_root: Path | None = None) -> tuple[dict, dict]: + """从 stages//meta.json 汇总测量值;返回 (measurements, stage_summary)。 + rollout 传 rollout 目录;oracle 传 stage_root=…/oracle/stages。06-hbm 的 G1 若 meta 里没有,回退读它的 stdout.log(就是那份 JSON)。""" + root = stage_root or (rd / "stages") + meas, summ = {}, {} + for name in ("04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"): + m = _latest_meta(root, name) + if not m: + continue + summ[name] = {"state": m.get("state"), "error_class": m.get("error_class"), "attempts": m.get("attempt", 0) + 1} + if m.get("state") in ("OK", "SKIPPED"): + for k, v in (m.get("measurements") or {}).items(): + meas[k] = v + if name == "06-hbm" and "G1" not in meas: + so = root / name / f"a{m.get('attempt', 0):02d}" / "stdout.log" + try: + meas["G1"] = json.loads(so.read_text()) + except Exception: + pass + if "06-hbm" not in summ or summ["06-hbm"]["state"] != "OK": + meas.setdefault("G1", {"hbm_exists": False}) + return meas, summ + + +def _cost(rd: Path) -> dict: + """成本只从 events.jsonl 数:turns = turn_end 事件数(别数 message_end,那会重复计三次)。 + + 实测(2026-09-15,dr-gateway / glm-5.3-flash):网关回的 usage 全是 0。 + 所以 tokens 可能整体不可用 → 记 tokens_reported=false,reward 的代价项据此退化为只按 turns 算, + 否则"没报 token"会被误读成"这条轨迹很省"。 + """ + ev = rd / "agent" / "events.jsonl" + turns = tokens = tool_calls = 0 + t0 = t1 = None + if ev.exists(): + with ev.open(errors="replace") as f: + for line in f: + try: + e = json.loads(line) + except json.JSONDecodeError: + continue + ts = e.get("timestamp") or e.get("ts") or (e.get("message") or {}).get("timestamp") + if isinstance(ts, (int, float)): + t0 = t0 or ts; t1 = ts + typ = e.get("type", "") + if typ == "turn_end": + turns += 1 + u = ((e.get("message") or {}).get("usage")) or e.get("usage") + if isinstance(u, dict): + tokens += int(u.get("input", 0) or u.get("input_tokens", 0) or 0) + tokens += int(u.get("output", 0) or u.get("output_tokens", 0) or 0) + elif typ == "tool_execution_start": + tool_calls += 1 + wall = None + if t0 and t1: + try: + wall = (float(t1) - float(t0)) / 1000.0 if float(t1) > 1e11 else float(t1) - float(t0) + except (TypeError, ValueError): + wall = None + return {"turns": turns, "tokens": tokens, "tokens_reported": tokens > 0, + "tool_calls": tool_calls, "wall_s": round(wall, 1) if wall else None} + + +def verify_rollout(rd: Path, write: bool = True) -> dict: + rd = rd.resolve() + jd = rd.parent.parent + job = json.loads((jd / "job.json").read_text()) + kk = int(rd.name) + oracle = json.loads((jd / "oracle" / "result.json").read_text()) if (jd / "oracle" / "result.json").exists() else None + claim = json.loads((rd / "claim.json").read_text()) if (rd / "claim.json").exists() else None + rmeta = json.loads((rd / "rollout.json").read_text()) if (rd / "rollout.json").exists() else {} + + meas, summ = collect_measurements(rd) + ev = gates.evaluate(meas, job["gates"]) + terminal = ev["terminal"] + + # agent 自报 BLOCKED / TIMEOUT 的处理 + blocked_correct = None + if rmeta.get("state") == "TIMEOUT": + terminal = "TIMEOUT" + elif claim and claim.get("terminal") == "BLOCKED": + terminal = "BLOCKED" + o_ec = (oracle or {}).get("error_class") + o_stage = (oracle or {}).get("stage_failed") + blocked_correct = bool(oracle) and oracle.get("terminal") == "FAIL" and \ + same_class(claim.get("error_class"), o_ec) and (claim.get("stage") in (None, o_stage)) + elif terminal == "UNVERIFIED" and (claim or {}).get("terminal") == "PASS": + terminal = "FAIL" # 说通过但没有可核验的 hbm/eval → 按 FAIL 记,claim≠verdict 另扣 + + claim_consistent = None + if claim: + claim_consistent = (claim.get("terminal") == terminal) if terminal in ("PASS", "FAIL", "BLOCKED") else None + + g3 = ev["gates"]["G3"] + o_g3 = ((oracle or {}).get("gates") or {}).get("G3") or {} + o_g4 = ((oracle or {}).get("gates") or {}).get("G4") or {} + review = bool(oracle) and oracle.get("terminal") == "FAIL" and terminal == "PASS" + + v_ec = gates.classify_gate_failure(ev) if terminal == "FAIL" else None + for st in summ.values(): + if st.get("state") not in ("OK", "SKIPPED") and st.get("error_class"): + v_ec = st["error_class"]; break + verdict = { + "schema": "bench.verdict/1", + "error_class": v_ec, + "job_id": job["job_id"], "run_id": job["run_id"], "rollout": kk, + "mode": rmeta.get("mode"), "policy": rmeta.get("policy"), + "gates": ev["gates"], "required": ev["required"], + "terminal": terminal, + "failed_gate": ev["failed_gate"], + "stages": summ, + "claim": claim, "claim_consistent": claim_consistent, + "blocked_correct": blocked_correct, + "oracle_ref": {"terminal": (oracle or {}).get("terminal"), "error_class": (oracle or {}).get("error_class"), + "stage_failed": (oracle or {}).get("stage_failed"), + "G3_delta": o_g3.get("delta"), "latency_ms_mean": o_g4.get("latency_ms_mean")} if oracle else None, + "review": review, + "cost": _cost(rd), + "verified_at": time.time(), + } + if write: + (rd / "verdict.json").write_text(json.dumps(verdict, indent=2, ensure_ascii=False, default=str)) + with db.connect() as con: + db.rollout_upsert(con, job["run_id"], job["job_id"], kk, rmeta.get("mode"), rmeta.get("policy")) + db.rollout_set(con, job["run_id"], job["job_id"], kk, state=terminal, + claim_terminal=(claim or {}).get("terminal"), verdict_terminal=terminal, + claim_consistent=None if claim_consistent is None else int(claim_consistent), + review=int(review), turns=verdict["cost"]["turns"], tokens=verdict["cost"]["tokens"], + wall_s=verdict["cost"]["wall_s"], verdict=json.dumps(verdict, default=str), + finished_at=time.time()) + return verdict diff --git a/harness/watch.py b/harness/watch.py new file mode 100644 index 0000000..e150cd7 --- /dev/null +++ b/harness/watch.py @@ -0,0 +1,117 @@ +"""bench watch — 终端实时看板:在跑什么、跑了多久、日志多久没动、谁可能卡死、还剩多少。 + + bench watch [--run r…] [--interval 30] [--once] + +"卡死"的判断口径与看门狗一致但更早预警: + 日志 idle > 5 min → ⚠ 提示 + 日志 idle > stall_s × 2/3 → 🔥 看门狗即将介入(stall_s 默认 900 s,且要求 CPU 也空闲才真杀) +看门狗真正杀掉的会以 STALLED 出现在"最近完成"里,error_class=stalled。 +""" +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + +from . import paths, db +from .scheduler import pool_usage, pool_size, estimate_stage + + +def _fmt(s: float | None) -> str: + if s is None: + return "-" + s = int(s) + return f"{s // 3600}h{(s % 3600) // 60:02d}m" if s >= 3600 else (f"{s // 60}m{s % 60:02d}s" if s >= 60 else f"{s}s") + + +def _log_age(run_id: str, job_id: str, rollout: int, stage: str, attempt: int) -> float | None: + jd = paths.RUNS / run_id / job_id + root = jd / "oracle" / "stages" if rollout < 0 else jd / "rollouts" / f"{rollout:02d}" / "stages" + sd = root / stage / f"a{attempt:02d}" + latest = 0.0 + for p in list(sd.glob("*.log")) + list((sd.parent / "work").glob("*.log")) + [sd]: + try: + latest = max(latest, p.stat().st_mtime) + except OSError: + pass + return (time.time() - latest) if latest else None + + +def snapshot(run_id: str) -> str: + cfg = paths.config() + stall_s = int(cfg["watchdog"]["stall_s"]) + lines = [] + with db.connect() as con: + run = db.get_run(con, run_id) + jobs = db.list_jobs(con, run_id, mode="oracle") + counts = {} + for j in jobs: + counts[j["state"]] = counts.get(j["state"], 0) + 1 + running = con.execute("SELECT * FROM stages WHERE run_id=? AND state='RUNNING' ORDER BY started_at", (run_id,)).fetchall() + done = con.execute("SELECT * FROM stages WHERE run_id=? AND state!='RUNNING' AND finished_at IS NOT NULL ORDER BY finished_at DESC LIMIT 8", (run_id,)).fetchall() + ec = db.error_class_counts(con, run_id) + # 完成的 job 平均耗时 → ETA + durs = [ (j["finished_at"] - j["started_at"]) for j in jobs if j.get("finished_at") and j.get("started_at") and j["state"] in ("PASS", "FAIL", "UNVERIFIED")] + from .registry import status_counts + try: + sc = status_counts() + except Exception: + sc = {} + now = time.strftime("%Y-%m-%d %H:%M:%S") + lines.append(f"bench watch · run {run_id} · {now} · {run.get('note','') if run else ''}") + lines.append(" jobs: " + " ".join(f"{k}={v}" for k, v in sorted(counts.items())) + f" (registry: {', '.join(f'{k}={v}' for k, v in sc.items())})") + lines.append(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items())) + pending = counts.get("PENDING", 0) + if durs and pending: + avg = sum(durs) / len(durs) + par = max(1, pool_usage().get("cpu", 1) or cfg["pools"]["cpu"]["workers"]) + lines.append(f" ETA: 剩 {pending} 个 job × 平均 {_fmt(avg)} / 并行 {par} ≈ {_fmt(pending * avg / par)}(已完成 {len(durs)} 个)") + lines.append("") + lines.append(f" RUNNING ({len(running)})") + lines.append(f" {'job_id':<48} {'stage':<13} {'att':<4} {'elapsed':>8} {'est':>7} {'log idle':>9} flag") + for s in running: + el = time.time() - (s["started_at"] or time.time()) + age = _log_age(run_id, s["job_id"], s["rollout"], s["stage"], s["attempt"]) + try: + with db.connect() as con: + jrow = db.get_job(con, run_id, s["job_id"]) + est = estimate_stage(con, jrow["model"], jrow["task"], s["stage"]) if jrow else None + except Exception: + est = None + flag = "" + if s["lane"] == "queued": + # rollout 提交的阶段还在等池槽位:不是卡死,别报看门狗 + lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}r{s['rollout']:02d} {_fmt(el):>8} {'-':>7} {'-':>9} ⏸ 排队等槽位") + continue + if age is not None and age > stall_s * 2 / 3: + flag = "🔥 看门狗即将介入" + elif age is not None and age > 300: + flag = "⚠ 日志 5 min 没动" + elif est and el > est * 3: + flag = "⏳ 超历史 p95×3(只告警)" + tag = f"r{s['rollout']:02d}" if s["rollout"] >= 0 else " " + lines.append(f" {s['job_id']:<48} {s['stage']:<13} a{s['attempt']:02d}{tag} {_fmt(el):>8} {_fmt(est):>7} {_fmt(age):>9} {flag}") + lines.append("") + lines.append(" 最近完成") + for s in done: + lines.append(f" {s['job_id']:<48} {s['stage']:<13} {s['state']:<8} {s['error_class'] or '':<16} {_fmt(s['duration_s']):>8}") + if ec: + lines.append("") + lines.append(" FAIL by error_class: " + " ".join(f"{r['error_class']}@{r['stage_failed']}={r['n']}" for r in ec)) + return "\n".join(lines) + + +def watch(run_id: str, interval: int = 30, once: bool = False) -> int: + while True: + text = snapshot(run_id) + if not once: + os.system("clear" if os.name != "nt" else "cls") + print(text) + if once: + return 0 + print(f"\n (每 {interval}s 刷新 · Ctrl-C 退出 · bench show 看细节)") + try: + time.sleep(interval) + except KeyboardInterrupt: + return 0 diff --git a/harness/watchdog.py b/harness/watchdog.py new file mode 100644 index 0000000..62cc50d --- /dev/null +++ b/harness/watchdog.py @@ -0,0 +1,139 @@ +"""无固定超时的子进程执行器。 + +STALLED 的唯一定义:stdout+stderr 文件 stall_s 秒不增长 且 进程树 CPU 使用率 < cpu_idle_pct%。 +真正在算的编译,跑 6 小时也不会被杀。hard_ceiling_s 是纯保险。 +""" +from __future__ import annotations + +import os +import signal +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path + +from .errors import STALLED_MARK + +CLK_TCK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100 + + +@dataclass +class RunResult: + state: str # OK | FAIL | STALLED + exit_code: int | None + duration_s: float + stdout: Path + stderr: Path + stalled_reason: str | None = None + io_blocked_s: float = 0.0 # 期间最长的连续 I/O 阻塞(D 状态)时长,进 meta.json 供诊断 + + +def _proc_tree_stat(root: int) -> tuple[int, int] | None: + """Linux: (root 及其全部后代的 utime+stime 总和 ticks, 处于 D 状态的进程数)。非 Linux 返回 None。 + + D = 不可中断睡眠 = 卡在 I/O。实测(2026-09-15)JuiceFS 写 1 MB 要 360 ms(本地盘 3 ms), + hbdk 编译大模型时会长时间待在 D 里:CPU ticks 不涨、日志不涨 —— 与"真挂死"在表象上完全一样。 + EfficientNet-B4/B5 因此被误判 STALLED 杀了两次(900s + 3600s)。所以 D 状态要单独看,不能算空闲。 + """ + proc = Path("/proc") + if not proc.exists(): + return None + children: dict[int, list[int]] = {} + ticks: dict[int, int] = {} + state: dict[int, str] = {} + for p in proc.iterdir(): + if not p.name.isdigit(): + continue + try: + stat = (p / "stat").read_text() + except OSError: + continue + # comm 可能含空格/括号:从最后一个 ')' 之后切 + rest = stat[stat.rfind(")") + 2:].split() + pid = int(p.name) + state[pid] = rest[0] + ppid = int(rest[1]) + ticks[pid] = int(rest[11]) + int(rest[12]) + children.setdefault(ppid, []).append(pid) + total, d_count, stack = 0, 0, [root] + while stack: + pid = stack.pop() + total += ticks.get(pid, 0) + if state.get(pid) == "D": + d_count += 1 + stack.extend(children.get(pid, [])) + return total, d_count + + +def _kill_tree(proc: subprocess.Popen) -> None: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + return + for _ in range(20): + if proc.poll() is not None: + return + time.sleep(0.5) + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def run(cmd: list[str] | str, *, cwd: Path, stdout: Path, stderr: Path, env: dict | None = None, + stall_s: int = 900, cpu_idle_pct: float = 5.0, poll_s: int = 30, + hard_ceiling_s: int = 86400, shell: bool = False, io_stall_s: int = 21600) -> RunResult: + stdout.parent.mkdir(parents=True, exist_ok=True) + t0 = time.time() + with open(stdout, "ab") as fo, open(stderr, "ab") as fe: + proc = subprocess.Popen(cmd, cwd=str(cwd), stdout=fo, stderr=fe, env=env, + shell=shell, start_new_session=True) + last_size = -1 + st = _proc_tree_stat(proc.pid) + last_ticks = st[0] if st else None + stalled_for = io_blocked_for = max_io_blocked = 0.0 + stalled_reason = None + while True: + rc = proc.poll() + if rc is not None: + break + time.sleep(poll_s) + now = time.time() + if now - t0 > hard_ceiling_s: + stalled_reason = f"hard_ceiling {hard_ceiling_s}s" + break + size = stdout.stat().st_size + stderr.stat().st_size + st = _proc_tree_stat(proc.pid) + ticks, d_count = (st if st else (None, 0)) + cpu_busy = True + if ticks is not None and last_ticks is not None: + cpu_pct = 100.0 * (ticks - last_ticks) / CLK_TCK / max(poll_s, 1) + cpu_busy = cpu_pct >= cpu_idle_pct + last_ticks = ticks + if size == last_size and not cpu_busy: + if d_count > 0: + # 卡在 I/O(JuiceFS 常见),不是挂死:计入 io_blocked,不累加 stall + io_blocked_for += poll_s + max_io_blocked = max(max_io_blocked, io_blocked_for) + stalled_for = 0.0 + else: + stalled_for += poll_s + io_blocked_for = 0.0 + else: + stalled_for = io_blocked_for = 0.0 + last_size = size + if stalled_for >= stall_s: + stalled_reason = f"log unchanged and cpu idle for {int(stalled_for)}s" + break + if io_blocked_for >= io_stall_s: + stalled_reason = f"blocked on I/O (D state) for {int(io_blocked_for)}s" + break + if stalled_reason: + _kill_tree(proc) + fe.write(f"\n{STALLED_MARK} {stalled_reason}\n".encode()) + fe.flush() + return RunResult("STALLED", proc.poll(), time.time() - t0, stdout, stderr, stalled_reason, + round(max_io_blocked, 1)) + rc = proc.returncode + return RunResult("OK" if rc == 0 else "FAIL", rc, time.time() - t0, stdout, stderr, + None, round(max_io_blocked, 1)) diff --git a/registry/boards.yaml b/registry/boards.yaml new file mode 100644 index 0000000..8f80157 --- /dev/null +++ b/registry/boards.yaml @@ -0,0 +1,26 @@ +# registry/boards.yaml — 板型 ↔ march 映射与板卡接入。凭证不在这里(.env.local)。 +version: 1 + +march_by_board: + s100: nash-e + s100p: nash-m + s600: nash-p + +boards: + rdk-sandbox: + host_env: BOARD_HOST # rdk-sandbox.d-robotics.cc + port_env: BOARD_PORT # 40010 + user_env: BOARD_USER + ssh_key_env: BOARD_SSH_KEY # 口令已泄露于明文对话,改密钥后轮换 + board_type: s100p # 2026-09-15 实测 /proc/device-tree/model = "D-Robotics RDK S100P V1P0" + march: nash-m + ucp_version: "3.13.6" # 实测(.env.oe-package 里写的 3.15.2 不对);Ubuntu 22.04 容器,6×A78AE,1 个 BPU core,21 GB + runtime_dir: /userdata/bench # 板上工作目录 + tools: + perf: python3 hbm_runtime # 沙箱没有 hrt_model_exec;用 hbm_runtime 包计时(harness/tools/board_bench.py) + +# 主机侧 march 校验用的字符串(G1 check_march 比对 hb_model_info 输出) +march_strings: + nash-e: ["nash-e", "Nash-E"] + nash-m: ["nash-m", "Nash-M"] + nash-p: ["nash-p", "Nash-P"] diff --git a/registry/budgets.yaml b/registry/budgets.yaml new file mode 100644 index 0000000..2ed5759 --- /dev/null +++ b/registry/budgets.yaml @@ -0,0 +1,28 @@ +# registry/budgets.yaml — 逐任务类耗时估算(只用于调度排序与 soft_alert,不用于杀进程) +# 首轮估算按参数量分档;jobs.sqlite 有实测后以实测 p95 为准(harness/scheduler.py::estimate) +version: 1 + +# 按参数量分档的首轮估算,单位秒 +by_params_m: + - { max: 10, compile: 600, board: 120, eval: 300 } + - { max: 50, compile: 1800, board: 180, eval: 600 } + - { max: 200, compile: 5400, board: 300, eval: 1200 } + - { max: null, compile: 14400, board: 600, eval: 2400 } + +# 任务类乘子(占据/光流/局部特征算子特殊,编译慢、失败率高) +task_multiplier: + cls: 1.0 + det: 1.2 + seg: 1.5 + iseg: 1.5 + pose: 1.2 + mde: 1.5 + stereo: 2.0 + flow: 2.5 + feat: 2.0 + occ: 3.0 + +# 看门狗参数可按任务覆盖;不写则用 config.yaml watchdog +watchdog_overrides: + occ: { stall_s: 1800 } + flow: { stall_s: 1800 } diff --git a/registry/datasets.yaml b/registry/datasets.yaml new file mode 100644 index 0000000..a81fa3b --- /dev/null +++ b/registry/datasets.yaml @@ -0,0 +1,153 @@ +# registry/datasets.yaml — 每个数据集的有序来源、版本、T0/T1 子集规格 +# 来源按顺序尝试:local → hf → oss → github → official。scripts/ingest_pool.py 读本文件。 +# sha256 字段在首次入池后由 ingest 写回 data/pool///manifest.json;这里的 expect_* 是入池校验用的硬约束。 +version: 1 +pool_root: data/pool +local_root: /mnt/data # 本机 JuiceFS 根 + +datasets: + imagenet1k: + version: val-2012 + tasks: [cls] + expect_count: 50000 + sources: + # ① 首选:HF parquet 14 片(已在本机,避开 tar 完整性风险) + - kind: local_parquet + path: chao01.wu/Data/HuggingFace/ImageNet_HF/imagenet/data + glob: "validation-*.parquet" + expect_shards: 14 + # ② 备用:tar(旁边有 .tmp 残片,需 tar -tzf | wc -l == 50000) + - kind: local_tar + path: chao01.wu/Data/Datasets/ImageNet/val_images.tar.gz + verify: "tar -tzf {path} | wc -l" + expect: 50000 + - kind: hf + repo: ILSVRC/imagenet-1k + split: validation + t1: { n: 2000, seed: 20260914, stratify: label } + calib: { n: 128, from: train, seed: 20260914, input_spec_default: 1x3x224x224, + train_parquet: chao01.wu/Data/HuggingFace/ImageNet_HF/imagenet/data, train_glob: "train-*.parquet" } + + coco2017: + version: val2017 + tasks: [det, iseg, pose] + expect_count: 5000 + sources: + - kind: local_dir + path: chao01.wu/Data/Datasets/COCO2017/val2017 + annotations: + - chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip + - kind: local_zip # 单文件顺序读,比逐文件 rsync 快 10 倍以上:--prefer local_zip + path: guannan.he/coco2017/val2017.zip + annotations: + - chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip + - kind: hf + repo: detection-datasets/coco + - kind: official + url: http://images.cocodataset.org/zips/val2017.zip # 26 KB/s,等于不通 + t1: { n: 1000, seed: 20260914, stratify: none } + calib: { n: 128, from: train2017, seed: 20260914, input_spec_default: 1x3x640x640, + train_dir: chao01.wu/Data/Datasets/COCO2017/train2017 } # 本机只有 200 张 train2017,够 128 + + cityscapes: + version: val-fine + tasks: [seg] + expect_count: 500 + sources: + - kind: local_zip + path: guannan.he/CityScapes/leftImg8bit_trainvaltest.zip + extract_only: "*/val/*" + extra: + - { path: guannan.he/CityScapes/gtFine_trainvaltest.zip, extract_only: "*/val/*" } + t1: { n: 500, seed: 20260914 } + calib: { n: 128, from: train, seed: 20260914 } + + ade20k: + version: val-2016 + tasks: [seg] + expect_count: 2000 + sources: + - kind: hf + repo: 1aurent/ADE20K + split: validation + t1: { n: 500, seed: 20260914 } + calib: { n: 128, from: train, seed: 20260914 } + + lvis: + version: v1-val + tasks: [iseg] + images_from: coco2017 # 图片复用 COCO,只需标注 + sources: + - kind: github + url: https://dl.fbaipublicfiles.com/lvis/lvis_v1_val.json.zip + - kind: hf + repo: lvis/lvis + t1: { n: 1000, seed: 20260914 } + + kitti2015: + version: data_scene_flow + tasks: [stereo, flow] + expect_count: 200 + sources: + - kind: local_dir + path: qi.xiong/Dataset/StereoDataset/KITTI/data_scene_flow + t1: { n: 200, seed: 20260914 } # 全量 + calib: { n: 128, from: training, seed: 20260914 } + note: stereo 与 flow 共用一份,manifest 分开 + + eth3d: + version: two-view + tasks: [stereo] + sources: + - kind: local_dir + path: qi.xiong/Dataset/StereoDataset/eth3d + t1: { n: all } + + nyuv2: + version: test-654 + tasks: [mde] + expect_count: 654 + sources: + - kind: hf + repo: sayakpaul/nyu_depth_v2 + split: validation + t1: { n: 654 } + + sintel: + version: train-final + tasks: [flow] + expect_count: 1041 + sources: + - kind: hf + repo: null # 待定:找到 HF 镜像后填 + - kind: official + url: http://files.is.tue.mpg.de/sintel/MPI-Sintel-complete.zip + t1: { n: all } + + hpatches: + version: sequences-release + tasks: [feat] + expect_count: 116 + sources: + - kind: github + url: https://huggingface.co/datasets/cvg/hpatches-sequences # 待核实 + - kind: official + url: http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz + t1: { n: all } + + nuscenes: + version: full-v1.0 + tasks: [occ] + sources: + - kind: local_dir + path: qi.xiong/Dataset/AutoDriveDataset/nuScenes/nuScenes_Full + t1: { n: 0 } # 第一轮不评 G3 + calib: { n: 128, from: samples, seed: 20260914, note: "occ 第一轮只取 128 帧做校准" } + + occ3d_nuscenes: + version: labels-v1 + tasks: [occ] + round: 2 # 第二轮再入池 + sources: + - kind: github + url: https://github.com/Tsinghua-MARS-Lab/Occ3D diff --git a/registry/gates.yaml b/registry/gates.yaml new file mode 100644 index 0000000..fbec3ec --- /dev/null +++ b/registry/gates.yaml @@ -0,0 +1,100 @@ +# registry/gates.yaml — 四道门阈值。随 run 快照进 job.json。 +# 容差 kind 必须显式:abs_pp(绝对百分点,Δ = quant − fp32)| rel_pct(相对劣化 %,(quant−fp32)/fp32) +# 来源列:交付指标文档 S* 编号;"proposed" = 本方案提议,第一个 run 后校准 +version: 1 + +G1: + require_hbm: true + check_march: true + check_input_dtype_layout: true + run_hb_verifier: true + log_must_not_contain: ["ERROR"] + +G2: + default_threshold: 0.99 + report: min # 只报 min,不报均值 + n_samples: 32 # 校准外样本,从 T1 另抽 + overrides: + # batch-1(r20260915-03)校准:ShuffleNetV2 ×0.5/1.0/1.5、MobileNetV2 top-1 只掉 0.55–1.5 pp(G3 合格), + # 但 1000 类 logits 的 32 样本 min 余弦 0.93–0.98。0.99 来自特征匹配场景,对 logits 太严 → cls 放到 0.95。 + # det 保持 0.99:yolov8n 余弦 0.9993 而 mAP 掉 15,G2 对检测头输出没有判别力,真正的判官是 G3。 + cls: 0.95 + occ: 0.97 + feat: # 逐输出 + desc: 0.99 + heatmap: 0.92 + +G3: + # T1 上 tol 自动放宽 tol − 2·SE,记 pass-screen;T2 才 pass + t1_screen_widen_se: 2 + tasks: + cls: + metric: top1 + dataset: imagenet1k + kind: abs_pp + tol: -1.0 + also_record: [top5] + source: "S9 MobileNetv2 −0.73 pp" + det: + metric: map_50_95 + dataset: coco2017 + kind: abs_pp + tol: -1.5 + also_record: [ap50] + source: "S2 DOSODv2 目标差 1.3 / 1.0" + seg: + metric: miou + dataset: [ade20k, cityscapes] + kind: abs_pp + tol: -1.0 + also_record: [per_class_iou] + source: "S9 Unet −0.44 pp" + iseg: + metric: mask_ap + dataset: [coco2017, lvis] + kind: abs_pp + tol: -1.5 + source: "proposed (借用 T02)" + pose: + metric: oks_ap + dataset: coco2017 + kind: abs_pp + tol: -1.5 + extra: { keypoint_mean_shift_px_max: 2 } + source: "S11 1–2 px 可接受;AP 借用" + mde: + metric: absrel + dataset: [kitti_eigen, nyuv2] + kind: rel_pct + tol: 10 + extra: { delta1_abs_pp_tol: -1.0, scale_alignment: required } + source: "S12 仅 0.90 相关性;proposed" + stereo: + metric: [epe, d1_all] + dataset: kitti2015 + kind: rel_pct + tol: 15 + source: "S4 SOW 量化允许 ≤ 15%" + flow: + metric: [epe2d, fl_all] + dataset: [sintel, kitti2015] + kind: rel_pct + tol: 15 + source: "proposed (借用 T07)" + feat: + metric: hpatches_homography_auc + dataset: hpatches + kind: abs_pp + tol: -2.0 + source: "S14 DFeature 0.9909 / 0.9282" + occ: + metric: [occ_iou, sem_miou] + dataset: occ3d_nuscenes + kind: abs_pp + tol: -1.5 + source: "S15 无阈值;proposed" + +G4: + mandatory: false + judge_only_if: latency_budget_ms # job.json 声明才判 + record: [latency_ms_mean, latency_ms_p95, bpu_util, ddr_bw_mbps] diff --git a/registry/matrix.yaml b/registry/matrix.yaml new file mode 100644 index 0000000..5f610ae --- /dev/null +++ b/registry/matrix.yaml @@ -0,0 +1,54 @@ +# registry/matrix.yaml — march × backend × qcfg × mode × policy 的组合与裁剪 +# 展开规则见 harness/registry.py::expand_matrix。展开结果写 state/jobs.sqlite,展开是幂等的。 +version: 1 + +marches: [nash-e, nash-m, nash-p] + +# 默认 qcfg:e/m 走 int8,p 走 fp16+int8 混合(段内只允许 a-z0-9. → 写作 fp16int8) +qcfg_by_march: + nash-e: int8 + nash-m: int8 + nash-p: fp16int8 + +backends: [ptq] # qat 在 P4 之后加 + +# 默认数据档;T2 只在专门的 run 里打开 +data_tier: t1 + +modes: + # oracle:全 march × backend × 全模型。每 job 一份 ground truth。 + oracle: + enabled: true + select: { priority: [A, B, C] } + marches: all + backends: all + + # e2e:agent 从零走完全流程。按 (model, backend) 只采主 march,避免轨迹重复。 + e2e: + enabled: true + select: { priority: [A], tasks: [cls, det] } # P3 先 cls/det;P4 扩到每类 6–10 个 + marches: [nash-e] + backends: [ptq] + policies: [teacher] + k: 4 # 独立 rollout 数,构成 GRPO 组 + + # repair:给 oracle 的失败现场。只吃 oracle 的 FAIL,不吃 rollout 的。 + repair: + enabled: true + when: oracle_fail + error_classes: [unsupported_op, shape_infer, calib_nan, verifier_mismatch, accuracy_collapse, accuracy_drop, numeric_mismatch] + marches: [nash-e] + backends: [ptq] + policies: [teacher] + k: 2 + +# 逐任务/逐模型覆盖。按顺序应用,后者覆盖前者。 +overrides: + - match: { task: occ } + gates: [G1, G2, G4] + g3: { verdict: na, reason: round1-compile-only } + g2_threshold: 0.97 + calib_source_ladder: [repo_demo, nuscenes_128, synthetic] + prompt: e2e-occ.md + - match: { task: feat } + g2_per_output: true diff --git a/registry/models.csv b/registry/models.csv new file mode 100644 index 0000000..3ea5379 --- /dev/null +++ b/registry/models.csv @@ -0,0 +1,339 @@ +model_id,name,task,priority,family,source,repo,weights,input_spec,resize,dataset,params_m,official_metric,status,notes,excel_id +resnet18,ResNet-18,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnet18/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,11.69,69.758,ready,TorchVision 预训练权重,可由 API 自动下载,1 +resnet34,ResNet-34,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnet34/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,21.8,73.314,ready,TorchVision 预训练权重,可由 API 自动下载,2 +resnet50,ResNet-50,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnet50/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,25.56,80.858,ready,TorchVision 预训练权重,可由 API 自动下载,3 +resnet101,ResNet-101,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnet101/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,44.55,81.886,ready,TorchVision 预训练权重,可由 API 自动下载,4 +resnet152,ResNet-152,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnet152/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,60.19,82.284,ready,TorchVision 预训练权重,可由 API 自动下载,5 +resnext50.32x4d,ResNeXt-50-32x4d,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnext50_32x4d/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,25.03,81.198,ready,TorchVision 预训练权重,可由 API 自动下载,6 +resnext101.32x8d,ResNeXt-101-32x8d,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://resnext101_32x8d/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,88.79,82.834,ready,TorchVision 预训练权重,可由 API 自动下载,7 +wide.resnet50.2,Wide-ResNet-50-2,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://wide_resnet50_2/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,68.88,81.602,ready,TorchVision 预训练权重,可由 API 自动下载,8 +wide.resnet101.2,Wide-ResNet-101-2,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://wide_resnet101_2/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,126.89,82.51,ready,TorchVision 预训练权重,可由 API 自动下载,9 +densenet121,DenseNet-121,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://densenet121/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,7.98,74.434,ready,TorchVision 预训练权重,可由 API 自动下载,10 +densenet161,DenseNet-161,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://densenet161/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,28.68,77.138,ready,TorchVision 预训练权重,可由 API 自动下载,11 +densenet169,DenseNet-169,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://densenet169/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,14.15,75.6,ready,TorchVision 预训练权重,可由 API 自动下载,12 +densenet201,DenseNet-201,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://densenet201/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,20.01,76.896,ready,TorchVision 预训练权重,可由 API 自动下载,13 +mobilenet.v2,MobileNetV2,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://mobilenet_v2/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,3.5,72.154,ready,TorchVision 预训练权重,可由 API 自动下载,14 +mobilenet.v3.small,MobileNetV3-Small,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://mobilenet_v3_small/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,2.54,67.668,ready,TorchVision 预训练权重,可由 API 自动下载,15 +mobilenet.v3.large,MobileNetV3-Large,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://mobilenet_v3_large/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,5.48,75.274,ready,TorchVision 预训练权重,可由 API 自动下载,16 +shufflenet.v2.x0.5,ShuffleNetV2-x0.5,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://shufflenet_v2_x0_5/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,1.37,60.552,ready,TorchVision 预训练权重,可由 API 自动下载,17 +shufflenet.v2.x1.0,ShuffleNetV2-x1.0,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://shufflenet_v2_x1_0/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,2.28,69.362,ready,TorchVision 预训练权重,可由 API 自动下载,18 +shufflenet.v2.x1.5,ShuffleNetV2-x1.5,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://shufflenet_v2_x1_5/IMAGENET1K_V1,1x3x224x224,232,imagenet1k,3.5,72.996,ready,TorchVision 预训练权重,可由 API 自动下载,19 +shufflenet.v2.x2.0,ShuffleNetV2-x2.0,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://shufflenet_v2_x2_0/IMAGENET1K_V1,1x3x224x224,232,imagenet1k,7.39,76.23,ready,TorchVision 预训练权重,可由 API 自动下载,20 +efficientnet.b0,EfficientNet-B0,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b0/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,5.29,77.692,ready,TorchVision 预训练权重,可由 API 自动下载,21 +efficientnet.b1,EfficientNet-B1,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b1/IMAGENET1K_V2,1x3x240x240,255,imagenet1k,7.79,79.838,ready,TorchVision 预训练权重,可由 API 自动下载,22 +efficientnet.b2,EfficientNet-B2,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b2/IMAGENET1K_V1,1x3x288x288,288,imagenet1k,9.11,80.608,ready,TorchVision 预训练权重,可由 API 自动下载,23 +efficientnet.b3,EfficientNet-B3,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b3/IMAGENET1K_V1,1x3x300x300,320,imagenet1k,12.23,82.008,ready,TorchVision 预训练权重,可由 API 自动下载,24 +efficientnet.b4,EfficientNet-B4,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b4/IMAGENET1K_V1,1x3x380x380,384,imagenet1k,19.34,83.384,ready,TorchVision 预训练权重,可由 API 自动下载,25 +efficientnet.b5,EfficientNet-B5,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b5/IMAGENET1K_V1,1x3x456x456,456,imagenet1k,30.39,83.444,ready,TorchVision 预训练权重,可由 API 自动下载,26 +efficientnet.b6,EfficientNet-B6,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b6/IMAGENET1K_V1,1x3x528x528,528,imagenet1k,43.04,84.008,ready,TorchVision 预训练权重,可由 API 自动下载,27 +efficientnet.b7,EfficientNet-B7,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_b7/IMAGENET1K_V1,1x3x600x600,600,imagenet1k,66.35,84.122,ready,TorchVision 预训练权重,可由 API 自动下载,28 +efficientnet.v2.s,EfficientNetV2-S,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_v2_s/IMAGENET1K_V1,1x3x384x384,384,imagenet1k,21.46,84.228,ready,TorchVision 预训练权重,可由 API 自动下载,29 +efficientnet.v2.m,EfficientNetV2-M,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_v2_m/IMAGENET1K_V1,1x3x480x480,480,imagenet1k,54.14,85.112,ready,TorchVision 预训练权重,可由 API 自动下载,30 +efficientnet.v2.l,EfficientNetV2-L,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://efficientnet_v2_l/IMAGENET1K_V1,1x3x480x480,480,imagenet1k,118.52,85.808,ready,TorchVision 预训练权重,可由 API 自动下载,31 +convnext.tiny,ConvNeXt-Tiny,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://convnext_tiny/IMAGENET1K_V1,1x3x224x224,236,imagenet1k,28.59,82.52,ready,TorchVision 预训练权重,可由 API 自动下载,32 +convnext.small,ConvNeXt-Small,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://convnext_small/IMAGENET1K_V1,1x3x224x224,230,imagenet1k,50.22,83.616,ready,TorchVision 预训练权重,可由 API 自动下载,33 +convnext.base,ConvNeXt-Base,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://convnext_base/IMAGENET1K_V1,1x3x224x224,232,imagenet1k,88.59,84.062,ready,TorchVision 预训练权重,可由 API 自动下载,34 +convnext.large,ConvNeXt-Large,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://convnext_large/IMAGENET1K_V1,1x3x224x224,232,imagenet1k,197.77,84.414,ready,TorchVision 预训练权重,可由 API 自动下载,35 +swin.t,Swin-T,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://swin_t/IMAGENET1K_V1,1x3x224x224,232,imagenet1k,28.29,81.474,ready,TorchVision 预训练权重,可由 API 自动下载,36 +swin.s,Swin-S,cls,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://swin_s/IMAGENET1K_V1,1x3x224x224,246,imagenet1k,49.61,83.196,ready,TorchVision 预训练权重,可由 API 自动下载,37 +swin.b,Swin-B,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://swin_b/IMAGENET1K_V1,1x3x224x224,238,imagenet1k,87.77,83.582,ready,TorchVision 预训练权重,可由 API 自动下载,38 +vit.b.16,ViT-B/16,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://vit_b_16/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,86.57,81.072,ready,TorchVision 预训练权重,可由 API 自动下载,39 +vit.b.32,ViT-B/32,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://vit_b_32/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,88.22,75.912,ready,TorchVision 预训练权重,可由 API 自动下载,40 +vit.l.16,ViT-L/16,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://vit_l_16/IMAGENET1K_V1,1x3x224x224,242,imagenet1k,304.33,79.662,ready,TorchVision 预训练权重,可由 API 自动下载,41 +vit.l.32,ViT-L/32,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://vit_l_32/IMAGENET1K_V1,1x3x224x224,256,imagenet1k,306.54,76.972,ready,TorchVision 预训练权重,可由 API 自动下载,42 +vit.h.14,ViT-H/14,cls,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://vit_h_14/IMAGENET1K_SWAG_E2E_V1,1x3x518x518,518,imagenet1k,633.47,88.552,ready,TorchVision 预训练权重,可由 API 自动下载,43 +maxvit.t,MaxViT-T,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://maxvit_t/IMAGENET1K_V1,1x3x224x224,224,imagenet1k,30.92,83.7,ready,TorchVision 预训练权重,可由 API 自动下载,44 +regnet.y.400mf,RegNetY-400MF,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://regnet_y_400mf/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,4.34,75.804,ready,TorchVision 预训练权重,可由 API 自动下载,45 +regnet.y.800mf,RegNetY-800MF,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://regnet_y_800mf/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,6.43,78.828,ready,TorchVision 预训练权重,可由 API 自动下载,46 +regnet.y.1.6gf,RegNetY-1.6GF,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://regnet_y_1_6gf/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,11.2,80.876,ready,TorchVision 预训练权重,可由 API 自动下载,47 +regnet.y.3.2gf,RegNetY-3.2GF,cls,B,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://regnet_y_3_2gf/IMAGENET1K_V2,1x3x224x224,232,imagenet1k,19.44,81.982,ready,TorchVision 预训练权重,可由 API 自动下载,48 +efficientvit.b0,EfficientViT-B0,cls,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x224x224,,imagenet1k,,,needs_adapter,官方仓库含预训练权重/模型入口,49 +efficientvit.b1,EfficientViT-B1,cls,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x224x224,,imagenet1k,,,needs_adapter,官方仓库含预训练权重/模型入口,50 +efficientvit.b2,EfficientViT-B2,cls,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x224x224,,imagenet1k,,,needs_adapter,官方仓库含预训练权重/模型入口,51 +efficientvit.b3,EfficientViT-B3,cls,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x224x224,,imagenet1k,,,needs_adapter,官方仓库含预训练权重/模型入口,52 +faster.r.cnn.r50.fpn,Faster R-CNN R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,53 +faster.r.cnn.r101.fpn,Faster R-CNN R101-FPN,det,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,54 +cascade.r.cnn.r50.fpn,Cascade R-CNN R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,55 +cascade.r.cnn.r101.fpn,Cascade R-CNN R101-FPN,det,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,56 +retinanet.r50.fpn,RetinaNet R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,57 +retinanet.r101.fpn,RetinaNet R101-FPN,det,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,58 +fcos.r50.fpn,FCOS R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,59 +atss.r50.fpn,ATSS R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,60 +gfl.r50.fpn,GFL R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,61 +vfnet.r50.fpn,VFNet R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,62 +foveabox.r50.fpn,FoveaBox R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,63 +fsaf.r50.fpn,FSAF R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,64 +reppoints.r50.fpn,RepPoints R50-FPN,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,65 +cornernet.hourglass.104,CornerNet Hourglass-104,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,66 +ssd300.vgg16,SSD300-VGG16,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,67 +ssdlite320.mobilenetv3.large,SSDLite320-MobileNetV3-Large,det,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,68 +yolov3.darknet53,YOLOv3-Darknet53,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,69 +yolox.tiny,YOLOX-Tiny,det,A,megvii-basedetection/yolox https:,custom,https://github.com/Megvii-BaseDetection/YOLOX,https://github.com/Megvii-BaseDetection/YOLOX,1x3x640x640,,coco2017,,,needs_adapter,官方 release/model zoo,70 +yolox.s,YOLOX-S,det,A,megvii-basedetection/yolox https:,custom,https://github.com/Megvii-BaseDetection/YOLOX,https://github.com/Megvii-BaseDetection/YOLOX,1x3x640x640,,coco2017,,,needs_adapter,官方 release/model zoo,71 +yolox.m,YOLOX-M,det,B,megvii-basedetection/yolox https:,custom,https://github.com/Megvii-BaseDetection/YOLOX,https://github.com/Megvii-BaseDetection/YOLOX,1x3x640x640,,coco2017,,,needs_adapter,官方 release/model zoo,72 +yolox.l,YOLOX-L,det,B,megvii-basedetection/yolox https:,custom,https://github.com/Megvii-BaseDetection/YOLOX,https://github.com/Megvii-BaseDetection/YOLOX,1x3x640x640,,coco2017,,,needs_adapter,官方 release/model zoo,73 +yolox.x,YOLOX-X,det,B,megvii-basedetection/yolox https:,custom,https://github.com/Megvii-BaseDetection/YOLOX,https://github.com/Megvii-BaseDetection/YOLOX,1x3x640x640,,coco2017,,,needs_adapter,官方 release/model zoo,74 +rtmdet.tiny,RTMDet-Tiny,det,A,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,75 +rtmdet.s,RTMDet-S,det,A,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,76 +rtmdet.m,RTMDet-M,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,77 +rtmdet.l,RTMDet-L,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,78 +rtmdet.x,RTMDet-X,det,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,79 +yolov5n,YOLOv5n,det,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov5n.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,80 +yolov5s,YOLOv5s,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov5s.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,81 +yolov5m,YOLOv5m,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov5m.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,82 +yolov5l,YOLOv5l,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov5l.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,83 +yolov5x,YOLOv5x,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov5x.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,84 +yolov8n,YOLOv8n,det,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8n.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,85 +yolov8s,YOLOv8s,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8s.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,86 +yolov8m,YOLOv8m,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8m.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,87 +yolov8l,YOLOv8l,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8l.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,88 +yolov8x,YOLOv8x,det,C,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8x.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,89 +yolo11n,YOLO11n,det,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11n.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,90 +yolo11s,YOLO11s,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11s.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,91 +yolo11m,YOLO11m,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11m.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,92 +yolo11l,YOLO11l,det,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11l.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,93 +yolo11x,YOLO11x,det,C,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11x.pt,1x3x640x640,,coco2017,,,ready,Ultralytics 权重名可自动下载,94 +detr.r50,DETR-R50,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,95 +deformable.detr.r50,Deformable-DETR-R50,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,96 +conditional.detr.r50,Conditional-DETR-R50,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,97 +dab.detr.r50,DAB-DETR-R50,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,98 +dino.r50,DINO-R50,det,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection model zoo,99 +rt.detr.r18,RT-DETR-R18,det,A,lyuwenyu/rt-detr https:,custom,https://github.com/lyuwenyu/RT-DETR,https://github.com/lyuwenyu/RT-DETR,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoint/model zoo,100 +rt.detr.r50,RT-DETR-R50,det,B,lyuwenyu/rt-detr https:,custom,https://github.com/lyuwenyu/RT-DETR,https://github.com/lyuwenyu/RT-DETR,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoint/model zoo,101 +rt.detrv2.r18,RT-DETRv2-R18,det,A,lyuwenyu/rt-detr https:,custom,https://github.com/lyuwenyu/RT-DETR,https://github.com/lyuwenyu/RT-DETR,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoint/model zoo,102 +rt.detrv2.r50,RT-DETRv2-R50,det,B,lyuwenyu/rt-detr https:,custom,https://github.com/lyuwenyu/RT-DETR,https://github.com/lyuwenyu/RT-DETR,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoint/model zoo,103 +fcn.resnet50,FCN-ResNet50,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,104 +fcn.resnet101,FCN-ResNet101,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,105 +deeplabv3.resnet50,DeepLabV3-ResNet50,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,106 +deeplabv3.resnet101,DeepLabV3-ResNet101,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,107 +deeplabv3.mobilenetv3.large,DeepLabV3-MobileNetV3-Large,seg,C,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,108 +deeplabv3plus.resnet50,DeepLabV3+-ResNet50,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,109 +deeplabv3plus.resnet101,DeepLabV3+-ResNet101,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,110 +pspnet.resnet50,PSPNet-ResNet50,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,111 +pspnet.resnet101,PSPNet-ResNet101,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,112 +upernet.resnet50,UPerNet-ResNet50,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,113 +upernet.swin.t,UPerNet-Swin-T,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,114 +upernet.swin.s,UPerNet-Swin-S,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,115 +upernet.swin.b,UPerNet-Swin-B,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,116 +segformer.b0,SegFormer-B0,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,117 +segformer.b1,SegFormer-B1,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,118 +segformer.b2,SegFormer-B2,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,119 +segformer.b3,SegFormer-B3,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,120 +segformer.b4,SegFormer-B4,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,121 +segformer.b5,SegFormer-B5,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,122 +mask2former.swin.t,Mask2Former-Swin-T,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,123 +mask2former.swin.s,Mask2Former-Swin-S,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,124 +mask2former.swin.b,Mask2Former-Swin-B,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,125 +ddrnet.23.slim,DDRNet-23-Slim,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,126 +ddrnet.23,DDRNet-23,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,127 +ddrnet.39,DDRNet-39,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,128 +pidnet.s,PIDNet-S,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,129 +pidnet.m,PIDNet-M,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,130 +pidnet.l,PIDNet-L,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,131 +bisenetv1.r18,BiSeNetV1-R18,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,132 +bisenetv2,BiSeNetV2,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,133 +stdc1.seg,STDC1-Seg,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,134 +stdc2.seg,STDC2-Seg,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,135 +fast.scnn,Fast-SCNN,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,136 +hrnet.w18,HRNet-W18,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,137 +hrnet.w48,HRNet-W48,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,138 +ocrnet.hrnet.w18,OCRNet-HRNet-W18,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,139 +ocrnet.hrnet.w48,OCRNet-HRNet-W48,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,140 +segnext.tiny,SegNeXt-Tiny,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,141 +segnext.small,SegNeXt-Small,seg,A,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,142 +segnext.base,SegNeXt-Base,seg,B,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,143 +segnext.large,SegNeXt-Large,seg,C,mmsegmentation,openmmlab,https://github.com/open-mmlab/mmsegmentation,https://github.com/open-mmlab/mmsegmentation/blob/main/model-index.yml,1x3x512x1024,,ade20k,,,needs_adapter,MMSegmentation model zoo,144 +efficientvit.seg.b0,EfficientViT-Seg-B0,seg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x512x1024,,ade20k,,,needs_adapter,官方 EfficientViT segmentation 权重,145 +efficientvit.seg.b1,EfficientViT-Seg-B1,seg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x512x1024,,ade20k,,,needs_adapter,官方 EfficientViT segmentation 权重,146 +efficientvit.seg.b2,EfficientViT-Seg-B2,seg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x512x1024,,ade20k,,,needs_adapter,官方 EfficientViT segmentation 权重,147 +efficientvit.seg.b3,EfficientViT-Seg-B3,seg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit,1x3x512x1024,,ade20k,,,needs_adapter,官方 EfficientViT segmentation 权重,148 +mask.r.cnn.r50.fpn,Mask R-CNN R50-FPN,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,149 +mask.r.cnn.r101.fpn,Mask R-CNN R101-FPN,iseg,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,150 +cascade.mask.r.cnn.r50.fpn,Cascade Mask R-CNN R50-FPN,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,151 +cascade.mask.r.cnn.r101.fpn,Cascade Mask R-CNN R101-FPN,iseg,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,152 +htc.r50.fpn,HTC R50-FPN,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,153 +htc.r101.fpn,HTC R101-FPN,iseg,C,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,154 +solo.r50,SOLO R50,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,155 +solov2.r50,SOLOv2 R50,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,156 +condinst.r50,CondInst R50,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,157 +queryinst.r50,QueryInst R50,iseg,B,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,158 +mask2former.swin.t.instance,Mask2Former-Swin-T-Instance,iseg,A,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,159 +mask2former.swin.s.instance,Mask2Former-Swin-S-Instance,iseg,A,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,160 +mask2former.swin.b.instance,Mask2Former-Swin-B-Instance,iseg,A,mmdetection,openmmlab,https://github.com/open-mmlab/mmdetection,https://github.com/open-mmlab/mmdetection/blob/main/docs/en/model_zoo.md,1x3x640x640,,coco2017,,,needs_adapter,MMDetection instance segmentation model zoo,161 +yolov8n.seg,YOLOv8n-seg,iseg,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8n-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,162 +yolov8s.seg,YOLOv8s-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8s-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,163 +yolov8m.seg,YOLOv8m-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8m-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,164 +yolov8l.seg,YOLOv8l-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8l-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,165 +yolov8x.seg,YOLOv8x-seg,iseg,C,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolov8x-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,166 +yolo11n.seg,YOLO11n-seg,iseg,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11n-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,167 +yolo11s.seg,YOLO11s-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11s-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,168 +yolo11m.seg,YOLO11m-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11m-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,169 +yolo11l.seg,YOLO11l-seg,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11l-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,170 +yolo11x.seg,YOLO11x-seg,iseg,C,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11x-seg.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,171 +sam.vit.b,SAM-ViT-B,iseg,B,facebookresearch/segment-anything https:,custom,https://github.com/facebookresearch/segment-anything,https://github.com/facebookresearch/segment-anything,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,172 +sam.vit.l,SAM-ViT-L,iseg,C,facebookresearch/segment-anything https:,custom,https://github.com/facebookresearch/segment-anything,https://github.com/facebookresearch/segment-anything,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,173 +sam.vit.h,SAM-ViT-H,iseg,C,facebookresearch/segment-anything https:,custom,https://github.com/facebookresearch/segment-anything,https://github.com/facebookresearch/segment-anything,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,174 +sam2.1.hiera.tiny,SAM2.1-Hiera-Tiny,iseg,A,facebookresearch/sam2 https:,custom,https://github.com/facebookresearch/sam2,https://github.com/facebookresearch/sam2,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,175 +sam2.1.hiera.small,SAM2.1-Hiera-Small,iseg,A,facebookresearch/sam2 https:,custom,https://github.com/facebookresearch/sam2,https://github.com/facebookresearch/sam2,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,176 +sam2.1.hiera.baseplus,SAM2.1-Hiera-Base+,iseg,B,facebookresearch/sam2 https:,custom,https://github.com/facebookresearch/sam2,https://github.com/facebookresearch/sam2,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,177 +sam2.1.hiera.large,SAM2.1-Hiera-Large,iseg,C,facebookresearch/sam2 https:,custom,https://github.com/facebookresearch/sam2,https://github.com/facebookresearch/sam2,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,178 +mobilesam,MobileSAM,iseg,A,chaoningzhang/mobilesam https:,custom,https://github.com/ChaoningZhang/MobileSAM,https://github.com/ChaoningZhang/MobileSAM,1x3x640x640,,coco2017,,,needs_adapter,官方权重,179 +edgesam,EdgeSAM,iseg,B,chongzhou96/edgesam https:,custom,https://github.com/chongzhou96/EdgeSAM,https://github.com/chongzhou96/EdgeSAM,1x3x640x640,,coco2017,,,needs_adapter,官方权重/ONNX,180 +efficientsam.ti,EfficientSAM-Ti,iseg,A,yformer/efficientsam https:,custom,https://github.com/yformer/EfficientSAM,https://github.com/yformer/EfficientSAM,1x3x640x640,,coco2017,,,needs_adapter,weights/ 目录含 checkpoint,181 +efficientsam.s,EfficientSAM-S,iseg,A,yformer/efficientsam https:,custom,https://github.com/yformer/EfficientSAM,https://github.com/yformer/EfficientSAM,1x3x640x640,,coco2017,,,needs_adapter,weights/ 目录含 checkpoint,182 +efficientvit.sam.l0,EfficientViT-SAM-L0,iseg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit/tree/master/applications/efficientvit_sam,1x3x640x640,,coco2017,,,needs_adapter,官方 pretrained models,183 +efficientvit.sam.l1,EfficientViT-SAM-L1,iseg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit/tree/master/applications/efficientvit_sam,1x3x640x640,,coco2017,,,needs_adapter,官方 pretrained models,184 +efficientvit.sam.l2,EfficientViT-SAM-L2,iseg,A,mit-han-lab/efficientvit https:,custom,https://github.com/mit-han-lab/efficientvit,https://github.com/mit-han-lab/efficientvit/tree/master/applications/efficientvit_sam,1x3x640x640,,coco2017,,,needs_adapter,官方 pretrained models,185 +fastsam.s,FastSAM-s,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://FastSAM-s.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,186 +fastsam.x,FastSAM-x,iseg,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://FastSAM-x.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics 自动/官方下载,187 +hq.sam.vit.b,HQ-SAM-ViT-B,iseg,A,syscv/sam-hq https:,custom,https://github.com/SysCV/sam-hq,https://github.com/SysCV/sam-hq,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,188 +hq.sam.vit.l,HQ-SAM-ViT-L,iseg,C,syscv/sam-hq https:,custom,https://github.com/SysCV/sam-hq,https://github.com/SysCV/sam-hq,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,189 +hq.sam.vit.h,HQ-SAM-ViT-H,iseg,C,syscv/sam-hq https:,custom,https://github.com/SysCV/sam-hq,https://github.com/SysCV/sam-hq,1x3x640x640,,coco2017,,,needs_adapter,官方 checkpoints,190 +repvit.sam,RepViT-SAM,iseg,A,thu-mig/repvit https:,custom,https://github.com/THU-MIG/RepViT,https://github.com/THU-MIG/RepViT,1x3x640x640,,coco2017,,,needs_adapter,RepViT-SAM 权重入口,191 +tinysam,TinySAM,iseg,A,xinghaochen/tinysam https:,custom,https://github.com/xinghaochen/TinySAM,https://github.com/xinghaochen/TinySAM,1x3x640x640,,coco2017,,,needs_adapter,官方权重,192 +slimsam.50,SlimSAM-50,iseg,B,czg1225/slimsam https:,custom,https://github.com/czg1225/SlimSAM,https://github.com/czg1225/SlimSAM,1x3x640x640,,coco2017,,,needs_adapter,SlimSAM-50/77 官方权重,193 +slimsam.77,SlimSAM-77,iseg,B,czg1225/slimsam https:,custom,https://github.com/czg1225/SlimSAM,https://github.com/czg1225/SlimSAM,1x3x640x640,,coco2017,,,needs_adapter,SlimSAM-50/77 官方权重,194 +efficienttam.ti,EfficientTAM-Ti,iseg,A,yformer/efficienttam https:,custom,https://github.com/yformer/EfficientTAM,https://github.com/yformer/EfficientTAM,1x3x640x640,,coco2017,,,needs_adapter,checkpoints/download_checkpoints.sh,195 +efficienttam.s,EfficientTAM-S,iseg,A,yformer/efficienttam https:,custom,https://github.com/yformer/EfficientTAM,https://github.com/yformer/EfficientTAM,1x3x640x640,,coco2017,,,needs_adapter,checkpoints/download_checkpoints.sh,196 +simplebaseline.r50,SimpleBaseline-R50,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,197 +simplebaseline.r101,SimpleBaseline-R101,pose,C,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,198 +simplebaseline.r152,SimpleBaseline-R152,pose,C,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,199 +hrnet.w32,HRNet-W32,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,200 +hrnet.w48.2,HRNet-W48,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,201 +litehrnet.18,LiteHRNet-18,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,202 +litehrnet.30,LiteHRNet-30,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,203 +rtmpose.tiny,RTMPose-Tiny,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,204 +rtmpose.s,RTMPose-S,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,205 +rtmpose.m,RTMPose-M,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,206 +rtmpose.l,RTMPose-L,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,207 +rtmpose.x,RTMPose-X,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,208 +vitpose.s,ViTPose-S,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,209 +vitpose.b,ViTPose-B,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,210 +vitpose.l,ViTPose-L,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,211 +vitpose.h,ViTPose-H,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,212 +mspn.r50,MSPN-R50,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,213 +mspn.r101,MSPN-R101,pose,C,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,214 +higherhrnet.w32,HigherHRNet-W32,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,215 +higherhrnet.w48,HigherHRNet-W48,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,216 +dekr.w32,DEKR-W32,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,217 +dekr.w48,DEKR-W48,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,218 +simcc.resnet50,SimCC-ResNet50,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,219 +rtmo.s,RTMO-S,pose,A,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,220 +rtmo.m,RTMO-M,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,221 +rtmo.l,RTMO-L,pose,B,mmpose,openmmlab,https://github.com/open-mmlab/mmpose,https://github.com/open-mmlab/mmpose/blob/main/model-index.yml,1x3x640x640,,coco2017,,,needs_adapter,MMPose model zoo,222 +yolo11n.pose,YOLO11n-pose,pose,A,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11n-pose.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics pose weights,223 +yolo11s.pose,YOLO11s-pose,pose,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11s-pose.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics pose weights,224 +yolo11m.pose,YOLO11m-pose,pose,B,ultralytics,ultralytics,https://docs.ultralytics.com/models/,ultralytics://yolo11m-pose.pt,1x3x640x640,,coco2017,,,needs_eval,Ultralytics pose weights,225 +openpose.body25,OpenPose-Body25,pose,B,cmu-perceptual-computing-lab/openpose https:,custom,https://github.com/CMU-Perceptual-Computing-Lab/openpose,https://github.com/CMU-Perceptual-Computing-Lab/openpose,1x3x640x640,,coco2017,,,needs_adapter,官方模型下载脚本/文档,226 +midas.small,MiDaS-Small,mde,A,isl-org/midas https:,custom,https://github.com/isl-org/MiDaS,https://github.com/isl-org/MiDaS,1x3x518x518,,kitti2015,,,needs_adapter,官方 model zoo,227 +dpt.hybrid,DPT-Hybrid,mde,B,isl-org/midas https:,custom,https://github.com/isl-org/MiDaS,https://github.com/isl-org/MiDaS,1x3x518x518,,kitti2015,,,needs_adapter,官方 model zoo,228 +dpt.large,DPT-Large,mde,C,isl-org/midas https:,custom,https://github.com/isl-org/MiDaS,https://github.com/isl-org/MiDaS,1x3x518x518,,kitti2015,,,needs_adapter,官方 model zoo,229 +depth.anything.v1.small,Depth-Anything-V1-Small,mde,A,liheyoung/depth-anything https:,custom,https://github.com/LiheYoung/Depth-Anything,https://github.com/LiheYoung/Depth-Anything,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,230 +depth.anything.v1.base,Depth-Anything-V1-Base,mde,B,liheyoung/depth-anything https:,custom,https://github.com/LiheYoung/Depth-Anything,https://github.com/LiheYoung/Depth-Anything,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,231 +depth.anything.v1.large,Depth-Anything-V1-Large,mde,C,liheyoung/depth-anything https:,custom,https://github.com/LiheYoung/Depth-Anything,https://github.com/LiheYoung/Depth-Anything,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,232 +depth.anything.v2.small,Depth-Anything-V2-Small,mde,A,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,233 +depth.anything.v2.base,Depth-Anything-V2-Base,mde,B,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,234 +depth.anything.v2.large,Depth-Anything-V2-Large,mde,C,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,235 +depth.anything.v2.metric.hypersim.small,Depth-Anything-V2-Metric-Hypersim-Small,mde,A,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,236 +depth.anything.v2.metric.hypersim.base,Depth-Anything-V2-Metric-Hypersim-Base,mde,B,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,237 +depth.anything.v2.metric.hypersim.large,Depth-Anything-V2-Metric-Hypersim-Large,mde,C,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,238 +depth.anything.v2.metric.vkitti.small,Depth-Anything-V2-Metric-VKITTI-Small,mde,A,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,239 +depth.anything.v2.metric.vkitti.base,Depth-Anything-V2-Metric-VKITTI-Base,mde,B,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,240 +depth.anything.v2.metric.vkitti.large,Depth-Anything-V2-Metric-VKITTI-Large,mde,C,depthanything/depth-anything-v2 https:,custom,https://github.com/DepthAnything/Depth-Anything-V2,https://github.com/DepthAnything/Depth-Anything-V2,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoints/HF,241 +zoedepth.n,ZoeDepth-N,mde,B,isl-org/zoedepth https:,custom,https://github.com/isl-org/ZoeDepth,https://github.com/isl-org/ZoeDepth,1x3x518x518,,kitti2015,,,needs_adapter,官方 torch hub/checkpoints,242 +zoedepth.k,ZoeDepth-K,mde,B,isl-org/zoedepth https:,custom,https://github.com/isl-org/ZoeDepth,https://github.com/isl-org/ZoeDepth,1x3x518x518,,kitti2015,,,needs_adapter,官方 torch hub/checkpoints,243 +zoedepth.nk,ZoeDepth-NK,mde,B,isl-org/zoedepth https:,custom,https://github.com/isl-org/ZoeDepth,https://github.com/isl-org/ZoeDepth,1x3x518x518,,kitti2015,,,needs_adapter,官方 torch hub/checkpoints,244 +metric3dv2.vit.small,Metric3Dv2-ViT-Small,mde,A,yvanyin/metric3d https:,custom,https://github.com/YvanYin/Metric3D,https://github.com/YvanYin/Metric3D,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint / torch.hub / ONNX,245 +metric3dv2.vit.large,Metric3Dv2-ViT-Large,mde,C,yvanyin/metric3d https:,custom,https://github.com/YvanYin/Metric3D,https://github.com/YvanYin/Metric3D,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint / torch.hub / ONNX,246 +metric3dv2.vit.giant2,Metric3Dv2-ViT-Giant2,mde,C,yvanyin/metric3d https:,custom,https://github.com/YvanYin/Metric3D,https://github.com/YvanYin/Metric3D,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint / torch.hub / ONNX,247 +metric3d.v1.convnext.tiny,Metric3D-v1-ConvNeXt-Tiny,mde,A,yvanyin/metric3d https:,custom,https://github.com/YvanYin/Metric3D,https://github.com/YvanYin/Metric3D,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint / torch.hub / ONNX,248 +metric3d.v1.convnext.large,Metric3D-v1-ConvNeXt-Large,mde,C,yvanyin/metric3d https:,custom,https://github.com/YvanYin/Metric3D,https://github.com/YvanYin/Metric3D,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint / torch.hub / ONNX,249 +unidepth.v1.convnext.l,UniDepth-V1-ConvNeXt-L,mde,B,lpiccinelli-eth/unidepth https:,custom,https://github.com/lpiccinelli-eth/UniDepth,https://huggingface.co/lpiccinelli,1x3x518x518,,kitti2015,,,needs_adapter,官方 Hugging Face model zoo,250 +unidepth.v1.vit.l,UniDepth-V1-ViT-L,mde,C,lpiccinelli-eth/unidepth https:,custom,https://github.com/lpiccinelli-eth/UniDepth,https://huggingface.co/lpiccinelli,1x3x518x518,,kitti2015,,,needs_adapter,官方 Hugging Face model zoo,251 +unidepth.v2.vit.s,UniDepth-V2-ViT-S,mde,A,lpiccinelli-eth/unidepth https:,custom,https://github.com/lpiccinelli-eth/UniDepth,https://huggingface.co/lpiccinelli,1x3x518x518,,kitti2015,,,needs_adapter,官方 Hugging Face model zoo,252 +unidepth.v2.vit.b,UniDepth-V2-ViT-B,mde,B,lpiccinelli-eth/unidepth https:,custom,https://github.com/lpiccinelli-eth/UniDepth,https://huggingface.co/lpiccinelli,1x3x518x518,,kitti2015,,,needs_adapter,官方 Hugging Face model zoo,253 +unidepth.v2.vit.l,UniDepth-V2-ViT-L,mde,C,lpiccinelli-eth/unidepth https:,custom,https://github.com/lpiccinelli-eth/UniDepth,https://huggingface.co/lpiccinelli,1x3x518x518,,kitti2015,,,needs_adapter,官方 Hugging Face model zoo,254 +depthpro,DepthPro,mde,B,apple/ml-depth-pro https:,custom,https://github.com/apple/ml-depth-pro,https://github.com/apple/ml-depth-pro,1x3x518x518,,kitti2015,,,needs_adapter,官方 checkpoint,255 +psmnet,PSMNet,stereo,A,jiarenchang/psmnet https:,custom,https://github.com/JiaRenChang/PSMNet,https://github.com/JiaRenChang/PSMNet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,256 +gwcnet,GwcNet,stereo,A,xy-guo/gwcnet https:,custom,https://github.com/xy-guo/GwcNet,https://github.com/xy-guo/GwcNet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,257 +ganet,GANet,stereo,A,feihuzhang/ganet https:,custom,https://github.com/feihuzhang/GANet,https://github.com/feihuzhang/GANet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,258 +aanet,AANet,stereo,A,haofeixu/aanet https:,custom,https://github.com/haofeixu/aanet,https://github.com/haofeixu/aanet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,259 +leastereo,LEAStereo,stereo,A,xueliancheng/leastereo https:,custom,https://github.com/XuelianCheng/LEAStereo,https://github.com/XuelianCheng/LEAStereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,260 +anynet,AnyNet,stereo,A,mileyan/anynet https:,custom,https://github.com/mileyan/AnyNet,https://github.com/mileyan/AnyNet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,261 +hitnet,HITNet,stereo,A,google-research/google-research,custom,https://github.com/google-research/google-research/tree/master/hitnet,https://github.com/google-research/google-research/tree/master/hitnet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,262 +mobilestereonet.2d,MobileStereoNet-2D,stereo,A,cogsys-tuebingen/mobilestereonet https:,custom,https://github.com/cogsys-tuebingen/mobilestereonet,https://github.com/cogsys-tuebingen/mobilestereonet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,263 +mobilestereonet.3d,MobileStereoNet-3D,stereo,A,cogsys-tuebingen/mobilestereonet https:,custom,https://github.com/cogsys-tuebingen/mobilestereonet,https://github.com/cogsys-tuebingen/mobilestereonet,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,264 +raft.stereo,RAFT-Stereo,stereo,A,princeton-vl/raft-stereo https:,custom,https://github.com/princeton-vl/RAFT-Stereo,https://github.com/princeton-vl/RAFT-Stereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,265 +iraftstereo.rvc,iRAFTStereo-RVC,stereo,A,princeton-vl/raft-stereo https:,custom,https://github.com/princeton-vl/RAFT-Stereo,https://github.com/princeton-vl/RAFT-Stereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,266 +crestereo,CREStereo,stereo,A,megvii-research/crestereo https:,custom,https://github.com/megvii-research/CREStereo,https://github.com/megvii-research/CREStereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,267 +igev.stereo,IGEV-Stereo,stereo,A,gangweix/igev https:,custom,https://github.com/gangweiX/IGEV,https://github.com/gangweiX/IGEV,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,268 +unimatch.stereo,UniMatch-Stereo,stereo,A,autonomousvision/unimatch https:,custom,https://github.com/autonomousvision/unimatch,https://github.com/autonomousvision/unimatch,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,269 +sttr,STTR,stereo,A,mli0603/stereo-transformer https:,custom,https://github.com/mli0603/stereo-transformer,https://github.com/mli0603/stereo-transformer,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,270 +foundationstereo,FoundationStereo,stereo,C,nvlabs/foundationstereo https:,custom,https://github.com/NVlabs/FoundationStereo,https://github.com/NVlabs/FoundationStereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,271 +fast.foundationstereo,Fast-FoundationStereo,stereo,C,nvlabs/fast-foundationstereo https:,custom,https://github.com/NVlabs/Fast-FoundationStereo,https://github.com/NVlabs/Fast-FoundationStereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,272 +waft.stereo,WAFT-Stereo,stereo,A,princeton-vl/waft-stereo https:,custom,https://github.com/princeton-vl/WAFT-Stereo,https://github.com/princeton-vl/WAFT-Stereo,1x3x384x1248,,kitti2015,,,needs_adapter,官方仓库;按 README 下载 checkpoint,273 +flownet2,FlowNet2,flow,A,nvidia/flownet2-pytorch https:,custom,https://github.com/NVIDIA/flownet2-pytorch,https://github.com/NVIDIA/flownet2-pytorch,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,274 +spynet,SpyNet,flow,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,https://docs.pytorch.org/vision/main/models,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,275 +pwc.net,PWC-Net,flow,A,nvlabs/pwc-net https:,custom,https://github.com/NVlabs/PWC-Net,https://github.com/NVlabs/PWC-Net,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,276 +liteflownet2,LiteFlowNet2,flow,A,twhui/liteflownet2 https:,custom,https://github.com/twhui/LiteFlowNet2,https://github.com/twhui/LiteFlowNet2,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,277 +raft.large,RAFT-Large,flow,C,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://raft_large/C_T_SKHT_V2,1x3x384x1024,256,kitti2015,5.26,,needs_adapter,官方仓库/model zoo,278 +raft.small,RAFT-Small,flow,A,torchvision,torchvision,https://docs.pytorch.org/vision/main/models,torchvision://raft_small/C_T_V2,1x3x384x1024,256,kitti2015,0.99,,needs_adapter,官方仓库/model zoo,279 +gma,GMA,flow,A,zacjiang/gma https:,custom,https://github.com/zacjiang/GMA,https://github.com/zacjiang/GMA,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,280 +gmflow,GMFlow,flow,A,haofeixu/gmflow https:,custom,https://github.com/haofeixu/gmflow,https://github.com/haofeixu/gmflow,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,281 +flowformer,FlowFormer,flow,A,drinkingcoder/flowformer-official https:,custom,https://github.com/drinkingcoder/FlowFormer-Official,https://github.com/drinkingcoder/FlowFormer-Official,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,282 +flowformerplusplus,FlowFormer++,flow,C,xiaoyushi97/flowformerplusplus https:,custom,https://github.com/XiaoyuShi97/FlowFormerPlusPlus,https://github.com/XiaoyuShi97/FlowFormerPlusPlus,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,283 +unimatch.flow,UniMatch-Flow,flow,A,autonomousvision/unimatch https:,custom,https://github.com/autonomousvision/unimatch,https://github.com/autonomousvision/unimatch,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,284 +sea.raft,SEA-RAFT,flow,A,princeton-vl/sea-raft https:,custom,https://github.com/princeton-vl/SEA-RAFT,https://github.com/princeton-vl/SEA-RAFT,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,285 +memflow,MemFlow,flow,A,dqiaole/memflow https:,custom,https://github.com/DQiaole/MemFlow,https://github.com/DQiaole/MemFlow,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,286 +flow1d,Flow1D,flow,A,haofeixu/flow1d https:,custom,https://github.com/haofeixu/flow1d,https://github.com/haofeixu/flow1d,1x3x384x1024,,kitti2015,,,needs_adapter,官方仓库/model zoo,287 +sift,SIFT,feat,A,opencv,opencv,https://docs.opencv.org/4.x/,https://docs.opencv.org/4.x/,1x1x480x640,,hpatches,,,skip,经典算子,非神经网络,不量化,288 +orb,ORB,feat,A,opencv,opencv,https://docs.opencv.org/4.x/,https://docs.opencv.org/4.x/,1x1x480x640,,hpatches,,,skip,经典算子,非神经网络,不量化,289 +akaze,AKAZE,feat,A,opencv,opencv,https://docs.opencv.org/4.x/,https://docs.opencv.org/4.x/,1x1x480x640,,hpatches,,,skip,经典算子,非神经网络,不量化,290 +superpoint,SuperPoint,feat,A,magicleap/supergluepretrainednetwork https:,custom,https://github.com/magicleap/SuperGluePretrainedNetwork,https://github.com/magicleap/SuperGluePretrainedNetwork,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,291 +disk,DISK,feat,A,cvg/lightglue https:,custom,https://github.com/cvg/LightGlue,https://github.com/cvg/LightGlue,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,292 +aliked,ALIKED,feat,A,shiaoming/aliked https:,custom,https://github.com/Shiaoming/ALIKED,https://github.com/Shiaoming/ALIKED,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,293 +xfeat,XFeat,feat,A,verlab/accelerated_features https:,custom,https://github.com/verlab/accelerated_features,https://github.com/verlab/accelerated_features,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,294 +d2.net,D2-Net,feat,A,mihaidusmanu/d2-net https:,custom,https://github.com/mihaidusmanu/d2-net,https://github.com/mihaidusmanu/d2-net,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,295 +r2d2,R2D2,feat,A,naver/r2d2 https:,custom,https://github.com/naver/r2d2,https://github.com/naver/r2d2,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,296 +aslfeat,ASLFeat,feat,A,lzx551402/aslfeat https:,custom,https://github.com/lzx551402/ASLFeat,https://github.com/lzx551402/ASLFeat,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,297 +keynetplushardnet,KeyNet+HardNet,feat,A,axelbarroso/key.net https:,custom,https://github.com/axelBarroso/Key.Net,https://github.com/axelBarroso/Key.Net,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,298 +dedode.v2,DeDoDe-v2,feat,A,parskatt/dedode https:,custom,https://github.com/Parskatt/DeDoDe,https://github.com/Parskatt/DeDoDe,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,299 +superglue.indoor,SuperGlue-Indoor,feat,A,magicleap/supergluepretrainednetwork https:,custom,https://github.com/magicleap/SuperGluePretrainedNetwork,https://github.com/magicleap/SuperGluePretrainedNetwork,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,300 +superglue.outdoor,SuperGlue-Outdoor,feat,A,magicleap/supergluepretrainednetwork https:,custom,https://github.com/magicleap/SuperGluePretrainedNetwork,https://github.com/magicleap/SuperGluePretrainedNetwork,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,301 +lightglue.superpoint,LightGlue-SuperPoint,feat,A,cvg/lightglue https:,custom,https://github.com/cvg/LightGlue,https://github.com/cvg/LightGlue,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,302 +lightglue.disk,LightGlue-DISK,feat,A,cvg/lightglue https:,custom,https://github.com/cvg/LightGlue,https://github.com/cvg/LightGlue,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,303 +lightglue.aliked,LightGlue-ALIKED,feat,A,cvg/lightglue https:,custom,https://github.com/cvg/LightGlue,https://github.com/cvg/LightGlue,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,304 +lightglue.sift,LightGlue-SIFT,feat,A,cvg/lightglue https:,custom,https://github.com/cvg/LightGlue,https://github.com/cvg/LightGlue,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,305 +loftr.indoor,LoFTR-Indoor,feat,A,zju3dv/loftr https:,custom,https://github.com/zju3dv/LoFTR,https://github.com/zju3dv/LoFTR,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,306 +loftr.outdoor,LoFTR-Outdoor,feat,A,zju3dv/loftr https:,custom,https://github.com/zju3dv/LoFTR,https://github.com/zju3dv/LoFTR,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,307 +matchformer.indoor,MatchFormer-Indoor,feat,A,jamycheung/matchformer https:,custom,https://github.com/jamycheung/MatchFormer,https://github.com/jamycheung/MatchFormer,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,308 +matchformer.outdoor,MatchFormer-Outdoor,feat,A,jamycheung/matchformer https:,custom,https://github.com/jamycheung/MatchFormer,https://github.com/jamycheung/MatchFormer,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,309 +aspanformer.indoor,ASpanFormer-Indoor,feat,A,apple/ml-aspanformer https:,custom,https://github.com/apple/ml-aspanformer,https://github.com/apple/ml-aspanformer,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,310 +aspanformer.outdoor,ASpanFormer-Outdoor,feat,A,apple/ml-aspanformer https:,custom,https://github.com/apple/ml-aspanformer,https://github.com/apple/ml-aspanformer,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,311 +dkm,DKM,feat,A,parskatt/dkm https:,custom,https://github.com/Parskatt/DKM,https://github.com/Parskatt/DKM,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,312 +roma,RoMa,feat,A,parskatt/roma https:,custom,https://github.com/Parskatt/RoMa,https://github.com/Parskatt/RoMa,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,313 +gluestick,GlueStick,feat,A,cvg/gluestick https:,custom,https://github.com/cvg/GlueStick,https://github.com/cvg/GlueStick,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,314 +xfeatpluslighterglue,XFeat+LighterGlue,feat,A,verlab/accelerated_features https:,custom,https://github.com/verlab/accelerated_features,https://github.com/verlab/accelerated_features,1x1x480x640,,hpatches,,,needs_adapter,官方权重/仓库,315 +monoscene.semantickitti,MonoScene-SemanticKITTI,occ,A,astra-vision/monoscene https:,custom,https://github.com/astra-vision/MonoScene,https://github.com/astra-vision/MonoScene,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,316 +monoscene.nyuv2,MonoScene-NYUv2,occ,A,astra-vision/monoscene https:,custom,https://github.com/astra-vision/MonoScene,https://github.com/astra-vision/MonoScene,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,317 +lmscnet.semantickitti,LMSCNet-SemanticKITTI,occ,A,astra-vision/lmscnet https:,custom,https://github.com/astra-vision/LMSCNet,https://github.com/astra-vision/LMSCNet,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,318 +tpvformer,TPVFormer,occ,A,wzzheng/tpvformer https:,custom,https://github.com/wzzheng/TPVFormer,https://github.com/wzzheng/TPVFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,319 +voxformer.s,VoxFormer-S,occ,A,nvlabs/voxformer https:,custom,https://github.com/NVlabs/VoxFormer,https://github.com/NVlabs/VoxFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,320 +voxformer.t,VoxFormer-T,occ,A,nvlabs/voxformer https:,custom,https://github.com/NVlabs/VoxFormer,https://github.com/NVlabs/VoxFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,321 +voxformer.t.3d,VoxFormer-T-3D,occ,A,nvlabs/voxformer https:,custom,https://github.com/NVlabs/VoxFormer,https://github.com/NVlabs/VoxFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,322 +occformer,OccFormer,occ,A,zhangyp15/occformer https:,custom,https://github.com/zhangyp15/OccFormer,https://github.com/zhangyp15/OccFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,323 +surroundocc,SurroundOcc,occ,A,weiyithu/surroundocc https:,custom,https://github.com/weiyithu/SurroundOcc,https://github.com/weiyithu/SurroundOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,324 +occnet.r50,OccNet-R50,occ,A,opendrivelab/occnet https:,custom,https://github.com/OpenDriveLab/OccNet,https://github.com/OpenDriveLab/OccNet,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,325 +flashocc.m0,FlashOcc-M0,occ,A,yzichen/flashocc https:,custom,https://github.com/Yzichen/FlashOCC,https://github.com/Yzichen/FlashOCC,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,326 +flashocc.m1,FlashOcc-M1,occ,A,yzichen/flashocc https:,custom,https://github.com/Yzichen/FlashOCC,https://github.com/Yzichen/FlashOCC,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,327 +panoptic.flashocc,Panoptic-FlashOcc,occ,A,yzichen/flashocc https:,custom,https://github.com/Yzichen/FlashOCC,https://github.com/Yzichen/FlashOCC,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,328 +sparseocc.vision.sjtu,SparseOcc-VISION-SJTU,occ,A,vision-sjtu/sparseocc https:,custom,https://github.com/VISION-SJTU/SparseOcc,https://github.com/VISION-SJTU/SparseOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,329 +sparseocc.mcg.nju,SparseOcc-MCG-NJU,occ,A,mcg-nju/sparseocc https:,custom,https://github.com/MCG-NJU/SparseOcc,https://github.com/MCG-NJU/SparseOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,330 +renderocc,RenderOcc,occ,A,pmj110119/renderocc https:,custom,https://github.com/pmj110119/RenderOcc,https://github.com/pmj110119/RenderOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,331 +selfocc,SelfOcc,occ,A,huang-yh/selfocc https:,custom,https://github.com/huang-yh/SelfOcc,https://github.com/huang-yh/SelfOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,332 +gaussianformer,GaussianFormer,occ,A,huang-yh/gaussianformer https:,custom,https://github.com/huang-yh/GaussianFormer,https://github.com/huang-yh/GaussianFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,333 +gaussianformer.2,GaussianFormer-2,occ,C,huang-yh/gaussianformer https:,custom,https://github.com/huang-yh/GaussianFormer,https://github.com/huang-yh/GaussianFormer,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,334 +gaussianocc,GaussianOcc,occ,A,ganwanshui/gaussianocc https:,custom,https://github.com/GANWANSHUI/GaussianOcc,https://github.com/GANWANSHUI/GaussianOcc,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,335 +ctf.occ,CTF-Occ,occ,A,tsinghua-mars-lab/occ3d https:,custom,https://github.com/Tsinghua-MARS-Lab/Occ3D,https://github.com/Tsinghua-MARS-Lab/Occ3D,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,336 +bevdetocc,BEVDetOCC,occ,A,huangjunjie2017/bevdet https:,custom,https://github.com/HuangJunJie2017/BEVDet,https://github.com/HuangJunJie2017/BEVDet,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,337 +bevdetocc.4d.stereo,BEVDetOCC-4D-Stereo,occ,A,yzichen/flashocc https:,custom,https://github.com/Yzichen/FlashOCC,https://github.com/Yzichen/FlashOCC,1x3x256x704,,kitti2015,,,needs_adapter,官方仓库/model zoo,338 diff --git a/reports/.gitkeep b/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..28dec19 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# harness 自身只依赖 pyyaml;工具链在各自 venv 里 +pyyaml>=6.0 diff --git a/scripts/_probe.py b/scripts/_probe.py new file mode 100644 index 0000000..daa6da1 --- /dev/null +++ b/scripts/_probe.py @@ -0,0 +1,24 @@ +"""一次性诊断:轨迹到底有没有、agent 侧依赖齐不齐。scp 到机器上跑,不进 git 主流程。""" +import glob +import os +import shutil +import sqlite3 + +DB = "/home/ming01.wang/bench-state/jobs.sqlite" +R = "/mnt/data/ming01.wang/model-compile-hbdk/bench" + +c = sqlite3.connect(DB) +c.execute("PRAGMA busy_timeout=30000") +print("rollouts by state:", dict(c.execute("select state,count(*) from rollouts group by 1").fetchall())) +print("rollouts executed (turns not null):", c.execute("select count(*) from rollouts where turns is not null").fetchone()[0]) +print("verdict rows:", c.execute("select count(*) from rollouts where verdict is not null").fetchone()[0]) +print("jobs by state:", dict(c.execute("select state,count(*) from jobs group by 1").fetchall())) +print("datasets/:", os.listdir(R + "/datasets") if os.path.isdir(R + "/datasets") else "-") +print("rollout dirs on disk:", len(glob.glob(R + "/runs/*/*/rollouts/*"))) +print("events.jsonl on disk:", len(glob.glob(R + "/runs/*/*/rollouts/*/agent/events.jsonl"))) +for b in ("pi", "node", "npm", "npx"): + print(f" {b} -> {shutil.which(b) or 'NOT FOUND'}") +od = os.path.expanduser("~/.opencode/bin") +print("~/.opencode/bin:", os.listdir(od) if os.path.isdir(od) else "-") +pi = os.path.expanduser("~/.pi") +print("~/.pi:", os.listdir(pi) if os.path.isdir(pi) else "NOT FOUND") diff --git a/scripts/board_probe.sh b/scripts/board_probe.sh new file mode 100755 index 0000000..fcfc902 --- /dev/null +++ b/scripts/board_probe.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# 确认沙箱板是哪块(s100 / s100p / s600 → nash-e / nash-m / nash-p),并把结果写进 registry/boards.yaml 提示。 +# 用法:source bench/.env.local; scripts/board_probe.sh +# 第一次:先生成密钥并用口令 ssh-copy-id 一次(口令已泄露于明文对话,之后请轮换): +# ssh-keygen -t ed25519 -f ~/.ssh/rdk_sandbox -N '' +# ssh-copy-id -i ~/.ssh/rdk_sandbox.pub -p 40010 @rdk-sandbox.d-robotics.cc +set -uo pipefail +: "${BOARD_HOST:?source bench/.env.local}"; : "${BOARD_PORT:=22}"; : "${BOARD_USER:?}"; : "${BOARD_SSH_KEY:=~/.ssh/rdk_sandbox}" +KEY="${BOARD_SSH_KEY/#\~/$HOME}" +SSH="ssh -o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new -i $KEY -p $BOARD_PORT $BOARD_USER@$BOARD_HOST" +echo "== device-tree model"; $SSH 'cat /proc/device-tree/model 2>/dev/null; echo' +echo "== hrut_somstatus"; $SSH 'hrut_somstatus 2>/dev/null | head -20 || echo n/a' +echo "== ucp / hrt version"; $SSH 'cat /etc/ucp_version 2>/dev/null; hrt_model_exec --version 2>/dev/null | head -2; hrut_bpuprofile --version 2>/dev/null | head -1' +echo "== cpuinfo / mem"; $SSH 'grep -m1 -i "model name\|hardware" /proc/cpuinfo; nproc; free -m | head -2' +echo "== bpu nodes"; $SSH 'ls /sys/class/bpu* /dev/bpu* 2>/dev/null; cat /sys/class/bpu*/bpu*/ratio 2>/dev/null' +echo "== disk"; $SSH 'df -h /userdata 2>/dev/null || df -h / | tail -1' +MODEL="$($SSH 'cat /proc/device-tree/model 2>/dev/null' | tr -d '\0' | tr 'A-Z' 'a-z')" +case "$MODEL" in + *s100p*|*nash-m*|*nashm*) echo ">> 推断 s100p → march nash-m" ;; + *s100*|*nash-e*|*nashe*) echo ">> 推断 s100 → march nash-e" ;; + *s600*|*nash-p*|*nashp*) echo ">> 推断 s600 → march nash-p" ;; + *) echo ">> 无法从 '$MODEL' 推断;看 hrut_somstatus 输出后手动填 registry/boards.yaml 的 board_type/march" ;; +esac diff --git a/scripts/excel_to_models.py b/scripts/excel_to_models.py new file mode 100644 index 0000000..9dfbe08 --- /dev/null +++ b/scripts/excel_to_models.py @@ -0,0 +1,240 @@ +"""《视觉ModelZoo_10类_不含VLM_338个候选模型.xlsx》→ registry/models.csv。 + + ~/toolchains/bench-venv/bin/python scripts/excel_to_models.py registry/models_源清单.xlsx [--dry-run] + +sheet「模型清单」列:ID | 任务分类 | 模型/有效变体 | 首选评测集 | 补充评测集 | 指标 | 官方仓库/模型库 | 权重下载位置 | 下载说明 | 端侧优先级 | 数据集入口 + +每行给一个 status,决定它现在能不能自动跑: + ready harness 现在就能端到端跑:torchvision 分类(权重 API 自动下)、ultralytics 检测(权重名自动下) + needs_eval 能取权重、能导出,但评测器还没写(ultralytics seg / pose;P4) + needs_adapter 需要模型仓库专用的取权重/导出适配器(mmdet / mmseg / mmpose / 各家 GitHub 仓库;P4–P5) + skip 不是神经网络(SIFT / ORB / AKAZE),不量化 +matrix.yaml 默认只展开 status=ready。其余留在 registry 里,bench status 会报"未开始:needs_adapter N"。 + +torchvision 的 builder 名与默认权重的预处理尺寸从 torchvision 本身查(所以要在 bench venv 里跑)。 +""" +from __future__ import annotations + +import argparse +import csv +import re +import sys +from collections import Counter +from pathlib import Path + +TASK_MAP = [ # 顺序即优先级:先匹配更具体的 + ("iseg", ["实例分割", "提示分割"]), + ("stereo", ["双目", "视差", "立体"]), + ("occ", ["占据", "占用", "occupancy"]), + ("flow", ["光流"]), + ("feat", ["局部特征", "特征提取", "匹配"]), + ("pose", ["关键点", "姿态"]), + ("mde", ["单目深度", "深度"]), + ("seg", ["语义分割", "分割"]), + ("det", ["目标检测", "检测"]), + ("cls", ["分类"]), +] +DATASET_MAP = [("imagenet1k", ["imagenet"]), ("coco2017", ["coco"]), ("ade20k", ["ade20k"]), ("cityscapes", ["cityscapes"]), + ("lvis", ["lvis"]), ("kitti2015", ["kitti 2015", "kitti2015", "kitti"]), ("nyuv2", ["nyu"]), ("sintel", ["sintel"]), + ("hpatches", ["hpatches"]), ("occ3d_nuscenes", ["occ3d", "nuscenes"]), ("eth3d", ["eth3d"]), ("semantickitti", ["semantickitti"])] +DEFAULT_INPUT = {"cls": "1x3x224x224", "det": "1x3x640x640", "iseg": "1x3x640x640", "pose": "1x3x640x640", + "seg": "1x3x512x1024", "mde": "1x3x518x518", "stereo": "1x3x384x1248", "flow": "1x3x384x1024", + "feat": "1x1x480x640", "occ": "1x3x256x704"} +CLASSICAL = {"sift", "orb", "akaze"} + + +def norm(s) -> str: + return re.sub(r"\s+", " ", str(s or "")).strip() + + +def slug(s: str) -> str: + s = s.lower().replace("+", "plus").replace("/", "-").replace(" ", "-").replace("_", "-") + s = re.sub(r"[^a-z0-9.\-]", "", s) + s = re.sub(r"-+", "-", s).strip("-.") + return s.replace("-", ".") # job_id 段内不允许 '-' + + +def map_task(v: str) -> str | None: + for t, keys in TASK_MAP: + if any(k in v for k in keys): + return t + return None + + +def map_dataset(v: str, task: str) -> str: + low = v.lower() + for ds, keys in DATASET_MAP: + if any(k in low for k in keys): + return ds + return {"cls": "imagenet1k", "det": "coco2017", "iseg": "coco2017", "pose": "coco2017", "seg": "ade20k", + "mde": "nyuv2", "stereo": "kitti2015", "flow": "sintel", "feat": "hpatches", "occ": "occ3d_nuscenes"}[task] + + +def map_priority(v: str) -> str: + return v.strip()[:1].upper() if v.strip()[:1].upper() in "ABC" else "C" + + +def source_of(repo: str) -> tuple[str, str]: + """→ (source, family)""" + r = repo.lower() + if "pytorch.org/vision" in r or "torchvision" in r: + return "torchvision", "torchvision" + if "ultralytics" in r: + return "ultralytics", "ultralytics" + for fam in ("mmsegmentation", "mmdetection", "mmpose", "mmdepth", "mmflow"): + if fam in r: + return "openmmlab", fam + if "opencv" in r: + return "opencv", "opencv" + m = re.search(r"github\.com/([^/]+/[^/]+)", r) + if m: + return "custom", m.group(1) + m = re.search(r"huggingface\.co/([^/]+/[^/]+)", r) + if m: + return "hf", m.group(1) + return "custom", r[:40] + + +# ---------------------------------------------------------------- torchvision 解析 + +_TV_MODELS: list[str] | None = None + + +def tv_models() -> list[str]: + global _TV_MODELS + if _TV_MODELS is None: + try: + import torchvision.models as tvm + _TV_MODELS = sorted(tvm.list_models()) + except Exception as e: # noqa: BLE001 + print(f"WARN torchvision 不可用({e}),builder 名靠规则猜", file=sys.stderr) + _TV_MODELS = [] + return _TV_MODELS + + +def tv_builder(display: str) -> str | None: + """'ResNeXt-50-32x4d' → 'resnext50_32x4d';'ViT-B/16' → 'vit_b_16';'ShuffleNetV2-x1.0' → 'shufflenet_v2_x1_0'""" + key = re.sub(r"[^a-z0-9]", "", display.lower()) + cands = {re.sub(r"[^a-z0-9]", "", m): m for m in tv_models()} + if key in cands: + return cands[key] + # 常见写法差异 + alt = key.replace("mobilenetv2", "mobilenetv2").replace("wideresnet", "wideresnet") + for k, m in cands.items(): + if k == alt: + return m + # 'regnety400mf' → 'regnet_y_400mf';'maxvitt' → 'maxvit_t' + for k, m in cands.items(): + if k.replace("_", "") == key: + return m + return None + + +def tv_meta(builder: str) -> dict: + """DEFAULT 权重的 resize / crop / 参数量。""" + try: + import torchvision.models as tvm + w = tvm.get_model_weights(builder).DEFAULT + t = w.transforms() + crop = getattr(t, "crop_size", [224]) + resize = getattr(t, "resize_size", [256]) + crop = crop[0] if isinstance(crop, (list, tuple)) else crop + resize = resize[0] if isinstance(resize, (list, tuple)) else resize + return {"weights": f"torchvision://{builder}/{w.name}", "crop": int(crop), "resize": int(resize), + "params_m": round(w.meta.get("num_params", 0) / 1e6, 2) if w.meta.get("num_params") else "", + "official": w.meta.get("_metrics", {}).get("ImageNet-1K", {}).get("acc@1", "")} + except Exception as e: # noqa: BLE001 + return {"weights": f"torchvision://{builder}/DEFAULT", "crop": 224, "resize": 256, "params_m": "", "official": "", + "note": f"tv_meta failed: {e}"} + + +# ---------------------------------------------------------------- 主流程 + +def convert(xlsx: Path) -> tuple[list[dict], list[str]]: + import openpyxl + wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) + ws = wb["模型清单"] if "模型清单" in wb.sheetnames else wb.worksheets[0] + rows = list(ws.iter_rows(values_only=True)) + hdr = [norm(c) for c in rows[0]] + col = {name: hdr.index(name) for name in hdr} + need = ["任务分类", "模型/有效变体", "首选评测集", "官方仓库/模型库", "权重下载位置", "下载说明", "端侧优先级"] + for n in need: + if n not in col: + raise SystemExit(f"Excel 缺列 {n!r},现有:{hdr}") + out, warns, seen = [], [], Counter() + for r in rows[1:]: + if not any(c not in (None, "") for c in r): + continue + g = lambda name: norm(r[col[name]]) + name = g("模型/有效变体") + task = map_task(g("任务分类")) + if not task: + warns.append(f"{name}: 任务 {g('任务分类')!r} 无法映射"); continue + source, family = source_of(g("官方仓库/模型库") + " " + g("权重下载位置")) + mid = slug(name) + seen[mid] += 1 + if seen[mid] > 1: + mid = f"{mid}.{seen[mid]}" + rec = {"model_id": mid, "name": name, "task": task, "priority": map_priority(g("端侧优先级")), + "family": family, "source": source, "repo": g("官方仓库/模型库"), "weights": g("权重下载位置"), + "input_spec": DEFAULT_INPUT[task], "resize": "", "dataset": map_dataset(g("首选评测集"), task), + "params_m": "", "official_metric": "", "status": "needs_adapter", + "notes": g("下载说明"), "excel_id": norm(r[col["ID"]]) if "ID" in col else ""} + low = name.lower() + if source == "torchvision": + b = tv_builder(name) + if not b: + warns.append(f"{name}: torchvision 里找不到 builder"); rec["status"] = "needs_adapter" + else: + m = tv_meta(b) + rec.update({"model_id": b.replace("_", "."), "weights": m["weights"], "params_m": m["params_m"], + "official_metric": m["official"], "resize": m["resize"]}) + if task == "cls": + rec["input_spec"] = f"1x3x{m['crop']}x{m['crop']}" + rec["status"] = "ready" + else: + rec["status"] = "needs_adapter" # RAFT/SpyNet 双输入 + if m.get("note"): + warns.append(f"{name}: {m['note']}") + elif source == "ultralytics": + wname = low.replace("yolo11", "yolo11").replace(" ", "") + ".pt" # yolov8n / yolo11n-seg / fastsam-s + if wname.startswith("fastsam"): + wname = "FastSAM-" + wname.split("-", 1)[1] + rec["weights"] = f"ultralytics://{wname}" + rec["status"] = "ready" if task == "det" else "needs_eval" + elif source == "opencv" or any(k == re.sub(r"[^a-z]", "", low) for k in CLASSICAL): + rec["status"] = "skip"; rec["notes"] = "经典算子,非神经网络,不量化" + out.append(rec) + return out, warns + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("xlsx") + ap.add_argument("--out", default=str(Path(__file__).resolve().parent.parent / "registry" / "models.csv")) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args() + rows, warns = convert(Path(a.xlsx)) + for w in warns: + print("WARN", w, file=sys.stderr) + print("rows:", len(rows), file=sys.stderr) + print("task:", dict(Counter(r["task"] for r in rows)), file=sys.stderr) + print("priority:", dict(Counter(r["priority"] for r in rows)), file=sys.stderr) + print("status:", dict(Counter(r["status"] for r in rows)), file=sys.stderr) + print("family:", dict(Counter(r["family"] for r in rows).most_common(12)), file=sys.stderr) + ready = [r for r in rows if r["status"] == "ready"] + print(f"ready ({len(ready)}):", ", ".join(r["model_id"] for r in ready), file=sys.stderr) + dup = [k for k, v in Counter(r["model_id"] for r in rows).items() if v > 1] + if dup: + raise SystemExit(f"model_id 重复:{dup}") + if a.dry_run: + return 0 + fields = list(rows[0].keys()) + with open(a.out, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=fields); w.writeheader(); w.writerows(rows) + print(f"wrote {len(rows)} → {a.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fix_env.sh b/scripts/fix_env.sh new file mode 100755 index 0000000..f0aa16f --- /dev/null +++ b/scripts/fix_env.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# 修 .env.oe-package 里的死路径:PYTHON_BIN 指向 <项目根>/oellm2/venv/bin/python,实际 QAT venv 在 ~/toolchains/oellm2/venv。 +# agent 按规则会读这个文件,所以 P1 第一件事。备份为 .env.oe-package.bak-<日期>。 +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +F="$ROOT/.horizon/.env.oe-package"; [ -f "$F" ] || F="$ROOT/.env.oe-package" +[ -f "$F" ] || { echo "没有 .env.oe-package(找了 .horizon/ 与项目根)"; exit 1; } +NEW="${1:-$HOME/toolchains/oellm2/venv/bin/python}" +[ -x "$NEW" ] || { echo "目标解释器不存在: $NEW"; exit 2; } +cp -n "$F" "$F.bak-$(date +%Y%m%d)" || true +echo "before:"; grep -n '^PYTHON_BIN=' "$F" || echo " (no PYTHON_BIN line)" +if grep -q '^PYTHON_BIN=' "$F"; then + sed -i.tmp "s|^PYTHON_BIN=.*|PYTHON_BIN=$NEW|" "$F" && rm -f "$F.tmp" +else + echo "PYTHON_BIN=$NEW" >> "$F" +fi +echo "after:"; grep -n '^PYTHON_BIN=' "$F" +# 顺手核对所有以 / 或 ~ 开头的路径值 +echo "--- 其他路径值核对 ---" +grep -E '^[A-Z_]+=(/|~|\$HOME)' "$F" | while IFS='=' read -r k v; do + p="${v/#\~/$HOME}"; p="${p/#\$HOME/$HOME}" + [ -e "$p" ] && echo "ok $k=$v" || echo "MISS $k=$v" +done diff --git a/scripts/gate_summary.py b/scripts/gate_summary.py new file mode 100644 index 0000000..d4690ab --- /dev/null +++ b/scripts/gate_summary.py @@ -0,0 +1,49 @@ +"""每个已结束 job 的四道门一行摘要(看 FAIL 到底败在哪一门、差多少)。 + + python3 scripts/gate_summary.py [--run r…] [--state FAIL] +""" +from __future__ import annotations + +import argparse +import json +import sqlite3 +import sys +from collections import Counter +from pathlib import Path + +BENCH = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BENCH)) +from harness import paths # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--run", action="append") + ap.add_argument("--state", default=None) + a = ap.parse_args() + con = sqlite3.connect(paths.DB_PATH) + runs = a.run or [con.execute("select run_id from runs order by created_at desc limit 1").fetchone()[0]] + q = "select run_id, job_id, state, error_class, stage_failed, result from jobs where mode='oracle' and run_id in (%s)" % ",".join("?" * len(runs)) + args = list(runs) + if a.state: + q += " and state=?"; args.append(a.state) + q += " and state in ('PASS','FAIL','UNVERIFIED') order by run_id, job_id" + fg = Counter() + for rid, jid, st, ec, sf, res in con.execute(q, args): + r = json.loads(res) if res else {} + g = r.get("gates", {}) + g1, g2, g3, g4 = (g.get(k, {}) for k in ("G1", "G2", "G3", "G4")) + fg[(st, r.get("failed_gate") or (sf and f"stage:{sf}") or "-")] += 1 + f1 = f"/{','.join(g1.get('failed', []))}" if g1.get("failed") else (f"/miss:{','.join(g1.get('missing', []))}" if g1.get("missing") else "") + cos = g2.get("cos_min") + print(f"{rid} {jid:<46} {st:<10} fg={r.get('failed_gate') or '-':<3} ec={ec or '-':<14} " + f"G1={g1.get('verdict', '-')}{f1} G2={g2.get('verdict', '-')}{'' if cos is None else f'({cos:.4f})'} " + f"G3={g3.get('verdict', '-')}" + + ("" if g3.get("fp32") is None else f"(fp32={g3['fp32']:.2f} q={g3['quant']:.2f} d={g3['delta']:+.2f} tol={g3.get('effective_tolerance', 0):.2f})") + + (f" lat_est={g1.get('static_perf', {}).get('latency_ms_est')}ms" if g1.get("static_perf", {}).get("latency_ms_est") else "")) + print("\nsummary:", dict(fg)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ingest_pool.py b/scripts/ingest_pool.py new file mode 100644 index 0000000..b343562 --- /dev/null +++ b/scripts/ingest_pool.py @@ -0,0 +1,223 @@ +"""数据入池:按 registry/datasets.yaml 的有序来源,把数据集复制进 data/pool///,写 manifest.json,只读。 + + python scripts/ingest_pool.py --dataset imagenet1k [--dataset coco2017 …] [--all-local] + python scripts/ingest_pool.py --dataset imagenet1k --verify # 只校验已入池的 manifest + +复制,不软链——别人的目录随时可能被清。入池后 chmod -R a-w。 +**data/ 必须在本地盘**:JuiceFS 小文件写入实测 ~0.5 s/文件,5 万张要 7 小时;本地盘几分钟。 +bench/data -> /home//bench-data 软链(README §0)。"复制不软链"说的是不要软链到别人的目录。 +pool 布局(各任务评测器依赖): + imagenet1k//images/_